import re
import difflib
from collections import Counter

def extract_pure(text):
    text = re.sub(r'[A-Za-z0-9\s\n\r\t\.,\-()\[\]{}<>/"\':;!?_=+\*&\^%\$#@~`\\|]', '', text)
    text = re.sub(r'[àáảãạâầấẩẫậăằắẳẵặèéẻẽẹêềếểễệìíỉĩịòóỏõọôồốổỗộơờớởỡợùúủũụưừứửữựỳýỷỹỵđÀÁẢÃẠÂẦẤẨẪẬĂẰẮẲẴẶÈÉẺẼẸÊỀẾỂỄỆÌÍỈĨỊÒÓỎÕỌÔỒỐỔỖỘƠỜỚỞỠỢÙÚỦŨỤƯỪỨỬỮỰỲÝỶỸỴĐ]', '', text)
    return text

std = extract_pure(open('./An Duc Tam Bao/tam_bao_021_050_standard.txt', 'r', encoding='utf-8').read())
tst = extract_pure(open('./An Duc Tam Bao/tam_bao_021_050.md', 'r', encoding='utf-8').read())

matcher = difflib.SequenceMatcher(None, std, tst)
replace_ops = []
delete_ops = []
insert_ops = []

for tag, i1, i2, j1, j2 in matcher.get_opcodes():
    if tag == 'replace':
        replace_ops.append(f"{std[i1:i2]} -> {tst[j1:j2]}")
    elif tag == 'delete':
        if len(std[i1:i2]) > 0: delete_ops.append(std[i1:i2])
    elif tag == 'insert':
        if len(tst[j1:j2]) > 0: insert_ops.append(tst[j1:j2])

print("Top Replacements (Standard -> Extracted):")
for k, v in Counter(replace_ops).most_common(20):
    print(f"{v} times: {k}")

print("\nTop Deletions from standard (Lost in Extracted):")
for k, v in Counter(delete_ops).most_common(10):
    print(f"{v} times: {k}")

print("\nTop Insertions in extracted (Invented by AI):")
for k, v in Counter(insert_ops).most_common(10):
    print(f"{v} times: {k}")