#!/usr/bin/env python3
import re

def read_file(path):
    with open(path, 'r', encoding='utf-8') as f:
        return f.read()

def extract_myanmar_words(text):
    # Extract sequences of Myanmar characters and possibly digits/punctuation
    # Use regex to find Myanmar words (including attached diacritics)
    # Myanmar Unicode range: U+1000–U+109F, plus U+AA60–U+AA7F? but ignore for now.
    # Also include common punctuation: ၊ ။ ၌ ၍ etc.
    words = re.findall(r'[\u1000-\u109F]+(?:[\u1030-\u1039\u103B-\u103E]*[\u1000-\u109F]*)*', text)
    return words

def clean_and_tokenize(text):
    # Remove non-Myanmar lines
    lines = text.splitlines()
    myanmar_lines = []
    for line in lines:
        if re.search(r'[\u1000-\u109F]', line):
            myanmar_lines.append(line)
    full = ' '.join(myanmar_lines)
    # Tokenize into words (Myanmar sequences)
    words = extract_myanmar_words(full)
    return words

standard = read_file("tam_bao_011_020_standard.txt")
gemini = read_file("tam_bao_011_020.md")
deepseek = read_file("tam_bao_011_020_deepseek.md")

std_words = clean_and_tokenize(standard)
gem_words = clean_and_tokenize(gemini)
deep_words = clean_and_tokenize(deepseek)

print(f"Standard word count: {len(std_words)}")
print(f"Gemini word count: {len(gem_words)}")
print(f"Deepseek word count: {len(deep_words)}")

# Calculate Jaccard similarity (intersection over union)
set_std = set(std_words)
set_gem = set(gem_words)
set_deep = set(deep_words)

jaccard_gem = len(set_std & set_gem) / len(set_std | set_gem) if len(set_std | set_gem) > 0 else 0
jaccard_deep = len(set_std & set_deep) / len(set_std | set_deep) if len(set_std | set_deep) > 0 else 0

print(f"\nJaccard similarity (word set):")
print(f"Gemini vs Standard: {jaccard_gem*100:.2f}%")
print(f"Deepseek vs Standard: {jaccard_deep*100:.2f}%")

# Also compute overlap (percentage of standard words present in gemini)
overlap_gem = len([w for w in std_words if w in gem_words]) / len(std_words) * 100 if std_words else 0
overlap_deep = len([w for w in std_words if w in deep_words]) / len(std_words) * 100 if std_words else 0
print(f"\nOverlap (standard words found in other):")
print(f"Gemini covers {overlap_gem:.2f}% of standard words")
print(f"Deepseek covers {overlap_deep:.2f}% of standard words")

# Check first 20 words
print("\nFirst 20 words of Standard:", ' '.join(std_words[:20]))
print("First 20 words of Gemini:", ' '.join(gem_words[:20]))
print("First 20 words of Deepseek:", ' '.join(deep_words[:20]))