#!/usr/bin/env python3
"""
Scan all files in the final directory for remaining Vietnamese text.
Reports files with Vietnamese content, showing context lines.

Vietnamese detection: Checks for lines containing Vietnamese characters
(ă, â, đ, ê, ô, ơ, ư, ă, â, ê, ô, ơ, ư, ỳ, ý, ỷ, ỹ, v.v.)
that are NOT part of the standard page markers, NOT Myanmar-only content.
"""

import os
import re
import sys

FINAL_DIR = "/home/tuan-nguyen/.openclaw/workspace/Chuan Muc Sadi/extracted/final"

# Vietnamese characters (lowercase + uppercase)
VIETNAMESE_CHARS = set(
    "ăâđêôơưừửữựấậầẩẫắặằẳẵéèẻẽẹêếềểễệíìỉĩịóòỏõọôốồổỗộơớờởỡợúùủũụưứừửữựýỳỷỹỵ"
    "ĂÂĐÊÔƠƯỪỬỮỰẤẬẦẪẮẶẰẲẴÉÈẺẼẸÊẾỀỂỄỆÍÌỈĨỊÓÒỎÕỌÔỐỒỔỖỘƠỚỜỞỠỢÚÙỦŨỤƯỨỪỬỮỰÝỲỶỸỴ"
)

# Allowlist: patterns that are OK to have Vietnamese in
ALLOWLIST_PATTERNS = [
    r'^\[Trang PDF\]',
    r'^## Trang',
    r'^\*\*Sādhu\*\*',
    r'^---$',
    r'^\s*$',
    r'^# ',
    r'^## ',
    r'^### ',
]

def has_vietnamese(text):
    """Check if text contains Vietnamese characters."""
    for ch in text:
        if ch in VIETNAMESE_CHARS:
            return True
    return False

def is_allowlisted(line):
    """Check if line matches allowlist patterns."""
    for pat in ALLOWLIST_PATTERNS:
        if re.match(pat, line):
            return True
    return False

def scan_file(filepath):
    """Scan a single file for Vietnamese text and return report."""
    results = []
    with open(filepath, 'r', encoding='utf-8') as f:
        lines = f.readlines()
    
    # First pass: check if file is mostly Vietnamese vs Myanmar
    total_nonempty = 0
    vi_lines = 0
    vi_allowlisted = 0
    suspicious_lines = []
    
    for i, line in enumerate(lines):
        stripped = line.rstrip('\n')
        if not stripped.strip():
            continue
        total_nonempty += 1
        
        # Check for Pāḷi diacritics that look like Vietnamese
        # Many Pāḷi texts use: ā ī ū ṃ ṅ ñ ṭ ḍ ṇ ḷ ṃ ṃ → these are NOT Vietnamese
        pali_chars = set("āīūṅñṭḍṇḷṃṁ")
        
        # Only flag if it has Vietnamese-exclusive characters (not Pāḷi diacritics)
        viet_chars_in_line = set(ch for ch in stripped if ch in VIETNAMESE_CHARS)
        
        if viet_chars_in_line:
            if is_allowlisted(stripped):
                vi_allowlisted += 1
            else:
                # Check if this is actually Vietnamese text or just a few chars
                # Vietnamese text typically has multiple accent marks
                score = len(viet_chars_in_line)
                if score >= 2 or (score >= 1 and any(c in "đĐ" for c in stripped)):
                    suspicious_lines.append((i + 1, stripped))
                    vi_lines += 1
    
    # Calculate percentage
    if total_nonempty == 0:
        pct = 0
    else:
        pct = (len(suspicious_lines) / total_nonempty) * 100
    
    return {
        'file': os.path.basename(filepath),
        'total_lines': len(lines),
        'nonempty_lines': total_nonempty,
        'vi_lines': vi_lines,
        'vi_pct': round(pct, 1),
        'suspicious': suspicious_lines,
    }

def scan_all():
    """Scan all files in the final directory."""
    files = sorted(
        [f for f in os.listdir(FINAL_DIR) if f.endswith('.md')],
        key=lambda x: int(re.search(r'(\d+)', x).group(1)) if re.search(r'(\d+)', x) else 0
    )
    
    all_reports = []
    total_vi_files = 0
    
    print("=" * 80)
    print(f"{'FILE':<35} {'VI%':<8} {'VI Lines':<10} {'Status'}")
    print("=" * 80)
    
    for fname in files:
        fpath = os.path.join(FINAL_DIR, fname)
        report = scan_file(fpath)
        all_reports.append(report)
        
        if report['vi_lines'] > 0:
            total_vi_files += 1
            status = "⚠️  HAS VIETNAMESE"
        else:
            status = "✅ OK"
        
        print(f"{fname:<35} {report['vi_pct']:<8} {report['vi_lines']:<10} {status}")
    
    print("\n" + "=" * 80)
    print(f"Total files: {len(files)}")
    print(f"Files with Vietnamese: {total_vi_files}")
    print(f"Files clean: {len(files) - total_vi_files}")
    print("=" * 80)
    
    # Now show details for files with Vietnamese
    print("\n\n=== DETAILED REPORT ===")
    for report in all_reports:
        if report['suspicious']:
            print(f"\n--- {report['file']} ({report['vi_pct']}% Vietnamese lines) ---")
            for lineno, line in report['suspicious'][:20]:  # Show first 20
                context_start = max(0, lineno - 2)
                print(f"  Line {lineno}: {line[:120]}")

if __name__ == "__main__":
    scan_all()