#!/usr/bin/env python3
"""
Xóa tất cả URL dhammadownload.com khỏi file Markdown đã extract.
Các pattern được phát hiện trong OCR output:
  - http://www.dhammadownload.com
  - http://www.dhhaownload.com  
  - http://www.ghammadownload.com
  - tp://www.hadownload.com
  - Các fragment như: dhammadownload, hadownload, ammadownload

Usage:
  python3 clean_urls.py <extracted_dir>
  python3 clean_urls.py <extracted_dir> --dry-run
"""
import os, sys, re, glob

URL_PATTERNS = [
    # Any line containing http:// or https:// (even OCR-damaged)
    re.compile(r'https?://', re.IGNORECASE),
    # Common URL fragments from OCR
    re.compile(r'\bwww\.', re.IGNORECASE),
    re.compile(r'\bdhamm?[a-z]*download\b', re.IGNORECASE),
    re.compile(r'\b[a-z]+ownload\.?com\b', re.IGNORECASE),
    # Lines that are purely URL artifacts
    re.compile(r'^[a-z]+download\.?com$', re.IGNORECASE),
    re.compile(r'^[a-z]+ownload$', re.IGNORECASE),
]


def clean_line(line):
    """Check if line matches URL patterns and should be removed."""
    stripped = line.strip()
    if not stripped:
        return None  # keep empty lines
    
    for pat in URL_PATTERNS:
        if pat.search(stripped):
            return None  # remove this line
    
    return line


def clean_file(filepath, dry_run=False):
    """Remove URL lines from a single file."""
    with open(filepath, 'r', encoding='utf-8') as f:
        lines = f.readlines()
    
    new_lines = []
    removed = 0
    for line in lines:
        result = clean_line(line)
        if result is None:
            removed += 1
        else:
            new_lines.append(result)
    
    if removed == 0:
        return 0
    
    if not dry_run:
        with open(filepath, 'w', encoding='utf-8') as f:
            f.writelines(new_lines)
    
    return removed


def main():
    if len(sys.argv) < 2:
        print("Usage: python3 clean_urls.py <extracted_dir> [--dry-run]")
        sys.exit(1)
    
    target_dir = sys.argv[1]
    dry_run = '--dry-run' in sys.argv
    
    md_files = sorted(glob.glob(os.path.join(target_dir, '*.md')))
    
    if not md_files:
        print(f"❌ Không tìm thấy file .md trong {target_dir}")
        sys.exit(1)
    
    total_removed = 0
    files_cleaned = 0
    
    for fpath in md_files:
        removed = clean_file(fpath, dry_run=dry_run)
        if removed > 0:
            files_cleaned += 1
            total_removed += removed
            fname = os.path.basename(fpath)
            print(f"  ✅ {fname}: {removed} dòng URL bị xóa")
    
    action = "[DRY RUN] Sẽ xóa" if dry_run else "Đã xóa"
    print(f"\n{action} {total_removed} dòng URL trong {files_cleaned}/{len(md_files)} files")


if __name__ == '__main__':
    main()