#!/usr/bin/env python3
"""
Gộp dòng bị ngắt trong extracted markdown files.
Quy tắc:
- Dòng KHÔNG kết thúc bằng ။ hoặc ၊ → nối với dòng sau (cách 1 space)
- Dòng kết thúc bằng ။ hoặc ၊ → giữ nguyên
- Dòng trống → giữ nguyên
- ## PAGE X → giữ nguyên
"""
import glob
import os
import re

DIR = "/home/tuan-nguyen/.openclaw/workspace/014-alahan-srilanka/extracted"

SENTENCE_END = re.compile(r'[။၊]\s*$')
PAGE_MARKER = re.compile(r'^## PAGE\s')

files = sorted(glob.glob(os.path.join(DIR, "output-*.md")))
total_before = 0
total_after = 0

for fpath in files:
    with open(fpath, 'r') as f:
        lines = f.readlines()
    
    total_before += len(lines)
    merged = []
    i = 0
    
    while i < len(lines):
        line = lines[i]
        stripped = line.rstrip('\n')
        
        # Keep page markers and empty lines as-is
        if PAGE_MARKER.match(stripped) or stripped == '':
            merged.append(line)
            i += 1
            continue
        
        # If line ends with sentence-ending punctuation, keep as-is
        if SENTENCE_END.search(stripped):
            merged.append(line)
            i += 1
            continue
        
        # Otherwise, merge with next lines until sentence end or page marker
        buf = stripped.lstrip()
        i += 1
        
        while i < len(lines):
            next_line = lines[i].rstrip('\n')
            
            # Stop at page marker or empty line
            if PAGE_MARKER.match(next_line) or next_line == '':
                break
            
            # Append with space
            buf += ' ' + next_line.lstrip()
            i += 1
            
            # Stop if we reached sentence end
            if SENTENCE_END.search(next_line):
                break
        
        merged.append(buf + '\n')
    
    # Write back
    with open(fpath, 'w') as f:
        f.writelines(merged)
    
    total_after += len(merged)
    if len(merged) != len(lines):
        print(f"  {os.path.basename(fpath)}: {len(lines)} → {len(merged)} dòng ({len(lines)-len(merged)} merged)")

# Regenerate full-merged
all_md = []
for fpath in sorted(glob.glob(os.path.join(DIR, "output-*.md"))):
    with open(fpath) as f:
        all_md.append(f.read())

with open(os.path.join(DIR, "full-merged.md"), 'w') as f:
    f.write('# သီဟိုဠ်ခေတ် စံတော်ဝင် အရိယာများ — OCR Raw Extraction (V9)\n\n')
    f.write('\n'.join(all_md))

print(f"\nTổng: {total_before} → {total_after} dòng ({total_before - total_after} merged)")
print(f"full-merged.md regenerated: {os.path.getsize(os.path.join(DIR, 'full-merged.md')):,} bytes")