
    j'                     R   d Z ddlZddlZddlZddlZej
                  j                  ej
                  j                  ej
                  j                  e	            ddd      Z
ej
                  j                  e
      sdZ
 ee
      5 Z ej                  e      Zddd       g dZd Zd	 Zd
 Zd Zedk(  rd eej.                        dk  r+ edej.                  d    d        ej2                  d        eej.                  d   ej.                  d          yy# 1 sw Y   xY w)uA  
pre-process.py — Xử lý cơ học văn bản Myanmar trước khi gửi cho Editor AI.
Tất cả các sửa lỗi ở đây là mechanical (regex-based), không cần AI.
Tiết kiệm 50-60% token so với việc để Editor AI tự sửa.

Usage:
  python3 pre-process.py <extracted_input.md> <merged_output.md>
    Nz..datazocr-mechanical-patterns.jsonzG/home/tuan-nguyen/.openclaw/workspace/data/ocr-mechanical-patterns.json))u   တို ့u   တို့)u   သော ည်u   သောည်)u   သော် လည်းu   သော်လည်း)u   ​ )u   ‌r   )u   ‍r   )u   ၁ဝu   ၁၀)u   ၂ဝu   ၂၀)u   ၃ဝu   ၃၀)u   ၄ဝu   ၄၀)u   ၅ဝu   ၅၀)u   ၆ဝu   ၆၀)u   ၇ဝu   ၇၀)u   ၈ဝu   ၈၀)u   ၉ဝu   ၉၀)u   ** ။u   ။ **c                    | j                  d      }g }d}d}|D ]@  }|j                         }|s'|r|j                  |       d}|j                  d       =|j                  d      s|j                  d      r'|r|j                  |       d}|j                  |       |j                  d      s|j                  d      r'|r|j                  |       d}|j                  |       t	        |xr |d   |v       }|j                  d	      s|j                  d
      rd}|r!|d|z   z  }|s|j                  |       d}*|s|}0|j                  |       C |r|j                  |       dj                  |      S )u   
    Gộp dòng: nối dòng không kết thúc bằng dấu kết thúc câu.
    Lưu ý: း (visarga) KHÔNG phải dấu kết thúc câu.
    
r   u   ။၊၏၌## PAGE#z---z***z***T )splitstripappend
startswithboolendswithjoin)textlinesresultbufferSENTENCE_ENDlinesends_sentences           O/home/tuan-nguyen/.openclaw/workspace/003-tang-chi-bo-giang-giai/pre-process.pymerge_linesr      sW   
 JJtEFF!L  JJLv}}V,rfMM"x<<	"all3&7v}}V,rfMM$<<!,,u"5v}}V,rfMM$Q81R5L#89::dqzz# McAgFf%FMM$1 4 v}}V$99V    c                    g }d}| D ]%  }|j                         }|r"|j                  d      s|j                  d      r|j                  |       |}Ld}|r2|j                         j	                  d      r|j                  d      sd}t        j                  d|      rd}t        j                  d	|      rd}|r7|j                         j	                  d
      rt        j                  d|      rd}|r&|r$|d   j                         r|j                  d       |j                  |       |}( dj                  |      S )z;Add blank lines between logical paragraphs for readability.r   r   r   F][Tu   ^[၀-၉]+\s*။u   ^\*\*[၀-၉]+\s*။\*\*u   ။u   ^[၀-၉(\[]r	   r   )r   r   r   rstripr   rematchr   )r   outprevr   r   
need_breaks         r   _unused_add_paragraph_breaksr'   G   s   
CD JJLALL+q||C/@JJtQd
 DKKM**3/S8IJ 88(!,J 880!4J DKKM**51bhh?OQR6SJ##b'--/JJrN

456 99S>r   c                     d}| }t         D ]2  \  }}||v s|j                  |      }|j                  ||      }||z  }4 t        D ]C  }|d   |d   }}|r|r||k(  r||v s|j                  |      }|j                  ||      }||z  }E ||fS )z8Apply mechanical OCR fixes from DB patterns + hardcoded.r   findreplace)	HARDCODEDcountr*   MECHANICAL_PATTERNS)r   fixedr   r)   r*   r,   pats          r   apply_mechanical_fixesr0   i   s    EF" g6>LL&E^^D'2FUNE	
 # FS^g7dgox6>LL&E^^D'2FUNE 5=r   c           	         t        | dd      5 }|j                         }d d d        t              }t        |      }t	        |      \  }}t        |dd      5 }|j                  |       d d d        t        |      }|dkD  r||z  dz  nd}	t        d| d| d	|	d
d       t        d|        |||fS # 1 sw Y   xY w# 1 sw Y   XxY w)Nrzutf-8)encodingwr   d   zPre-process: z -> z chars (z.0fz%)z!Lines: merged, Mechanical fixes: )openreadlenr   r0   writeprint)

input_pathoutput_pathfraw	src_charsmergedcleaned	fix_count	out_charsratios
             r   pre_processrE   |   s    	j#	0 AffhCIF/7GY	k3	1 Q	GI-6]Y"S(E	M)D8E#;b
IJ	-i[
9:i** 
 s   B7C7C C__main__   zUsage: z <input.md> <output.md>      )__doc__sysr"   jsonospathr   dirnameabspath__file__PATTERNS_PATHexistsr6   r=   loadr-   r+   r   r'   r0   rE   __name__r8   argvr:   exit r   r   <module>rY      s    RWW__RWW__X-FG!6+IK	ww~~m$]M	- 'A#$))A,'	%PD&+ z
388}q}$;<=SXXa[)	 s' 's   DD&