
    @jR)                        d Z ddlZddlZddlZddlZddlZ ed      Z ej                  d      Z	h dZ
h dZ ej                  d      Z ej                  d       ej                  d	       ej                  d
       ej                  d       ej                  d      gZdZd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zedk(  r e        yy)u  
Cleanup header/footer leaks từ JSON Cloud Vision → Markdown sạch.

Detects and removes:
  - Even pages: page number + book title + separator artifacts
  - Odd pages:  section/chapter running headers (Paṇṇāsaka, vagga titles, etc.)

Strategy: Block-level detection (y-coordinate + pattern) → Text-level prefix removal.

Usage:
  python3 cleanup_header_leaks.py <json_dir> <output_dir>
    Nu!   ၀၁၂၃၄၅၆၇၈၉ဝu(   ^အင်္ဂုတ္တိုရ်>   	   ပထမ   ဆဋ္ဌ   တတိယ   ဒုတိယ   ပဉ္စမ   သတ္တမ   စတုတ္ထ>      ။..*J-./=z	^'?\d+'?$u   ပဏ္ဏာသကu   ^\d+[\-\s]+\S+ဝဂ်u   ^\d+[\-\s]+\S+ပါဠိu   ^\d+[\-\s]+\S+သုတ်u9   ပတ္ထနာနှင့်\s*ပဋိညာဉ်gQ?c                 h    | j                         } | rt        |       dkD  ryt        d | D              S )N   Fc              3   ,   K   | ]  }|t         v   y wN)MYANMAR_DIGITS).0cs     E/home/tuan-nguyen/.openclaw/workspace/scripts/cleanup_header_leaks.py	<genexpr>z)is_myanmar_page_number.<locals>.<genexpr>)   s     1qqN"1s   )striplenalltexts    r   is_myanmar_page_numberr    %   s.    ::<D3t9q=1D111    c                 ^    t        t        j                  | j                  dd                  S )zCOnly matches if text STARTS with book title (not just contains it).  )boolBOOK_TITLE_REsearchreplacer   s    r   is_book_titler)   ,   s#    $$T\\#r%:;<<r!   c                 P    | j                         j                  d      }|t        v S )Nz[](). )r   SECTION_MARKERSr   ts     r   is_section_markerr.   1   s#    

8$Ar!   c                 r    | j                         }|t        v xs t        t        j	                  |            S r   )r   ARTIFACT_MARKERSr%   ARTIFACT_REmatchr,   s     r   is_artifactr3   6   s.    

A  >D):):1)=$>>r!   c                 d    | j                  dd      }t        D ]  }|j                  |      s y y)z7Check if text matches odd-page running header patterns.r#   r$   TF)r(   ODD_HEADER_RESr'   )r   compactpats      r   is_odd_page_headerr8   ;   s5    ll3#G ::g r!   c           	          g }| j                  dg       D ]R  }|j                  dg       D ];  }dj                  d |j                  dg       D              }|j                  |       = T dj                  |      S )N
paragraphswordsr$   c              3   @   K   | ]  }|j                  d d        yw)r   r$   Nget)r   ss     r   r   z%extract_block_text.<locals>.<genexpr>H   s     P1QUU62.P   symbolsr#   )r>   joinappend)blockr;   parawordw_texts        r   extract_block_textrH   D   sv    E		,+ !HHWb) 	!DWWPB8OPPFLL 	!! 88E?r!   c                 p    | j                  di       j                  dg       }|rt        d |D              S y)NboundingBoxnormalizedVerticesc              3   @   K   | ]  }|j                  d d        yw)yr   Nr=   )r   vs     r   r   z"get_block_y_min.<locals>.<genexpr>P   s     -Q155a=-r@   g      ?)r>   min)rD   nvs     r   get_block_y_minrQ   M   s7    	="	%	)	)*>	CB	-"---r!   c                 j  
 | j                  dg       }g }t               }|dz  dk(  }t        |      D ]  \  
}
dkD  r n|j                  d      dvr!t        |      }t	        |      j                  dd      }|t        k\  rQt        |      st        |      st        |      r#|j                  |       |j                  
       |r.t        |      r#|j                  |       |j                  
       t        |      s|dk  s|j                  |       |j                  
        |rt        |      D ]u  \  
}
dkD  r |S |j                  d      dvr"
|v r't	        |      j                  dd      }t        |      sNt        
fd	|D              }	|	se|j                  |       w |S )
z
    Identify header block texts to remove.
    
    Even pages: page number + book title + adjacent artifacts
    Odd pages:  section/chapter running headers + adjacent artifacts
    blocks      	blockType)NTEXTr   
TEXT_BLOCKr#   r$   g?c              3   @   K   | ]  }t        |z
        d k    yw)rU   N)abs)r   piis     r   r   z-get_header_texts_to_remove.<locals>.<genexpr>   s     JBs1r6{a/Js   )r>   set	enumeraterQ   rH   r(   HEADER_Y_CUTOFFr)   r    r.   rC   addr8   r3   any)	page_datapage_numrS   texts_to_removeprimary_indicesis_oddrD   y_mintext_compactadjacentr\   s             @r   get_header_texts_to_removerj   T   s    ]]8R(FOeO\QF f% #5q599[!)MM&)%088bAO# &*@*NRcdpRq""<0" *<8""<0" &54<""<0"3#8 !&) 	9HAu1u  yy%-QQO#-e4<<S"EL<(J/JJ#**<8	9 r!   c                    | s| S | j                  d      }g }d}|dz  dk(  }|rt        |      n	t               }t        |      D ]  \  }}	|	j                         }
|
j	                  dd      }|s|r|s0|j                  |	       B||v rd}Id}|D ]F  }t        |      dkD  st        t        |      t        |            }|d	k\  s6|d
| |d
| k(  sBd}d} n |r|dk  rL|j                  d      rd}t        |      rt        |      dk  rd}t        |      rd}t        |      rd}|r|dk  rt        |      rd}|j                  |	        |r:|d   j                         s'|j                  d       |r|d   j                         s'|r9|d   j                         s&|j                          |r|d   j                         s&dj                  |      S )a  
    Remove header lines from full text.
    
    Strategies:
    1. Exact/partial match against detected header block texts
    2. Prefix check: book title, page numbers, section markers (first 5 lines)
    3. Odd-page header patterns (first 3 lines on odd pages)
    
FrT   rU   r#   r$   Tr   
   NuK   အင်္ဂုတ္တိုရ်ပါဠိတော်နိဿယ   r   )splitr]   r^   r   r(   rC   r   rO   
startswithr    r.   r3   r8   poprB   )	full_textheader_texts_to_removerc   lines
kept_linesremoved_anyrf   header_compactsline_idxlineline_strippedline_compactmatchedhcommons                  r   clean_page_textr      s    OOD!EJK\QF5Kc01QTQVO#E* 3 $

$,,S"5[d# ?*K   	A1vzS.A7R<L&$9QwZ$G"G"&K	  a<&&'tu"%l3L8IQ8N" ."<(" hl'9,'GK$g3 l Z]002q Z]002
Z^113 Z^113 99Z  r!   c                     t        | dd      5 }t        j                  |      }ddd       t        j                  j                  t        j                  j                  |             d   }t        j                  j                  || d      }g }d}t        j                  dg             D ]  \  }	}
||	z   }|
j                  di       }|j                  d	g       }|j                  d
d      j                         }|r-|r+t        |d   |      }|t        |      z  }t        |||      }n|}|j                  d|        |j                  d       |r|j                  |       n|j                  d       |j                  d        t        j                  |d       t        |dd      5 }|j!                  dj                  |             ddd       |t        |j                  dg             |fS # 1 sw Y   xY w# 1 sw Y   4xY w)u2   Process one JSON file → one clean Markdown file.rzutf-8)encodingNr   z.md	responsesfullTextAnnotationpagesr   r$   z## PAGE u   *(trang trống)*T)exist_okwrl   )openjsonloadospathsplitextbasenamerB   r^   r>   r   rj   r   r   rC   makedirswrite)	json_path
output_dir
start_pagefdatar   out_pathru   total_removedr\   responserc   ftar   rs   header_texts
clean_texts                    r   process_json_filer      s   	iw	/ 1yy| ww 0 0 ;<Q?Hww||J8*C(89HEM +r!:; 8>ll/4$GGFB'--/	Y5eAhILS..M(L(KJ"Jxz*+RLL$LL,-R'* KK
T*	hg	. "!			% !" S+r23]BBE >" "s   G70!H7HHc                     t        t        j                        dk  r t        d       t        j                  d       t        j                  d   } t        j                  d   }t        t        j                  t        j                  j                  | d            d       }|s#t        d|         t        j                  d       t        d	t        |       d
       t        dt         d       t                d}d}|D ]  }t        t        j                  dt        j                  j                  |            j                  d            }t!        |||      \  }}}	||z  }||	z  }|	dkD  rdnd}
t        d|
 dt        j                  j                  |       d| d|	 d	        t        dt        |       dt        j                  j#                  |              t        d| d| d       y )Nrn   z>Usage: python3 cleanup_header_leaks.py <json_dir> <output_dir>rU   rT   z*.jsonc                     t        t        j                  dt        j                  j                  |             j                  d            S )Noutput-(\d+)-torU   )intrer'   r   r   r   group)r   s    r   <lambda>zmain.<locals>.<lambda>  s2    c"))$68H8H8KLRRSTUV r!   )keyu,   ❌ Không tìm thấy file JSON nào trong u   📄 Cleanup header leaks từ z file JSON...u      Ngưỡng: y < u,    | Chẵn: page#+book | Lẻ: section titlesr   r   u   ✅u   ▫️z   r#   z (u    trang, xóa )u   
🎉 Done! u    file .md → u      Tổng: z header blocks)r   sysargvprintexitsortedglobr   r   rB   r_   r   r   r'   r   r   r   abspath)json_dirr   
json_filestotal_pagesr   jfbaser   r   removedtags              r   mainr     s   
388}qNOxx{H!J		"'',,x23VJ
 <XJGH	+C
O+<M
JK	//[
\]	GKM X299/1A1A"1EFLLQOP#4RT#J %u {eC5"''**845RwmG9TUVWX 
M#j/*.9T8U
VW	K}M-
OPr!   __main__)__doc__r   r   r   r   r   r]   r   compiler&   r+   r0   r1   r5   r_   r    r)   r.   r3   r8   rH   rQ   rj   r   r   r   __name__ r!   r   <module>r      s      
 89

FG O> bjj& BJJ'(BJJ)*BJJ,-BJJ,-BJJKL 2=
 
?
9xN!b$CN"QJ zF r!   