
    ,-jq*                        d Z ddlZddlZddlZddlZddlZddlmZ  ej                  dej                         ej                  dej                         ej                  dej                        gZ
 ej                  d       ej                  d       ej                  d	      gZd
ZdZdZd Zd Zd Zd ZefdZd Zd ZddZddZd Zedk(  r e        yy)u  
Trích xuất text từ JSON Cloud Vision → Markdown (V3 — Split multi-line blocks).
Khắc phục vấn đề layout hỗn loạn: sắp xếp paragraph theo Y (trên→dưới),
sau đó X (trái→phải), group theo hàng dựa trên Y overlap.
Đặc biệt: tách các block nhiều dòng thành từng dòng riêng để xử lý TOC.

Usage:
  python3 json_to_markdown_v3.py <json_dir> <output_dir> [--dry-run] [--debug]
Example:
  python3 json_to_markdown_v3.py "010-pali-thaykha/ocr/raw/" "010-pali-thaykha/extracted/"
    N)defaultdictzscanned\s+withzcam\s*scannerzCS\s*CamScanneru    ^စာမျက်နှာ\s*$u2   ^မာတိကာခေါင်းစဉ်\s*$u&   ^အကြောင်းအရာ\s*$gQ?g)\(?333333?c                    d}g }d}| j                  dg       D ]  }|j                  dg       D ]  }|j                  dd      }||z  }||z  }|j                  di       }|j                  d      sC|d   d   }|dv r|d	z  }|d	z  }Z|d
v s_|dz  }|j                         r|j                  |j                                d}  |j                         r|j                  |j                                ||fS )zExtract text from a paragraph, tracking line breaks.
    Returns (full_text, line_texts) where line_texts is list of lines.
     wordssymbolstextpropertydetectedBreaktype)SPACE
SURE_SPACE )EOL_SURE_SPACE
LINE_BREAK
)getstripappend)	parar	   linescurrent_linewordsymcharpropbts	            D/home/tuan-nguyen/.openclaw/workspace/scripts/json_to_markdown_v3.py"extract_paragraph_text_with_breaksr   /   s    DEL"% &88Ir* 	&C7762&DDLDD L77:r*Dxx(/*6200CKD C'L;;DLD#))+\%7%7%9:#%L	&&& \'')*;    c                     | j                  di       j                  dg       }|syt        d |D              }t        d |D              }t        d |D              }t        d |D              }||||fS )z9Get (x_min, y_min, x_max, y_max) from normalizedVertices.boundingBoxnormalizedVertices)r   r   r   r   c              3   @   K   | ]  }|j                  d d        ywxr   Nr   .0vs     r   	<genexpr>z#get_bounding_box.<locals>.<genexpr>U        -!c1-   c              3   @   K   | ]  }|j                  d d        ywyr   Nr'   r(   s     r   r+   z#get_bounding_box.<locals>.<genexpr>V   r,   r-   c              3   @   K   | ]  }|j                  d d        ywr%   r'   r(   s     r   r+   z#get_bounding_box.<locals>.<genexpr>W   r,   r-   c              3   @   K   | ]  }|j                  d d        ywr/   r'   r(   s     r   r+   z#get_bounding_box.<locals>.<genexpr>X   r,   r-   )r   minmax)objvertsx_miny_minx_maxy_maxs         r   get_bounding_boxr;   P   su    GGM2&**+?DE-u--E-u--E-u--E-u--E5%''r    c                 @    t         D ]  }|j                  |       s y y)z%Check if text is a scanner watermark.TF)SCANNER_PATTERNSsearch)r	   pats     r   is_scanner_artifactr@   \   s%     ::d r    c                     | t         kD  S )z%Check if paragraph is in footer zone.)FOOTER_Y_MIN)r8   s    r   is_footer_artifactrC   d   s    <r    c                     t        | |      }t        ||      }t        d||z
        }t        || z
  d      }t        ||z
  d      }	t        ||	      }
||
z  S )z,Check if two Y ranges overlap significantly.r   gMbP?)r4   r3   )y1_miny1_maxy2_miny2_max	thresholdoverlap_startoverlap_endoverlaprange1range2	min_ranges              r   	y_overlaprP   i   sb    'Mff%K![=01G&%(F&%(FFF#IYr    c           	         g }| D ]  \  }}}}}|j                  d      }|D cg c]#  }|j                         s|j                         % }}t        |      dk  r|j                  |||||r|d   n|f       tt        |      }	||z
  |	z  }
t	        |      D ](  \  }}|||
z  z   }||
z   }|j                  |||||f       *  |S c c}w )a  Split multi-line paragraphs into individual line entries.
    
    For paragraphs with multiple lines (containing 
 in text), 
    estimate Y position of each line and create separate entries.
    
    Returns list of (x_min, x_mid, x_max, text_line) for each line fragment.
    r      r   )splitr   lenr   	enumerate)
paragraphsresultr7   r8   r9   r:   r	   r   l	num_linesline_heightiline
line_y_min
line_y_maxs                  r   split_multi_line_blockr_   v   s     F,6 L(ueUD

4 $)7qQWWY77u:?MM5%5uQxdST E
I 5=I5K$U+ L4"Q_4
'+5
uj%TJKLL" M 8s
   CCc                    | sg S t        | d       }g }|d   g}|d   d   }|d   d   }|dd D ]r  }|\  }}}}	}t        ||||	      }
|
dkD  r*|j                  |       t        ||      }t	        ||	      }H|j                  d        |j                  |       |g}|}|	}t |r$|j                  d        |j                  |       |S )	zGroup paragraphs into rows based on Y overlap.
    
    Each paragraph is (x_min, y_min, x_max, y_max, text).
    Returns list of rows, each row is a list of paragraphs sorted by X.
    c                     | d   | d   fS )NrR   r    ps    r   <lambda>z,group_paragraphs_into_rows.<locals>.<lambda>   s    QqT1Q4L r    keyr   rR      Nc                     | d   S Nr   rb   rc   s    r   re   z,group_paragraphs_into_rows.<locals>.<lambda>   s
    1Q4 r    c                     | d   S rj   rb   rc   s    r   re   z,group_paragraphs_into_rows.<locals>.<lambda>   s
    qt r    )sortedrP   r   r3   r4   sort)rV   sorted_parasrowscurrent_rowcurrent_y_mincurrent_y_maxr   _r8   r:   rL   s              r   group_paragraphs_into_rowsrt      s     	 **@ALD?#K OA&M OA&MQR  " $5!UAM=%GQ;t$u5Mu5M0KK$&K!M!M"  ^,K Kr    c           	      t   | j                  dg       }g }|D ]  }|j                  d      dvr|j                  dg       D ]a  }t        |      \  }}|j                         }	|	s$t        |      \  }
}}}t	        |	      r@t        |      rL|j                  |
||||	f       c  |sd| dS t        |      }t        |      }d| dg}|D ]  }g }d}t        |      D ]g  \  }\  }
}}}}|dkD  rD|
|z
  }|d	kD  r|j                  d
       n(|dkD  r|j                  d       n|j                  d       |j                  |       |}i |j                  dj                  |              |j                  d       |r+|j                  ddt        |       dt        |       d       dj                  |      S )zBConvert a single page's Cloud Vision data to sorted Markdown text.blocks	blockType)NTEXTrV   ## PAGE    

*(trang trống)*
r   r   r   z    gQ?z  r   rR   z<!-- z rows, z fragments -->r   )r   r   r   r;   r@   rC   r   r_   rt   rU   joininsertrT   )	page_datapage_numdebugrv   all_paragraphsblockr   r	   rs   text_strippedr7   r8   r9   r:   line_fragmentsro   output_linesrow	row_texts
prev_x_maxr[   gaps                         r   page_to_textr      s   ]]8R(F N O99[!7IIlB/ 	OD8>GD! JJLM )9$)?&E5%"=1!%(!!5%}"MN	O	O* (#:;; ,N;N &n5D xj)2.L 0	
5>s^ 	1A1ueUD1uj(:$$V,4Z$$T*$$S)T"J	 	BGGI./#0& As4yk^9L8M^\]99\""r    c                 T   t        | dd      5 }t        j                  |      }ddd       t        j                  j                  t        j                  j                  |             d   }t        j                  j                  || d      }g }j                  dg       }	t        |	      D ]  \  }
}||
z   }|j                  di       }|j                  d	g       }|rt        |d   ||
      }n2|j                  dd      j                         }|r
d| d| d}nd| d}|j                  |        t        j                  |d       t        |dd      5 }|j                  dj                  |             ddd       |t        |	      fS # 1 sw Y   nxY w# 1 sw Y   #xY w)u,   Process one JSON file → one Markdown file.rzutf-8)encodingNr   z.md	responsesfullTextAnnotationpagesr   r	   r   ry   z

r   rz   T)exist_okw)openjsonloadospathsplitextbasenamer{   r   rU   r   r   r   makedirswriterT   )	json_path
output_dir
start_pager   fdatar   out_path
page_textsr   r[   responser~   ftar   	page_textraw_texts                    r   json_to_markdownr      s   	iw	/ 1yy| ww 0 0 ;<Q?Hww||J8*C(89HJb)I + %8>ll/4$$U1XxuEIwwvr*002H&xjXJbA	&xj0GH	)$%  KK
T*	hg	. '!			*%&' S^##; 4' 's   F!FFF'c                    
 t        t        j                        dk  r t        d       t        j                  d       t        j                  d   } t        j                  d   }dt        j                  v }dt        j                  v }t        t        j                  t        j                  j                  | d            d 	      }|s#t        d
|         t        j                  d       t        dt        |       d       d}|D ]  }t        |||dz   |      \  }}||z  }|rt        |d      5 
dj                  
fdt        d      D              }	d d d        t        dd        t        dt        j                  j                  |       d| d       t        d        t        	       t        d| d       t        dt        j                  j                  |       d| d        |sGt        dt        |       dt        j                  j                  |              t        d| d       y y # 1 sw Y   xY w) Nrh   zSUsage: python3 json_to_markdown_v3.py <json_dir> <output_dir> [--dry-run] [--debug]rR      z	--dry-runz--debugz*.jsonc                     t        t        j                  dt        j                  j                  |             j                  d            S )Nzoutput-(\d+)-torR   )intrer>   r   r   r   group)r   s    r   re   zmain.<locals>.<lambda>-  s2    c"))$68H8H8KLRRSTUV r    rf   u,   ❌ Không tìm thấy file JSON nào trong u   📄 Tìm thấy z
 file JSONr   r   r   r   c              3   >   K   | ]  }j                           y w)N)readline)r)   rs   r   s     r   r+   zmain.<locals>.<genexpr>=  s     !B1!**,!Bs      r   z<============================================================u   📄 z (u    trang) — PREVIEW:z... (dry-run, file saved at )u      ✅ z trang)u   
🎉 Done! u    file .md → u      Tổng: z trang)rT   sysargvprintexitrl   globr   r   r{   r   r   ranger   abspath)json_dirr   dry_runr   
json_filestotal_pagesjfr   r   previewr   s             @r   mainr   !  s   
388}qcdxx{H!JSXX%G!E		"'',,x23VJ
 <XJGH	c*o.j
9:K J*2z;?RWX%uh$ C''!Bb	!BBCBvh- E"''**845Rw>RSTVH'N0
!<=GBGG,,X67r%HIJ c*o.nRWW__Z=X<YZ[K=/0 C Cs   3$H>>I	__main__)F)__doc__r   r   r   r   r   collectionsr   compile
IGNORECASEr=   HEADER_LABELS_MYHEADER_Y_MAXrB   Y_OVERLAP_THRESHOLDr   r;   r@   rC   rP   r_   rt   r   r   r   __name__rb   r    r   <module>r      s  
 
 
   	 #
 BJJ "--0BJJ/BJJ!2==1  BJJ23BJJDEBJJ89    
B	( 
 9L 
<%P@#F$D'1T zF r    