Ë
    çÍ:j*  ã                   óv   — d dl Z d dlZd dlZd dlZd dlmZ d dlmZmZm	Z	m
Z
mZmZ d dlmZ dZ G d„ de«      Zy)é    N)ÚPIPE)Ú_java_optionsÚconfig_javaÚfind_dirÚ	find_fileÚfind_jarÚjava)Ú
TokenizerIz!https://nlp.stanford.edu/softwarec                   óf   ‡ — e Zd ZdZdZ	 	 	 	 	 	 	 	 	 	 	 	 d
d„Zd„ Zˆ fd„Zd„ Zd„ Z	d„ Z
dd	„Zˆ xZS )ÚStanfordSegmenteru[  Interface to the Stanford Segmenter

    If stanford-segmenter version is older than 2016-10-31, then path_to_slf4j
    should be provieded, for example::

        seg = StanfordSegmenter(path_to_slf4j='/YOUR_PATH/slf4j-api.jar')

    >>> from nltk.tokenize.stanford_segmenter import StanfordSegmenter
    >>> seg = StanfordSegmenter() # doctest: +SKIP
    >>> seg.default_config('zh') # doctest: +SKIP
    >>> sent = u'è¿™æ˜¯æ–¯å�¦ç¦�ä¸­æ–‡åˆ†è¯�å™¨æµ‹è¯•'
    >>> print(seg.segment(sent)) # doctest: +SKIP
    è¿™ æ˜¯ æ–¯å�¦ç¦� ä¸­æ–‡ åˆ†è¯�å™¨ æµ‹è¯•
    <BLANKLINE>
    >>> seg.default_config('ar') # doctest: +SKIP
    >>> sent = u'Ù‡Ø°Ø§ Ù‡Ùˆ ØªØµÙ†ÙŠÙ� Ø³ØªØ§Ù†Ù�ÙˆØ±Ø¯ Ø§Ù„Ø¹Ø±Ø¨ÙŠ Ù„Ù„ÙƒÙ„Ù…Ø§Øª'
    >>> print(seg.segment(sent.split())) # doctest: +SKIP
    Ù‡Ø°Ø§ Ù‡Ùˆ ØªØµÙ†ÙŠÙ� Ø³ØªØ§Ù†Ù�ÙˆØ±Ø¯ Ø§Ù„Ø¹Ø±Ø¨ÙŠ Ù„ Ø§Ù„ÙƒÙ„Ù…Ø§Øª
    <BLANKLINE>
    zstanford-segmenter.jarc                 óT  — t        j                  dt        «       t        j                  t	        d«      t        d¬«       t        j                  dt        «       t        | j                  |ddt        |¬«      }|�t        d	|d
dt        |¬«      }nd }t        j                  j                  d„ ||fD «       «      | _        || _        || _        || _        || _        || _        || _        |	| _        || _        |
€i n|
}
dj                  d„ |
j)                  «       D «       «      | _        y )NÚalwaysz}
The StanfordTokenizer will be deprecated in version 3.2.5.
Please use [91mnltk.parse.corenlp.CoreNLPTokenizer[0m instead.'é   )Ú
stacklevelÚignore©ÚSTANFORD_SEGMENTER© )Úenv_varsÚ
searchpathÚurlÚverbosezslf4j-api.jar)ÚSLF4Jr   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­w©Nr   )Ú.0Ú_s     úu/home/mcse/projects/srt_converter/srt-converter-venv/lib/python3.12/site-packages/nltk/tokenize/stanford_segmenter.pyú	<genexpr>z-StanfordSegmenter.__init__.<locals>.<genexpr>j   s   è ø€ ò -
Ø°a±mŒAñ-
ùs   ‚Šú,c              3   óV   K  — | ]!  \  }}|› d t        j                  |«      › �–— Œ# y­w)ú=N)ÚjsonÚdumps)r   ÚkeyÚvals      r   r   z-StanfordSegmenter.__init__.<locals>.<genexpr>x   s-   è ø€ ò %
Ù+3¨3°ˆsˆe�1”T—Z‘Z “_Ð%Ô&ñ%
ùs   ‚'))ÚwarningsÚsimplefilterÚDeprecationWarningÚwarnÚstrr   Ú_JARÚ_stanford_urlÚosÚpathsepÚjoinÚ_stanford_jarÚ_java_classÚ_modelÚ_sihan_corpora_dictÚ_sihan_post_processingÚ_keep_whitespacesÚ_dictÚ	_encodingÚjava_optionsÚitemsÚ_options_cmd)ÚselfÚpath_to_jarÚpath_to_slf4jÚ
java_classÚpath_to_modelÚpath_to_dictÚpath_to_sihan_corpora_dictÚsihan_post_processingÚkeep_whitespacesÚencodingÚoptionsr   r9   Ústanford_segmenterÚslf4js                  r   Ú__init__zStanfordSegmenter.__init__8   s-  € ô  	×Ñ˜hÔ(:Ô;Ü�‰ÜðZóô
 Øõ	
ô 	×Ñ˜hÔ(:Ô;ä%Ø�I‰IØØ,ØÜØô
Ðð Ð$ÜØØØ8ØÜ!Øô‰Eð ˆEô  ŸZ™ZŸ_™_ñ -
Ø*¨EÐ2ô-
ó 
ˆÔð &ˆÔØ#ˆŒØ#=ˆÔ Ø&;ˆÔ#Ø!1ˆÔØ!ˆŒ
à!ˆŒØ(ˆÔØ˜‘"¨WˆØŸH™Hñ %
Ø7>·}±}³ô%
ó 
ˆÕó    c                 óô  — d}t         j                  j                  d«      r>t         j                  j	                  t         j                  j                  d«      d«      h}d| _        d| _        d| _        |dk(  r
d| _        d}n{|d	k(  rhd
| _        d}d| _        d}	 t        ||t        dd¬«      | _        d}	 t        |t        dd¬«      }t         j                  j	                  ||«      | _        nt        d|› �«      ‚	 t        ||t        dd¬«      | _        y# t        $ r}t        d|z  «      |‚d}~ww xY w# t        $ r}t        d|z  «      |‚d}~ww xY w# t        $ r}t        d|z  «      |‚d}~ww xY w)z©
        Attempt to initialize Stanford Word Segmenter for the specified language
        using the STANFORD_SEGMENTER and STANFORD_MODELS environment variables
        r   r   ÚdataNÚfalseÚarz=edu.stanford.nlp.international.arabic.process.ArabicSegmenterz'arabic-segmenter-atb+bn+arztrain.ser.gzÚzhz%edu.stanford.nlp.ie.crf.CRFClassifierzpku.gzÚtruezdict-chris6.ser.gzF)ÚSTANFORD_MODELS)r   r   r   r   z_Could not find '%s' (tried using env. variables STANFORD_MODELS and <STANFORD_SEGMENTER>/data/)z./data/r   )r   r   r   zMCould not find '%s' (tried using the STANFORD_SEGMENTER environment variable)zUnsupported language )rQ   r   )r.   ÚenvironÚgetÚpathr0   r7   r4   r5   r2   r   r-   ÚLookupErrorr   r3   )r<   ÚlangÚsearch_pathÚmodelrA   ÚeÚ	sihan_dirÚpath_to_sihan_dirs           r   Údefault_configz StanfordSegmenter.default_config|   s¹  € ð ˆÜ�:‰:�>‰>Ð.Ô/ÜŸ7™7Ÿ<™<¬¯
©
¯©Ð7KÓ(LÈfÓUÐVˆKð ˆŒ
Ø#'ˆÔ Ø&-ˆÔ#à�4Š<àOð Ôð >‰Eà�TŠ\ØFˆDÔØˆEØ*0ˆDÔ'à/ˆLðÜ&Ø Ø*Ü%Ø!Ø1ô�”
ð "ˆIðÜ$,ØÜ%Ø!Ø4ô	%Ð!ô ,.¯7©7¯<©<Ð8IÈ9Ó+U�Õ(ô Ð 5°d°VÐ<Ó=Ð=ð	Ü#ØØ&Ü!ØØBôˆD�Køô3 ò Ü!ðPà"ñ#óð ð	ûðûô  ò Ü!ð?ØAJñKóð ðûðûô  ò 	ÜðLØNSñTóð ðûð	úsH   ÂD Â88D: Ä E Ä	D7Ä#D2Ä2D7Ä:	EÅEÅEÅ	E7Å#E2Å2E7c                 ó$   •— t         ‰| �  |«       y r   )ÚsuperÚtokenize)r<   ÚsÚ	__class__s     €r   r_   zStanfordSegmenter.tokenizeÅ   s   ø€ Ü‰Ñ˜ÕrJ   c                 óø   — | j                   d| j                  d| j                  d|g}| j                  �5|j	                  d| j
                  d| j                  d| j                  g«       | j                  |«      }|S )ú ú-loadClassifierú-keepAllWhitespacesú	-textFileú-serDictionaryú-sighanCorporaDictú-sighanPostProcessing)r2   r3   r6   r4   Úextendr7   r5   Ú_execute)r<   Úinput_file_pathÚcmdÚstdouts       r   Úsegment_filezStanfordSegmenter.segment_fileÈ   s†   € ð ×ÑØØ�K‰KØ!Ø×"Ñ"ØØð
ˆð ×#Ñ#Ð/Ø�J‰Jà$Ø—J‘JØ(Ø×,Ñ,Ø+Ø×/Ñ/ðô	ð —‘˜sÓ#ˆàˆrJ   c                 ó&   — | j                  |g«      S r   )Úsegment_sents)r<   Útokenss     r   ÚsegmentzStanfordSegmenter.segmentã   s   € Ø×!Ñ! 6 (Ó+Ð+rJ   c                 ó‚  — | j                   }t        j                  d¬«      \  }| _        t	        j
                  |d«      }dj                  d„ |D «       «      }t        |t        «      r|r|j                  |«      }|j                  |«       |j                  «        | j                  d| j                  d| j                  d| j                  g}| j                  �5|j!                  d	| j"                  d
| j                  d| j$                  g«       | j'                  |«      }t	        j(                  | j                  «       |S )rc   T)ÚtextÚwbú
c              3   ó>   K  — | ]  }d j                  |«      –— Œ y­w)rc   N)r0   )r   Úxs     r   r   z2StanfordSegmenter.segment_sents.<locals>.<genexpr>î   s   è ø€ Ò:¨1˜3Ÿ8™8 AŸ;Ñ:ùs   ‚rd   re   rf   rg   rh   ri   )r8   ÚtempfileÚmkstempÚ_input_file_pathr.   Úfdopenr0   Ú
isinstancer+   ÚencodeÚwriteÚcloser2   r3   r6   r4   rj   r7   r5   rk   Úunlink)r<   Ú	sentencesrE   Ú	_input_fhÚ_inputrm   rn   s          r   rq   zStanfordSegmenter.segment_sentsæ   s  € à—>‘>ˆä+3×+;Ñ+;ÀÔ+FÑ(ˆ	�4Ô(ô —I‘I˜i¨Ó.ˆ	Ø—‘Ñ:°	Ô:Ó:ˆÜ�fœcÔ"¡xØ—]‘] 8Ó,ˆFØ�‰˜ÔØ�‰Ôð ×ÑØØ�K‰KØ!Ø×"Ñ"ØØ×!Ñ!ð
ˆð ×#Ñ#Ð/Ø�J‰Jà$Ø—J‘JØ(Ø×,Ñ,Ø+Ø×/Ñ/ðô	ð —‘˜sÓ#ˆô 	�	‰	�$×'Ñ'Ô(àˆrJ   c                 óP  ‡‡— | j                   }|j                  d|g«       | j                  }|r|j                  d| j                  g«       dj                  t        «      }dd lŠdd l}| j                  Šg d¢}ˆfd„}t        ˆfd„|D «       «      s:|j                  j                  d«      }	 |‰«      }
|	|
k7  rt        d	‰› d
|
› d�«      ‚t        | j                  |¬«       t        || j                  t        t        ¬«      \  }}|j!                  |«      }t        |d¬«       |S )Nz-inputEncodingz-optionsrc   r   )zstanford-segmenterzstanford-corenlpÚnltkc                 óò   •‡— ‰j                  «       }t        | d«      5 Št        ˆfd„d«      D ]  }|j                  |«       Œ 	 d d d «       |j	                  «       S # 1 sw Y   |j	                  «       S xY w)NÚrbc                  ó&   •— ‰ j                  d«      S )Ni   )Úread)Úfs   €r   ú<lambda>z?StanfordSegmenter._execute.<locals>.sha256sum.<locals>.<lambda>#  s   ø€ ¨!¯&©&°«,€ rJ   rJ   )Úsha256ÚopenÚiterÚupdateÚ	hexdigest)ÚfileÚhÚchunkrŒ   Úhashlibs      @€r   Ú	sha256sumz-StanfordSegmenter._execute.<locals>.sha256sum   si   ù€ Ø—‘Ó ˆAÜ�d˜DÓ!ð $ QÜ!Ó"6¸Ó<ò $�EØ—H‘H˜U•Oñ$÷$ð —;‘;“=Ð ÷$ð —;‘;“=Ð ús   Ÿ&AÁA6c              3   ó&   •K  — | ]  }|‰v –— Œ
 y ­wr   r   )r   ÚpÚjar_paths     €r   r   z-StanfordSegmenter._execute.<locals>.<genexpr>'  s   øè ø€ Ò8 Q�1˜”=Ñ8ùs   ƒÚNLTK_SEGMENTER_ALLOW_SHA256uF   
[SECURITY BLOCKED] Unverified Stanford Segmenter JAR detected:
  â†’ z©

This prevents arbitrary code execution via malicious JAR injection.
To allow execution, verify and approve its SHA256 checksum:

  export NLTK_SEGMENTER_ALLOW_SHA256="z"
)rF   r   )Ú	classpathrn   ÚstderrF)r8   rj   r;   r0   r   r–   r.   r1   ÚanyrR   rS   ÚRuntimeErrorr   r9   r	   r   Údecode)r<   rm   r   rE   r;   Údefault_optionsr.   Útrusted_pathsr—   Úuser_checksumÚjar_checksumrn   Ú_stderrr–   rš   s                @@r   rk   zStanfordSegmenter._execute  s  ù€ Ø—>‘>ˆØ�
‰
Ð$ hÐ/Ô0Ø×(Ñ(ˆÙØ�J‰J˜
 D×$5Ñ$5Ð6Ô7àŸ(™(¤=Ó1ˆó 	Ûà×%Ñ%ˆÚJˆô	!ô Ó8¨-Ô8Ô8ØŸJ™JŸN™NÐ+HÓIˆMÙ$ XÓ.ˆLà Ò,Ü"ðØ%˜Jð '=ð >J¸NÈ#ð	Oóð ô 	˜D×-Ñ-°wÕ?äØ˜4×-Ñ-´dÄ4ô
‰ˆ�ð —‘˜xÓ(ˆô 	˜O°UÕ;àˆrJ   )NNNNNNrM   rM   zUTF-8NFz-mx2g)F)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r,   rI   r\   r_   ro   rs   rq   rk   Ú__classcell__)ra   s   @r   r   r       s]   ø„ ñð* $€Dð ØØØØØ#'Ø%Ø ØØØØóB
òHGôRòò6,ò(÷T0rJ   r   )r#   r.   rz   r'   Ú
subprocessr   Únltk.internalsr   r   r   r   r   r	   Únltk.tokenize.apir
   r-   r   r   rJ   r   ú<module>r®      s8   ðó Û 	Û Û Ý ÷÷ õ )à3€ô`˜
õ `rJ   