Ë
    çÍ:ju  ã                   óR   — d Z ddlZddlZddlmZ ddlmZ ddlmZ  G d„ de«      Z	y)a  
This is a NLTK port of the tokenizer used in the NIST BLEU evaluation script,
https://github.com/moses-smt/mosesdecoder/blob/master/scripts/generic/mteval-v14.pl#L926
which was also ported into Python in
https://github.com/lium-lst/nmtpy/blob/master/nmtpy/metrics/mtevalbleu.py#L162
é    N)Úperluniprops)Ú
TokenizerI)Úxml_unescapec            	       óz  — e Zd ZdZ ej
                  d«      dfZ ej
                  d«      dfZ ej
                  d«      dfZ ej
                  d«      d	fZ	 ej
                  d
«      dfZ
 ej
                  d«      d	fZee	e
egZ edj                   e ej"                  d«      «      «      «      Z edj                   e ej"                  d«      «      «      «      Z edj                   e ej"                  d«      «      «      «      Z ej*                  dde«      Z ej*                  dde«      Z ej*                  dde«      Z ej
                  d«      dfZ ej
                  de› de› d�«      d	fZ ej
                  de› de› d�«      dfZ ej
                  de› d�«      dfZeeeegZd„ Zdd„Z	 dd„Z y)ÚNISTTokenizeruT  
    This NIST tokenizer is sentence-based instead of the original
    paragraph-based tokenization from mteval-14.pl; The sentence-based
    tokenization is consistent with the other tokenizers available in NLTK.

    >>> from nltk.tokenize.nist import NISTTokenizer
    >>> nist = NISTTokenizer()
    >>> s = "Good muffins cost $3.88 in New York."
    >>> expected_lower = [u'good', u'muffins', u'cost', u'$', u'3.88', u'in', u'new', u'york', u'.']
    >>> expected_cased = [u'Good', u'muffins', u'cost', u'$', u'3.88', u'in', u'New', u'York', u'.']
    >>> nist.tokenize(s, lowercase=False) == expected_cased
    True
    >>> nist.tokenize(s, lowercase=True) == expected_lower  # Lowercased.
    True

    The international_tokenize() is the preferred function when tokenizing
    non-european text, e.g.

    >>> from nltk.tokenize.nist import NISTTokenizer
    >>> nist = NISTTokenizer()

    # Input strings.
    >>> albb = u'Alibaba Group Holding Limited (Chinese: é˜¿é‡Œå·´å·´é›†å›¢æŽ§è‚¡ æœ‰é™�å…¬å�¸) is a Chinese e-commerce company...'
    >>> amz = u'Amazon.com, Inc. (/ËˆÃ¦mÉ™zÉ’n/) is an American electronic commerce...'
    >>> rkt = u'Rakuten, Inc. (æ¥½å¤©æ ªå¼�ä¼šç¤¾ Rakuten Kabushiki-gaisha) is a Japanese electronic commerce and Internet company based in Tokyo.'

    # Expected tokens.
    >>> expected_albb = [u'Alibaba', u'Group', u'Holding', u'Limited', u'(', u'Chinese', u':', u'é˜¿é‡Œå·´å·´é›†å›¢æŽ§è‚¡', u'æœ‰é™�å…¬å�¸', u')']
    >>> expected_amz = [u'Amazon', u'.', u'com', u',', u'Inc', u'.', u'(', u'/', u'ËˆÃ¦', u'm']
    >>> expected_rkt = [u'Rakuten', u',', u'Inc', u'.', u'(', u'æ¥½å¤©æ ªå¼�ä¼šç¤¾', u'Rakuten', u'Kabushiki', u'-', u'gaisha']

    >>> nist.international_tokenize(albb)[:10] == expected_albb
    True
    >>> nist.international_tokenize(amz)[:10] == expected_amz
    True
    >>> nist.international_tokenize(rkt)[:10] == expected_rkt
    True

    # Doctest for patching issue #1926
    >>> sent = u'this is a fooâ˜„sentence.'
    >>> expected_sent = [u'this', u'is', u'a', u'foo', u'â˜„', u'sentence', u'.']
    >>> nist.international_tokenize(sent) == expected_sent
    True
    z	<skipped>Ú u   â€¨ú z([\{-\~\[-\` -\&\(-\+\:-\@\/])z \1 z([^0-9])([\.,])z\1 \2 z([\.,])([^0-9])z \1 \2z
([0-9])(-)ÚNumberÚPunctuationÚSymbolz[]^\\-]z\\\g<0>z([ -]+)z([z])([z])c                 ó    — | j                   \  }}|j                  ||«      }t        |«      }| j                  \  }}|j                  ||«      }|S )z8Performs the language independent string substituitions.)Ú
STRIP_SKIPÚsubr   ÚSTRIP_EOL_HYPHEN)ÚselfÚtextÚregexpÚsubstitutions       úg/home/mcse/projects/srt_converter/srt-converter-venv/lib/python3.12/site-packages/nltk/tokenize/nist.pyÚlang_independent_subz"NISTTokenizer.lang_independent_sub   sQ   € ð
  $Ÿ™Ñˆ�Ø�z‰z˜,¨Ó-ˆÜ˜DÓ!ˆØ#×4Ñ4Ñˆ�Ø�z‰z˜,¨Ó-ˆØˆó    c                 óV  — t        |«      }| j                  |«      }|r@d|z   dz   }|r|j                  «       }| j                  D ]  \  }}|j	                  ||«      }Œ dj                  |j                  «       «      }t        |j                  «       «      }|r|S |j                  «       S ©Nr	   )Ústrr   ÚlowerÚLANG_DEPENDENT_REGEXESr   ÚjoinÚsplitÚstrip)r   r   Ú	lowercaseÚwestern_langÚ
return_strr   r   s          r   ÚtokenizezNISTTokenizer.tokenize‹   sŸ   € Ü�4‹yˆà×(Ñ(¨Ó.ˆáà˜‘: Ñ#ˆDÙØ—z‘z“|�Ø(,×(CÑ(Cò 6Ñ$�˜Ø—z‘z ,°Ó5‘ð6ð �x‰x˜Ÿ
™
›Ó%ˆô �4—:‘:“<Ó ˆÙ!ˆtÐ3 t§z¡z£|Ð3r   c                 ó¤  — t        |«      }| j                  \  }}|j                  ||«      }| j                  \  }}|j                  ||«      }t	        |«      }|r|j                  «       }| j                  D ]  \  }}|j                  ||«      }Œ dj                  |j                  «       j                  «       «      }|r|S |j                  «       S r   )
r   r   r   r   r   r   ÚINTERNATIONAL_REGEXESr   r   r   )r   r   r    Úsplit_non_asciir"   r   r   s          r   Úinternational_tokenizez$NISTTokenizer.international_tokenizež   sÁ   € ô �4‹yˆð  $Ÿ™Ñˆ�Ø�z‰z˜,¨Ó-ˆØ#×4Ñ4Ñˆ�Ø�z‰z˜,¨Ó-ˆÜ˜DÓ!ˆáØ—:‘:“<ˆDà$(×$>Ñ$>ò 	2Ñ ˆF�LØ—:‘:˜l¨DÓ1‰Dð	2ð
 �x‰x˜Ÿ
™
›×*Ñ*Ó,Ó-ˆÙ!ˆtÐ3 t§z¡z£|Ð3r   N)FTF)!Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚreÚcompiler   r   ÚPUNCTÚPERIOD_COMMA_PRECEEDÚPERIOD_COMMA_FOLLOWÚDASH_PRECEED_DIGITr   r   r   Úsetr   ÚcharsÚ
pup_numberÚ	pup_punctÚ
pup_symbolr   Únumber_regexÚpunct_regexÚsymbol_regexÚNONASCIIÚPUNCT_1ÚPUNCT_2ÚSYMBOLSr%   r   r#   r'   © r   r   r   r      s  „ ñ+ð\ �—‘˜KÓ(¨"Ð,€Jà!�r—z‘z (Ó+¨SÐ0ÐàˆB�J‰JÐ8Ó9¸7ÐB€Eà%˜2Ÿ:™:Ð&8Ó9¸:ÐEÐà$˜"Ÿ*™*Ð%7Ó8¸*ÐDÐà#˜Ÿ™ LÓ1°:Ð=Ðð 	ØØØð	Ðñ �R—W‘W™SÐ!3 ×!3Ñ!3°HÓ!=Ó>Ó?Ó@€JÙ�B—G‘G™CÐ 2 × 2Ñ 2°=Ó AÓBÓCÓD€IÙ�R—W‘W™SÐ!3 ×!3Ñ!3°HÓ!=Ó>Ó?Ó@€Jð �2—6‘6˜* j°*Ó=€LØ�"—&‘&˜ Z°Ó;€KØ�2—6‘6˜* j°*Ó=€Lð ˆr�z‰zÐ*Ó+¨WÐ4€Hð 	ˆ�
‰
�R˜�~ T¨+¨°bÐ9Ó:Øð€Gð
 	ˆ�
‰
�R˜�} D¨¨°bÐ9Ó:Øð€Gð
 ˆb�j‰j˜2˜l˜^¨2Ð.Ó/°Ð8€Gà% w°¸ÐAÐò
ó4ð( GLô4r   r   )
r+   Úior,   Únltk.corpusr   Únltk.tokenize.apir   Únltk.tokenize.utilr   r   r>   r   r   ú<module>rC      s)   ðñó 
Û 	å $Ý (Ý +ôY4�Jõ Y4r   