Ë
    çÍ:j<'  ã                   ó^   — d dl mZ d dlmZmZ d„ Zd„ Zd„ Z G d„ d«      Zd„ Z	d	„ Z
d
„ Zd„ Zy)é    )Úfinditer)ÚescapeÚunescapec              #   óô   K  — t        |«      dk(  rt        d«      ‚d}	 	 | j                  ||«      }|dk7  r||f–— |t        |«      z   }Œ-# t        $ r  |t        | «      k7  r|t        | «      f–— Y yw xY w­w)a¯  
    Return the offsets of the tokens in *s*, as a sequence of ``(start, end)``
    tuples, by splitting the string at each occurrence of *sep*.

        >>> from nltk.tokenize.util import string_span_tokenize
        >>> s = '''Good muffins cost $3.88\nin New York.  Please buy me
        ... two of them.\n\nThanks.'''
        >>> list(string_span_tokenize(s, " ")) # doctest: +NORMALIZE_WHITESPACE
        [(0, 4), (5, 12), (13, 17), (18, 26), (27, 30), (31, 36), (37, 37),
        (38, 44), (45, 48), (49, 55), (56, 58), (59, 73)]

    :param s: the string to be tokenized
    :type s: str
    :param sep: the token separator
    :type sep: str
    :rtype: iter(tuple(int, int))
    r   z!Token delimiter must not be emptyN)ÚlenÚ
ValueErrorÚindex)ÚsÚsepÚleftÚrights       úg/home/mcse/projects/srt_converter/srt-converter-venv/lib/python3.12/site-packages/nltk/tokenize/util.pyÚstring_span_tokenizer      sŽ   è ø€ ô$ ˆ3ƒx�1‚}ÜÐ<Ó=Ð=Ø€DØ
ð	Ø—G‘G˜C Ó&ˆEØ˜ŠzØ˜E�kÒ!ð ”s˜3“xÑˆð øô
 ò 	Ø”s˜1“vŠ~ØœC ›F�lÒ"Ùð	üs'   ‚A8 A ½A8Á&A5Á2A8Á4A5Á5A8c              #   ó�   K  — d}t        || «      D ]"  }|j                  «       \  }}||k7  r||f–— |}Œ$ |t        | «      f–— y­w)a  
    Return the offsets of the tokens in *s*, as a sequence of ``(start, end)``
    tuples, by splitting the string at each successive match of *regexp*.

        >>> from nltk.tokenize.util import regexp_span_tokenize
        >>> s = '''Good muffins cost $3.88\nin New York.  Please buy me
        ... two of them.\n\nThanks.'''
        >>> list(regexp_span_tokenize(s, r'\s')) # doctest: +NORMALIZE_WHITESPACE
        [(0, 4), (5, 12), (13, 17), (18, 23), (24, 26), (27, 30), (31, 36),
        (38, 44), (45, 48), (49, 51), (52, 55), (56, 58), (59, 64), (66, 73)]

    :param s: the string to be tokenized
    :type s: str
    :param regexp: regular expression that matches token separators (must not be empty)
    :type regexp: str
    :rtype: iter(tuple(int, int))
    r   N)r   Úspanr   )r
   Úregexpr   Úmr   Únexts         r   Úregexp_span_tokenizer   .   sX   è ø€ ð$ €DÜ�f˜aÓ ò ˆØ—f‘f“h‰ˆˆtØ�DŠ=Ø˜�+ÒØ‰ð	ð
 ”�A“ˆ,Óùs   ‚AAc              #   ó@   K  — d}| D ]  \  }}||z
  ||z
  f–— |}Œ y­w)a«  
    Return a sequence of relative spans, given a sequence of spans.

        >>> from nltk.tokenize import WhitespaceTokenizer
        >>> from nltk.tokenize.util import spans_to_relative
        >>> s = '''Good muffins cost $3.88\nin New York.  Please buy me
        ... two of them.\n\nThanks.'''
        >>> list(spans_to_relative(WhitespaceTokenizer().span_tokenize(s))) # doctest: +NORMALIZE_WHITESPACE
        [(0, 4), (1, 7), (1, 4), (1, 5), (1, 2), (1, 3), (1, 5), (2, 6),
        (1, 3), (1, 2), (1, 3), (1, 2), (1, 5), (2, 7)]

    :param spans: a sequence of (start, end) offsets of the tokens
    :type spans: iter(tuple(int, int))
    :rtype: iter(tuple(int, int))
    r   N© )ÚspansÚprevr   r   s       r   Úspans_to_relativer   I   s8   è ø€ ð  €DØò ‰ˆˆeØ�T‰k˜5 4™<Ð'Ò'Ø‰ñùs   ‚c                   óD   — e Zd ZdZdZdZdZdZdZdZ	dZ
d	Zeeeeee	e
egZy
)ÚCJKCharsa^  
    An object that enumerates the code points of the CJK characters as listed on
    https://en.wikipedia.org/wiki/Basic_Multilingual_Plane#Basic_Multilingual_Plane

    This is a Python port of the CJK code point enumerations of Moses tokenizer:
    https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/detokenizer.perl#L309
    ©i   iÿ  ©i€.  iÏ¤  ©i@¨  i¨  ©i ¬  i¯×  ©i ù  iÿú  ©i0þ  iOþ  ©ieÿ  iÜÿ  ©i   iÿÿ N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚHangul_JamoÚCJK_RadicalsÚPhags_PaÚHangul_SyllablesÚCJK_Compatibility_IdeographsÚCJK_Compatibility_FormsÚKatakana_Hangul_HalfwidthÚSupplementary_Ideographic_PlaneÚrangesr   ó    r   r   r   _   se   „ ñð €Kð* "€Lð €Hð &Ðð $2Ð ð -Ðð !/Ðð'Ð#ð 	ØØØØ$ØØ!Ø'ð	�Fr2   r   c           
      óx   — t        dD ��cg c]  \  }}|t        | «      cxk  xr |k  nc ‘Œ  c}}«      S c c}}w )uŒ  
    Python port of Moses' code to check for CJK character.

    >>> CJKChars().ranges
    [(4352, 4607), (11904, 42191), (43072, 43135), (44032, 55215), (63744, 64255), (65072, 65103), (65381, 65500), (131072, 196607)]
    >>> is_cjk(u'ã�¾')
    True
    >>> is_cjk(u'ï¹Ÿ')
    False

    :param character: The character that needs to be checked.
    :type character: char
    :return: bool
    )r   r   r   r    r!   r"   r#   r$   )ÚanyÚord)Ú	characterÚstartÚends      r   Úis_cjkr9   ¡   s@   € ô ð	÷	
á��sð ”S˜“^Ö* sÖ*ó	
óð ùó	
s   ‹#6
c           	      ó(   — t        | ddddddœ¬«      S )aŒ  
    This function transforms the input text into an "escaped" version suitable
    for well-formed XML formatting.

    Note that the default xml.sax.saxutils.escape() function don't escape
    some characters that Moses does so we have to manually add them to the
    entities dictionary.

        >>> input_str = ''')| & < > ' " ] ['''
        >>> expected_output =  ''')| &amp; &lt; &gt; ' " ] ['''
        >>> escape(input_str) == expected_output
        True
        >>> xml_escape(input_str)
        ')&#124; &amp; &lt; &gt; &apos; &quot; &#93; &#91;'

    :param text: The text that needs to be escaped.
    :type text: str
    :rtype: str
    ú&apos;ú&quot;ú&#124;ú&#91;ú&#93;)ú'ú"ú|ú[ú]©Úentities)r   ©Útexts    r   Ú
xml_escaperI   Á   s(   € ô( ØàØØØØñ
ô	ð 	r2   c           	      ó(   — t        | ddddddœ¬«      S )aj  
    This function transforms the "escaped" version suitable
    for well-formed XML formatting into humanly-readable string.

    Note that the default xml.sax.saxutils.unescape() function don't unescape
    some characters that Moses does so we have to manually add them to the
    entities dictionary.

        >>> from xml.sax.saxutils import unescape
        >>> s = ')&#124; &amp; &lt; &gt; &apos; &quot; &#93; &#91;'
        >>> expected = ''')| & < > ' " ] ['''
        >>> xml_unescape(s) == expected
        True

    :param text: The text that needs to be unescaped.
    :type text: str
    :rtype: str
    r@   rA   rB   rC   rD   )r;   r<   r=   r>   r?   rE   )r   rG   s    r   Úxml_unescaperK   á   s(   € ô& ØàØØØØñ
ô	ð 	r2   c           	      óÌ   — d}g }| D ]6  }	 |j                  ||«      }|t        |«      z   }|j                  ||f«       Œ8 |S # t        $ r}t        d|› d|› d�«      |‚d}~ww xY w)aÅ  
    This module attempt to find the offsets of the tokens in *s*, as a sequence
    of ``(start, end)`` tuples, given the tokens and also the source string.

        >>> from nltk.tokenize import TreebankWordTokenizer
        >>> from nltk.tokenize.util import align_tokens
        >>> s = str("The plane, bound for St Petersburg, crashed in Egypt's "
        ... "Sinai desert just 23 minutes after take-off from Sharm el-Sheikh "
        ... "on Saturday.")
        >>> tokens = TreebankWordTokenizer().tokenize(s)
        >>> expected = [(0, 3), (4, 9), (9, 10), (11, 16), (17, 20), (21, 23),
        ... (24, 34), (34, 35), (36, 43), (44, 46), (47, 52), (52, 54),
        ... (55, 60), (61, 67), (68, 72), (73, 75), (76, 83), (84, 89),
        ... (90, 98), (99, 103), (104, 109), (110, 119), (120, 122),
        ... (123, 131), (131, 132)]
        >>> output = list(align_tokens(tokens, s))
        >>> len(tokens) == len(expected) == len(output)  # Check that length of tokens and tuples are the same.
        True
        >>> expected == list(align_tokens(tokens, s))  # Check that the output is as expected.
        True
        >>> tokens == [s[start:end] for start, end in output]  # Check that the slices of the string corresponds to the tokens.
        True

    :param tokens: The list of strings that are the result of tokenization
    :type tokens: list(str)
    :param sentence: The original string
    :type sentence: str
    :rtype: list(tuple(int,int))
    r   zsubstring "z" not found in "rA   N)r	   r   r   Úappend)ÚtokensÚsentenceÚpointÚoffsetsÚtokenr7   Úes          r   Úalign_tokensrT      sŒ   € ð< €EØ€GØò 'ˆð	VØ—N‘N 5¨%Ó0ˆEð œ˜E›
Ñ"ˆØ�‰˜˜u�~Õ&ð'ð €Nøô	 ò 	VÜ˜{¨5¨'Ð1AÀ(ÀÈ1ÐMÓNÐTUÐUûð	Vús   ‹AÁ	A#ÁAÁA#N)Úrer   Úxml.sax.saxutilsr   r   r   r   r   r   r9   rI   rK   rT   r   r2   r   ú<module>rW      s>   ðõ ß -ò òDò6÷,?ñ ?òDò@ò@ó>'r2   