Ë
    þÍ:jí   ã                   óP   — d dl Z d dlmZ d dlmZmZmZ d dlZ G d„ d«      ZdZ	dZ
y)é    N)Úcached_property)ÚListÚOptionalÚTuplec            
       ó$  — e Zd ZdZ	 	 ddej
                  dedee   dee   fd„Z	e
defd	„«       Ze
defd
„«       Ze
defd„«       Ze
defd„«       Ze
defd„«       Ze
defd„«       Ze
defd„«       Ze
defd„«       Zedefd„«       Zedee   fd„«       Zdedee   fd„Zdee   defd„Zdee   defd„Ze
dee   fd„«       Zdee   deee   eee      f   fd„Zdee   deee   eee      f   fd„Zdee   deee   eee      f   fd„Zy)Ú	Tokenizerz-Simple wrapper around a tokenizers.Tokenizer.NÚ	tokenizerÚmultilingualÚtaskÚlanguagec                 ó�  — || _         |r¨|t        vr%t        d|›ddj                  t        «      ›d�«      ‚|t        vr%t        d|›ddj                  t        «      ›d�«      ‚| j                   j                  d|z  «      | _        | j                   j                  d|z  «      | _        || _        y d | _        d | _        d| _        y )Nú'z'' is not a valid task (accepted tasks: z, ú)z9' is not a valid language code (accepted language codes: z<|%s|>Úen)	r	   Ú_TASKSÚ
ValueErrorÚjoinÚ_LANGUAGE_CODESÚtoken_to_idr   r   Úlanguage_code)Úselfr	   r
   r   r   s        úm/home/mcse/projects/srt_converter/srt-converter-venv/lib/python3.12/site-packages/faster_whisper/tokenizer.pyÚ__init__zTokenizer.__init__   sµ   € ð #ˆŒáØœ6Ñ!Ý â˜TŸY™Y¤vÕ.ð0óð ð
 œÑ.Ý â §¡¬?Õ!;ð=óð ð
 Ÿ™×2Ñ2°8¸d±?ÓCˆDŒIØ ŸN™N×6Ñ6°xÀ(Ñ7JÓKˆDŒMØ!)ˆDÕàˆDŒIØ ˆDŒMØ!%ˆDÕó    Úreturnc                 ó8   — | j                   j                  d«      S )Nz<|transcribe|>©r	   r   ©r   s    r   Ú
transcribezTokenizer.transcribe*   s   € à�~‰~×)Ñ)Ð*:Ó;Ð;r   c                 ó8   — | j                   j                  d«      S )Nz<|translate|>r   r   s    r   Ú	translatezTokenizer.translate.   ó   € à�~‰~×)Ñ)¨/Ó:Ð:r   c                 ó8   — | j                   j                  d«      S )Nz<|startoftranscript|>r   r   s    r   ÚsotzTokenizer.sot2   s   € à�~‰~×)Ñ)Ð*AÓBÐBr   c                 ó8   — | j                   j                  d«      S )Nz<|startoflm|>r   r   s    r   Úsot_lmzTokenizer.sot_lm6   r"   r   c                 ó8   — | j                   j                  d«      S )Nz<|startofprev|>r   r   s    r   Úsot_prevzTokenizer.sot_prev:   s   € à�~‰~×)Ñ)Ð*;Ó<Ð<r   c                 ó8   — | j                   j                  d«      S )Nz<|endoftext|>r   r   s    r   ÚeotzTokenizer.eot>   r"   r   c                 ó8   — | j                   j                  d«      S )Nz<|notimestamps|>r   r   s    r   Úno_timestampszTokenizer.no_timestampsB   s   € à�~‰~×)Ñ)Ð*<Ó=Ð=r   c                 ór   — | j                   j                  d«      xs | j                   j                  d«      S )Nz<|nospeech|>z<|nocaptions|>r   r   s    r   Ú	no_speechzTokenizer.no_speechF   s2   € à�~‰~×)Ñ)¨.Ó9ò 
¸T¿^¹^×=WÑ=WØó>
ð 	
r   c                 ó    — | j                   dz   S )Né   )r,   r   s    r   Útimestamp_beginzTokenizer.timestamp_beginL   s   € à×!Ñ! AÑ%Ð%r   c                 ó¼   — | j                   g}| j                  �|j                  | j                  «       | j                  �|j                  | j                  «       |S ©N)r$   r   Úappendr   )r   Úsequences     r   Úsot_sequencezTokenizer.sot_sequenceP   sF   € à—H‘H�:ˆà�=‰=Ð$Ø�O‰O˜DŸM™MÔ*à�9‰9Ð Ø�O‰O˜DŸI™IÔ&àˆr   Útextc                 óP   — | j                   j                  |d¬«      j                  S )NF)Úadd_special_tokens)r	   ÚencodeÚids)r   r7   s     r   r:   zTokenizer.encode\   s"   € Ø�~‰~×$Ñ$ T¸eÐ$ÓD×HÑHÐHr   Útokensc                 ó€   — |D �cg c]  }|| j                   k  sŒ|‘Œ }}| j                  j                  |«      S c c}w r3   )r*   r	   Údecode)r   r<   ÚtokenÚtext_tokenss       r   r>   zTokenizer.decode_   s;   € Ø*0ÖE °E¸D¿H¹HÓ4D’uÐEˆÐEØ�~‰~×$Ñ$ [Ó1Ð1ùò Fs   …;š;c           	      ón  — g g}|D ]_  }|| j                   k\  r:d|| j                   z
  dz  d›d�}|j                  |«       |j                  g «       ŒL|d   j                  |«       Œa dj                  |D �cg c]/  }t        |t        «      r|n| j
                  j                  |«      ‘Œ1 c}«      S c c}w )Nz<|g{®Gáz”?z.2fz|>éÿÿÿÿÚ )r1   r4   r   Ú
isinstanceÚstrr	   r>   )r   r<   Úoutputsr?   Ú	timestampÚss         r   Údecode_with_timestampsz Tokenizer.decode_with_timestampsc   s²   € Ø�$ˆàò 	*ˆEØ˜×,Ñ,Ò,Ø  %¨$×*>Ñ*>Ñ">À$Ñ!FÀsÐ KÈ2ÐN�	Ø—‘˜yÔ)Ø—‘˜rÕ"à˜‘×"Ñ" 5Õ)ð	*ð �w‰wØLSÖTÀq”*˜Q¤Ô$‰Q¨$¯.©.×*?Ñ*?ÀÓ*BÑBÒTó
ð 	
ùÚTs   Á74B2c                 ó¼  — t        d«      }|dj                  «       z  }t        d«      }t        d„ |D «       «      sJ ‚| j	                  d«      d   | j	                  d«      d   h}|t        |«      z   D ]S  }| j	                  |«      | j	                  d|z   «      fD ])  }t        |«      d	k(  s||v sŒ|j                  |d   «       Œ+ ŒU t        t        |«      «      S )
u…  
        Returns the list of tokens to suppress in order to avoid any speaker tags or non-speech
        annotations, to prevent sampling texts that are not actually spoken in the audio, e.g.

        - â™ªâ™ªâ™ª
        - ( SPEAKING FOREIGN LANGUAGE )
        - [DAVID] Hey there,

        keeping basic punctuations like commas, periods, question marks, exclamation points, etc.
        u#   "#()*+/:;<=>@[\]^_`{|}~ã€Œã€�ã€Žã€�uK   << >> <<< >>> -- --- -( -[ (' (" (( )) ((( ))) [[ ]] {{ }} â™ªâ™ª â™ªâ™ªâ™ªu   â™©â™ªâ™«â™¬â™­â™®â™¯c              3   óN   K  — | ]  }d t        |«      cxk  xr dk  nc –— Œ y­w)i@&  i&  N)Úord)Ú.0Úcs     r   ú	<genexpr>z.Tokenizer.non_speech_tokens.<locals>.<genexpr>ˆ   s!   è ø€ ÒE°!�6œS ›VÖ- v×-Ð-ÑEùs   ‚#%z -r   z 'ú r0   )	ÚlistÚsplitÚsetÚallr:   ÚlenÚaddÚtupleÚsorted)r   ÚsymbolsÚmiscellaneousÚresultÚsymbolr<   s         r   Únon_speech_tokenszTokenizer.non_speech_tokensr   sê   € ô Ð=Ó>ˆØØZ×`Ñ`Óbñ	
ˆô Ð3Ó4ˆÜÑE°}ÔEÔEÐEÐEð —+‘+˜dÓ# AÑ&¨¯©°DÓ(9¸!Ñ(<Ð=ˆØ¤ ]Ó 3Ñ3ò 	*ˆFà—‘˜FÓ#Ø—‘˜C &™LÓ)ðò *�ô �v“; !Ò# v°Ò'>Ø—J‘J˜v a™yÕ)ñ*ð	*ô ”V˜F“^Ó$Ð$r   c                 ób   — | j                   dv r| j                  |«      S | j                  |«      S )N>   ÚjaÚloÚmyÚthÚzhÚyue)r   Úsplit_tokens_on_unicodeÚsplit_tokens_on_spaces)r   r<   s     r   Úsplit_to_word_tokenszTokenizer.split_to_word_tokens–   s7   € ð ×ÑÐ!FÑFð ×/Ñ/°Ó7Ð7à×*Ñ*¨6Ó2Ð2r   c                 óz  — | j                  |«      }d}g }g }g }d}|D ]‡  }|j                  |«       | j                  |«      }		 |	j                  |«      }
|
|z  }
|
�|
t	        |«      k  sŒM||
   |k(  sŒV|j                  |	«       |j                  |«       g }|t	        |	«      z  }Œ‰ ||fS # t        $ r d }
Y Œ_w xY w)Nu   ï¿½r   )rI   r4   Úindexr   rU   )r   r<   Údecoded_fullÚreplacement_charÚwordsÚword_tokensÚcurrent_tokensÚunicode_offsetr?   ÚdecodedÚreplacement_char_indexs              r   re   z!Tokenizer.split_tokens_on_unicode¡   sð   € ð ×2Ñ2°6Ó:ˆØ#ÐàˆØˆØˆØˆàò 	/ˆEØ×!Ñ! %Ô(Ø×1Ñ1°.ÓAˆGð.Ø)0¯©Ð7GÓ)HÐ&Ø&¨.Ñ8Ð&ð &Ð-Ø&¬¨\Ó):Ó:Ø Ð!7Ñ8Ð<LÓLà—‘˜WÔ%Ø×"Ñ" >Ô2Ø!#�Ø¤# g£,Ñ.‘ð#	/ð& �kÐ!Ð!øô ò .Ø)-Ò&ð.ús   ÁB,Â,B:Â9B:c                 ó”  — | j                  |«      \  }}g }g }t        ||«      D ]ž  \  }}|d   | j                  k\  }|j                  d«      }	|j	                  «       t
        j                  v }
|s|	s|
st        |«      dk(  r#|j                  |«       |j                  |«       Œ€|d   |z   |d<   |d   j                  |«       Œ  ||fS )Nr   rP   rB   )
re   Úzipr*   Ú
startswithÚstripÚstringÚpunctuationrU   r4   Úextend)r   r<   ÚsubwordsÚsubword_tokens_listrl   rm   ÚsubwordÚsubword_tokensÚspecialÚ
with_spacerw   s              r   rf   z Tokenizer.split_tokens_on_spacesÁ   sÕ   € ð )-×(DÑ(DÀVÓ(LÑ%ˆÐ%ØˆØˆä'*¨8Ð5HÓ'Iò 		7Ñ#ˆG�^Ø$ QÑ'¨4¯8©8Ñ3ˆGØ ×+Ñ+¨CÓ0ˆJØ!Ÿ-™-›/¬V×-?Ñ-?Ð?ˆKÙ™*©´s¸5³zÀQ²Ø—‘˜WÔ%Ø×"Ñ" >Õ2à! "™I¨Ñ/��b‘	Ø˜B‘×&Ñ& ~Õ6ð		7ð �kÐ!Ð!r   )NN) Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
tokenizersr   Úboolr   rE   r   r   Úintr   r!   r$   r&   r(   r*   r,   r.   Úpropertyr1   r   r6   r:   r>   rI   r   r]   rg   re   rf   © r   r   r   r   	   sQ  „ Ù7ð #Ø"&ñ&à×'Ñ'ð&ð ð&ð �s‰mð	&ð
 ˜3‘-ó&ð< ð<˜Cò <ó ð<ð ð;˜3ò ;ó ð;ð ðC�Sò Có ðCð ð;˜ò ;ó ð;ð ð=˜#ò =ó ð=ð ð;�Sò ;ó ð;ð ð>˜sò >ó ð>ð ð
˜3ò 
ó ð
ð
 ð& ò &ó ð&ð ð	˜d 3™iò 	ó ð	ðI˜3ð I 4¨¡9ó Ið2˜T #™Yð 2¨3ó 2ð
¨T°#©Yð 
¸3ó 
ð ð!% 5¨¡:ò !%ó ð!%ðF	3Ø˜3‘ið	3à	ˆt�C‰y˜$˜t C™y™/Ð)Ñ	*ó	3ð"Ø˜3‘ið"à	ˆt�C‰y˜$˜t C™y™/Ð)Ñ	*ó"ð@"Ø˜3‘ið"à	ˆt�C‰y˜$˜t C™y™/Ð)Ñ	*ô"r   r   )r   r!   )dÚafÚamÚarÚasÚazÚbaÚbeÚbgÚbnÚboÚbrÚbsÚcaÚcsÚcyÚdaÚdeÚelr   ÚesÚetÚeuÚfaÚfiÚfoÚfrÚglÚguÚhaÚhawÚheÚhiÚhrÚhtÚhuÚhyÚidÚisÚitr_   ÚjwÚkaÚkkÚkmÚknÚkoÚlaÚlbÚlnr`   ÚltÚlvÚmgÚmiÚmkÚmlÚmnÚmrÚmsÚmtra   ÚneÚnlÚnnÚnoÚocÚpaÚplÚpsÚptÚroÚruÚsaÚsdÚsiÚskÚslÚsnÚsoÚsqÚsrÚsuÚsvÚswÚtaÚteÚtgrb   ÚtkÚtlÚtrÚttÚukÚurÚuzÚviÚyiÚyorc   rd   )rv   Ú	functoolsr   Útypingr   r   r   rƒ   r   r   r   r‡   r   r   ú<module>rç      s0   ðÛ å %ß (Ñ (ã ÷J"ñ J"ðZ
€ð
e�r   