:j-VddlZddlZddlmZddlmZddlmZmZm Z m Z ddl Z ddl Z ddlmcmZddl mZmZddlmZddlmZdd lmZ dd lmZd ZeGd dZGddej@Z GddejBZ!GddejDZ"d!dZ#edZ$GddejJZ&GddejJZ'GddejJZ(GddejJZ)Gdd ejJZ*y#eeef$rdZd ZYwxYw)"N)contextmanager) dataclass)DictIterableOptionalTuple)Tensornn)decode)detect_language) transcribe)scaled_dot_product_attentionTFcreZdZUeed<eed<eed<eed<eed<eed<eed<eed<eed <eed <y ) ModelDimensionsn_mels n_audio_ctx n_audio_state n_audio_head n_audio_layern_vocab n_text_ctx n_text_state n_text_head n_text_layerN)__name__ __module__ __qualname__int__annotations__b/home/mcse/projects/srt_converter/srt-converter-venv/lib/python3.12/site-packages/whisper/model.pyrrs; K LOr"rc(eZdZdedeffd ZxZS) LayerNormxreturncpt||jj|jSN)superforwardfloattypedtype)selfr& __class__s r#r+zLayerNorm.forward(s'wqwwy)..qww77r")rrrr r+ __classcell__r0s@r#r%r%'s88F88r"r%ceZdZdedefdZy)Linearr&r'ctj||jj|j|j dS|j j|jSr))Flinearweighttor.bias)r/r&s r#r+zLinear.forward-sVxx KKNN177 #II%D  ,099<<+@  r"N)rrrr r+r!r"r#r4r4,s  F r"r4c6eZdZdededeedeffd ZxZS)Conv1dr&r8r:r'ct|||j|j|dS|j|jSr))r* _conv_forwardr9r.)r/r&r8r:r0s r#r>zConv1d._conv_forward6sIw$ vyy!4<4  =AWWQWW=M  r")rrrr rr>r1r2s@r#r<r<5s2  !' /7/?   r"r<c|dzdk(sJtj||dzdz z }tj| tj|dzz}tj|ddtj f|tj ddfz}tj tj|tj|gdS)z*Returns sinusoids for positional embeddingrr Ndim) nplogtorchexparangenewaxiscatsincos)lengthchannels max_timescalelog_timescale_incrementinv_timescales scaled_times r# sinusoidsrR>s a<1   ff]3x1}q7HIYY 77%,,xST}:UUVN,,v&q"**}5rzzST}8UUK 99eii ,eii .DE1 MMr"c#Ktj} dt_d|t_y#|t_wxYww)NF)MultiHeadAttentionuse_sdpa) prev_states r# disable_sdparWGs4#,,J1&+# &0#j#s?/ ? <?ceZdZdZdedeffd Z ddedeedeedeefd Z dd ed ed edeed e e j ee j ff dZ xZ S)rTTn_staten_headct|||_t|||_t||d|_t|||_t|||_y)NF)r:)r*__init__rZr4querykeyvalueout)r/rYrZr0s r#r\zMultiHeadAttention.__init__TsO  GW- '77GW- '7+r"r&xamaskkv_cachec.|j|}|||j|vr+|j||n|}|j||n|}n||j}||j}|j||||\}} |j || fSr))r]r^r_ qkv_attentionr`) r/r&rarbrcqkvwvqks r#r+zMultiHeadAttention.forward\s JJqM  rzTXXX-Ebjb1A  13A"A$A##Aq!T2Bxx|Rr"rfrgrhr'c|j\}}}||jzdz}|jg|jdd|jdjdddd}|jg|jdd|jdjdddd}|jg|jdd|jdjdddd}trRt j rBt||||duxr|dkD} | jddddjd} d} | | fS||z||zjdd z} || |d|d|fz} | j} tj| d j|j} | |zjddddjd} | j} | | fS) Ngпr@rr ) is_causal) start_dimrA)shaperZviewpermuteSDPA_AVAILABLErTrUrflatten transposer,r6softmaxr9r.detach) r/rfrgrhrbn_batchn_ctxrYscalear`rjws r#rez MultiHeadAttention.qkv_attentionrs#$''DKK'E1 AFF 1AGGBQK 1 1b 1 9 9!Q1 E AFF 1AGGBQK 1 1b 1 9 9!Q1 E AFF 1AGGBQK 1 1b 1 9 9!Q1 E 099,1a4t#3#A A))Aq!Q'//!/CGn M #GV 4 M9  !)  Ns1B!r&ctj|j|}tj|j|}|j ddd}|j dd|j j k(sJd||j zj|j}|jD] }||} |j|}|S)zt x : torch.Tensor, shape = (batch_size, n_mels, n_ctx) the mel spectrogram of the audio rr@r Nzincorrect audio shape) r6gelurrrsrqrr9r.rr)r/r&blocks r#r+zAudioEncoder.forwards FF4::a= ! FF4::a= ! IIaA wwqr{d77===V?VV= ** * . .qww 7[[ EaA  LLOr")rrrrr\r r+r1r2s@r#rrs< * *"% *03 *=@ *KN *r"rc PeZdZdededededef fd Zd deded eefd ZxZ S) TextDecoderrrzrYrZrc t|tj|||_tj t j|||_tjt|Dcgc]}t||dc}|_ t||_t j||jt j" j%d}|j'd|dycc}w)NT)rr rbF persistent)r*r\r Embeddingtoken_embedding ParameterrEemptyrrrrrr%lnfill_rCinftriu_r) r/rrzrYrZrrrbr0s r#r\zTextDecoder.__init__s !||GW=$&LLUG1L$M!8: w 'wM 9  G${{5%(..w7==a@ VTe< s5C>r&rarcc|r/tt|jjdnd}|j ||j |||jdzz}|j |j}|jD]}||||j|}|j|}|tj|jjj |jddzj}|S)z x : torch.LongTensor, shape = (batch_size, <= n_ctx) the text tokens xa : torch.Tensor, shape = (batch_size, n_audio_ctx, n_audio_state) the encoded audio features to be attended on r rrlr)nextitervaluesrqrrr9r.rrbrrErvr8r,)r/r&rarcoffsetrlogitss r#r+zTextDecoder.forwards >qww GAN N %'  r"r)) rrrrr\r rrr+r1r2s@r#rrsN==#&=14=>A=LO=&Vx~r"rc2eZdZdeffd ZdefdZdejfdZ dejdejfd Z dejdejd e e ejffd Z ed Zed ZedZddeefdZeZeZeZxZS)Whisperdimsc(t|||_t|jj|jj |jj |jj|jj|_ t|jj|jj|jj|jj|jj|_t#j$|jj|jjt"j&}d||jjdzd|j)d|j+dy)Nr.Tr@alignment_headsFr)r*r\rrrrrrrencoderrrrrrrdecoderrEzerosrr to_sparse)r/r all_headsr0s r#r\zWhisper.__init__s  # II   II ! ! II # # II " " II # #   # II   II II " " II ! ! II " "   KK II " "DII$9$9 48 $))((A-/0 . 0C0C0ERWXr"dumpctjtjt j |t j}tj|j|jj|jj}|jd|jdy)NrrFr)rC frombuffergzip decompressbase64 b85decodercopyrE from_numpyreshaperrrrr)r/rarrayrbs r#set_alignment_headszWhisper.set_alignment_headss OOF,,T2 34 $& &.. II " "DII$9$9  .0@USr"melc$|j|Sr))r)r/rs r# embed_audiozWhisper.embed_audios||C  r"tokensaudio_featuresc&|j||Sr))r)r/rrs r#rzWhisper.logits"s||FN33r"r'cD|j||j|Sr))rr)r/rrs r#r+zWhisper.forward%s||FDLL$566r"cHt|jjSr))r parametersdevicer/s r#rzWhisper.device*sDOO%&---r"c4|jjdk\S)Ni)rrrs r#is_multilingualzWhisper.is_multilingual.syy  E))r"c`|jjdz t|jz S)Ni5)rrrrrs r# num_languageszWhisper.num_languages2s'yy  5(3t/C/C+DDDr"cachecinigfddtjffd }jj|fS)a The `MultiHeadAttention` module optionally accepts `kv_cache` which stores the key and value tensors calculated for the previous positions. This method returns a dictionary that stores all caches, and the necessary hooks for the key and value projection modules that save the intermediate tensors to be reused during later calculations. Returns ------- cache : Dict[nn.Module, torch.Tensor] A dictionary object mapping the key/value projection modules to its cache hooks : List[RemovableHandle] List of PyTorch RemovableHandle objects to stop the hooks to be called c|vs&|jdjjkDr ||<|Stj||gdj |<|S)Nr rA)rqrrrErIrx)moduleroutputrr/s r# save_to_cachez5Whisper.install_kv_cache_hooks..save_to_cacheGskU"fll1o 8L8L&L &f = !& 5=&*Aq I P P Rf = r"layerct|trUj|jj j|j j yyr)) isinstancerTappendr^register_forward_hookr_)rhooksrs r# install_hooksz5Whisper.install_kv_cache_hooks..install_hooksOsI%!34 UYY<<]KL U[[>>}MN5r")r Modulerapply)r/rrrrs`` @@r#install_kv_cache_hookszWhisper.install_kv_cache_hooks6sO#. 5 B ! O O =)e|r"r))rrrrr\bytesrrEr rrrstrr+propertyrrrrrrdetect_language_functionr transcribe_functionrdecode_functionr r1r2s@r#rrsY_Y2TT!u||!4U\\45<<47<<7).7 c5<< 7 ..**EEHTNB/O$J Fr"r)i')+rr contextlibr dataclassesrtypingrrrrnumpyrCrEtorch.nn.functionalr functionalr6r decodingr rr rrrrrt ImportError RuntimeErrorOSErrorrr%r4r<rRrWrrTrrrrr!r"r#rs %!22 /A9@N      8 8  RYY  RYY N11::zRYY@299B*"))*Z]bii]Q \7+#' Ns D D('D(