Ë
    ÿÍ:j›  ã                   ó’   — d dl Z d dlZd dlZd dlmZmZmZmZmZ d dl	Z	d dl
mZ d dlmZ erd dlmZ  G d„ d«      Z G d„ d	e«      Zy)
é    N)ÚTYPE_CHECKINGÚAnyÚLiteralÚOptionalÚUnion)Ú!_TORCHMETRICS_GREATER_EQUAL_1_0_0)Ú_PATH)ÚFabricc                   ó  ‡ — e Zd ZdZ	 	 	 	 	 	 	 d ded   dededee   dee   d	ee   d
e	fˆ fd„Z
 ej                  «       dddej                  dededdf
d„«       Zdej                  de	fd„Zdddeddfd„Zdeeej                  f   deeej                  f   de	fd„Zdeeej                  f   deeej                  f   de	fd„Zdej                  de	fd„Zdej                  ddfd„Zdeeef   fd„Zdeeef   ddfd„Zˆ xZS )!ÚSpikeDetectionaÊ  Spike Detection Callback.

    Terminates training with a ``TrainingSpikeException`` when a loss-spike was detected and
    saves the batches to skip when resuming to a file.

    We skip the current and the previous batch since it is unclear whether the previous batch
    altered the weights in a way that it causes the spike or just the current batch is corrupted somehow.

    Args:
        mode: Whether to minimize or maximize the tracked metric
        window: A running mean of metrics with ``window`` size. Serves as reference value for spikes.
        warmup: After how many batches spike-tracking should start
        atol: An absolute tolerance.  Every diff between the running mean and the current value,
            that's not an improvement and above ``atol`` will be considered a spike
        rtol: A relative tolerance. Every diff between the running mean and the current value,
            that's higher than ``rtol * running_mean`` is considered a spike
        exclude_batches_path: Where to save the file that contains the batches to exclude.
            Will default to current directory.
        finite_only: If set to ``False``, consider non-finite values like NaN, inf and -inf a spike as well.

    NÚmode)ÚminÚmaxÚwindowÚwarmupÚatolÚrtolÚexclude_batches_pathÚfinite_onlyc                 ó@  •— t         rddlm} ddlm}	 nt        d«      ‚t        ‰
| �  «        d| _         |	 |dd¬«      |¬«      | _	        d| j                  _
        d| j                  _        || _        || _        || _        || _        g | _        || _        || _        y )	Nr   )Ú
MeanMetric)ÚRunningzJSpikeDetection requires `torchmetrics>=1.0.0` Please upgrade your version.ç        F)Údist_sync_on_stepÚsync_on_compute)r   )r   Útorchmetrics.aggregationr   Útorchmetrics.wrappersr   ÚRuntimeErrorÚsuperÚ__init__Úlast_valÚrunning_meanr   r   r   r   r   r   Úbad_batchesr   r   )Úselfr   r   r   r   r   r   r   r   r   Ú	__class__s             €úu/home/mcse/projects/srt_converter/srt-converter-venv/lib/python3.12/site-packages/lightning_fabric/utilities/spike.pyr    zSpikeDetection.__init__&   s”   ø€ õ -Ý;Þ5äÐkÓlÐlÜ‰ÑÔà47ˆŒá#¡JÀÐX]Ô$^ÐgmÔnˆÔà.3ˆ×ÑÔ+Ø,1ˆ×ÑÔ)àˆŒ	ØˆŒØˆŒ	ØˆŒ	Ø&(ˆÔØ$8ˆÔ!Ø&ˆÕó    Úfabricr
   ÚlossÚbatchÚ	batch_idxÚreturnc                 ó  — |dk(  r/| j                   j                  |j                  j                  «       | j                  €t        j                  «       | _        t        | j                  «      j                  d«      s/t
        j                  j                  | j                  d«      | _        t        || j                  k\  xr | j                  |«      «      }|j                  j                  «        |j                  j                  |d¬«      }|r| j!                  ||«       y| j"                  xsG |j                  j                  t        t%        j&                  |«      j)                  «       «      d¬«      }|r| j+                  |«       yy)z)Checks if we currently have a loss-spike.r   Nz.jsonzskip_batches.jsonF)ÚallT)r"   ÚtoÚstrategyÚroot_devicer   ÚosÚgetcwdÚstrÚendswithÚpathÚjoinÚboolr   Ú	_is_spikeÚbarrierÚreduce_boolean_decisionÚ_handle_spiker   ÚtorchÚisfiniter.   Ú_update_stats)r$   r(   r)   r*   r+   Úis_spikeÚis_spike_globalÚis_finite_alls           r&   Úon_train_batch_endz!SpikeDetection.on_train_batch_endF   s5  € ð ˜Š>Ø×Ñ× Ñ  §¡×!<Ñ!<Ô=à×$Ñ$Ð,Ü(*¯	©	«ˆDÔ%ä�4×,Ñ,Ó-×6Ñ6°wÔ?Ü(*¯©¯©°T×5NÑ5NÐPcÓ(dˆDÔ%ä˜	 T§[¡[Ñ0ÒI°T·^±^ÀDÓ5IÓJˆØ�‰×ÑÔ!ð !Ÿ/™/×AÑAÀ(ÐPUÐAÓVˆáØ×Ñ˜v yÕ1à ×,Ñ,ò °·±×0WÑ0WÜ”U—^‘^ DÓ)×-Ñ-Ó/Ó0°dð 1Xó 1ˆMñ Ø×"Ñ" 4Õ(ð r'   c                 ó„  — t        j                  «       5  t        j                  d«       | j                  j	                  «       }d d d «       || j
                  z
  }| j                  rt        j                  |«      sy| j                  |«      ry| j                  |«      xr | j                  ||«      S # 1 sw Y   ŒrxY w)NÚignoreTF)ÚwarningsÚcatch_warningsÚsimplefilterr"   Úcomputer!   r   r=   r>   Ú
_is_betterÚ_check_atolÚ_check_rtol)r$   r)   Úrunning_valÚ	curr_diffs       r&   r9   zSpikeDetection._is_spikea   s¢   € ô ×$Ñ$Ó&ñ 	6Ü×!Ñ! (Ô+Ø×+Ñ+×3Ñ3Ó5ˆK÷	6ð ˜4Ÿ=™=Ñ(ˆ	à×Ò¤E§N¡N°4Ô$8Øà�?‰?˜9Ô%Øà×Ñ  kÓ2ÒZ°t×7GÑ7GÈÈkÓ7ZÐZ÷	6ð 	6ús   •0B6Â6B?c                 óª  — | j                   j                  |dz
  |g«       |j                  dk(  r�| j                  €J ‚t	        j
                  t        j                  j                  | j                  «      d¬«       t        | j                  d«      5 }t        j                  | j                   |d¬«       d d d «       t        |¬«      ‚# 1 sw Y   ŒxY w)	Né   r   T)Úexist_okÚwé   )Úindent)r+   )r#   ÚextendÚglobal_rankr   r2   Úmakedirsr6   ÚdirnameÚopenÚjsonÚdumpÚTrainingSpikeException)r$   r(   r+   Úfs       r&   r<   zSpikeDetection._handle_spikeq   s¬   € ð 	×Ñ×Ñ ¨Q¡°	Ð :Ô;à×Ñ Ò"Ø×,Ñ,Ð8Ð8Ð8Ü�K‰KœŸ™Ÿ™¨×(AÑ(AÓBÈTÕRä�d×/Ñ/°Ó5ð 9¸Ü—	‘	˜$×*Ñ*¨A°aÕ8÷9ô %¨yÔ9Ð9÷9ð 9ús   Â#C	Ã	CÚval_aÚval_bc                 ó|   — | j                   d u xs- t        t        ||z
  «      t        | j                   «      k\  «      S ©N)r   r8   Úabs©r$   r^   r_   s      r&   rK   zSpikeDetection._check_atol€   s1   € Ø—	‘	˜TÐ!ÒP¤d¬3¨u°u©}Ó+=ÄÀTÇYÁYÃÑ+OÓ&PÐPr'   c                 ó‚   — | j                   d u xs0 t        t        ||z
  «      t        | j                   |z  «      k\  «      S ra   )r   r8   rb   rc   s      r&   rL   zSpikeDetection._check_rtolƒ   s8   € Ø—	‘	˜TÐ!ÒX¤d¬3¨u°u©}Ó+=ÄÀTÇYÁYÐQVÑEVÓAWÑ+WÓ&XÐXr'   Údiff_valc                 óÞ   — | j                   dk(  rt        |dk  j                  «       «      S | j                   dk(  rt        |dk\  j                  «       «      S t        d| j                   › �«      ‚)Nr   r   r   r   z*Invalid mode. Has to be min or max, found )r   r8   r.   Ú
ValueError)r$   re   s     r&   rJ   zSpikeDetection._is_better†   sa   € Ø�9‰9˜ÒÜ˜ S™×-Ñ-Ó/Ó0Ð0Ø�9‰9˜ÒÜ˜ Q™×+Ñ+Ó-Ó.Ð.äÐEÀdÇiÁiÀ[ÐQÓRÐRr'   Úvalc                 óH   — | j                   j                  |«       || _        y ra   )r"   Úupdater!   )r$   rh   s     r&   r?   zSpikeDetection._update_statsŽ   s   € à×Ñ× Ñ  Ô%Øˆ�r'   c           
      ó–  — t        | j                  t        j                  «      r| j                  j	                  «       n| j                  | j
                  | j                  | j                  | j                  | j                  | j                  | j                  j                  «       | j                  j                  j                  «       dœ	S )N)	r!   r   r   r   r   r#   Úbad_batches_pathÚrunningÚmean)Ú
isinstancer!   r=   ÚTensorÚitemr   r   r   r   r#   r   r"   Ú
state_dictÚbase_metric)r$   s    r&   rr   zSpikeDetection.state_dict“   sŽ   € ä0:¸4¿=¹=Ì%Ï,É,Ô0W˜Ÿ™×*Ñ*Ô,Ð]a×]jÑ]jØ—I‘IØ—k‘kØ—I‘IØ—I‘IØ×+Ñ+Ø $× 9Ñ 9Ø×(Ñ(×3Ñ3Ó5Ø×%Ñ%×1Ñ1×<Ñ<Ó>ñ

ð 
	
r'   rr   c                 óô  — |j                  d«      | _        |j                  d«      | _        |j                  d«      | _        |j                  d«      | _        |j                  d«      | _        |j                  d«      | _        |j                  d«      | _        | j                  j                  |j                  d«      «       | j                  j                  j                  |j                  d	«      «       y )
Nr!   r   r   r   r   r#   rl   rm   rn   )Úpopr!   r   r   r   r   r#   r   rm   Úload_state_dictr"   rs   )r$   rr   s     r&   rv   zSpikeDetection.load_state_dict    s³   € Ø"Ÿ™ zÓ2ˆŒØ—N‘N 6Ó*ˆŒ	Ø —n‘n XÓ.ˆŒØ—N‘N 6Ó*ˆŒ	Ø—N‘N 6Ó*ˆŒ	Ø%Ÿ>™>¨-Ó8ˆÔØ$.§N¡NÐ3EÓ$FˆÔ!Ø�‰×$Ñ$ Z§^¡^°IÓ%>Ô?Ø×Ñ×%Ñ%×5Ñ5°j·n±nÀVÓ6LÕMr'   )r   é
   rP   Ng       @NT)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Úintr   Úfloatr	   r8   r    r=   Úno_gradrp   r   rC   r9   r<   r   rK   rL   rJ   r?   Údictr4   rr   rv   Ú__classcell__©r%   s   @r&   r   r      sÒ  ø„ ñð0 ',ØØØ $Ø #Ø04Ø ñ'à�lÑ#ð'ð ð'ð ð	'ð
 �u‰oð'ð �u‰oð'ð ' u™oð'ð õ'ð@ €U‡]�]ƒ_ð)¨ð )¸¿¹ð )Ècð )Ð^að )Ðfjò )ó ð)ð4[˜eŸl™lð [¨tó [ð : Hð :¸ð :Àó :ðQ  u¨e¯l©lÐ':Ñ!;ð QÀEÈ%ÐQV×Q]ÑQ]ÐJ]ÑD^ð QÐcgó QðY  u¨e¯l©lÐ':Ñ!;ð YÀEÈ%ÐQV×Q]ÑQ]ÐJ]ÑD^ð YÐcgó YðS 5§<¡<ð S°Dó Sð §¡ð °$ó ð

˜D  c ™Nó 
ð	N¨$¨s°C¨x©.ð 	N¸T÷ 	Nr'   r   c                   ó0   ‡ — e Zd ZdZdededefˆ fd„Zˆ xZS )r\   z,Exception to be raised with Training Spikes.r+   ÚargsÚkwargsc                 ó2   •— t        ‰| �  d|› �g|¢­i |¤Ž y )Nz!Training spike detected in batch )r   r    )r$   r+   rƒ   r„   r%   s       €r&   r    zTrainingSpikeException.__init__¯   s#   ø€ Ü‰ÑÐ<¸Y¸KÐHÐZÈ4ÒZÐSYÓZr'   )rx   ry   rz   r{   r|   r   r    r€   r�   s   @r&   r\   r\   ¬   s*   ø„ Ù6ð[ #ð [¨cð [¸S÷ [ñ [r'   r\   )rZ   r2   rF   Útypingr   r   r   r   r   r=   Ú"lightning_fabric.utilities.importsr   Ú lightning_fabric.utilities.typesr	   Úlightning_fabric.fabricr
   r   r   r\   © r'   r&   ú<module>r‹      s@   ðÛ Û 	Û ß ?Õ ?ã å PÝ 2áÝ.÷ZNñ ZNôz[˜\õ [r'   