Ë
    úÍ:j¢?  ã            "       ó"  — d dl mZmZmZ d dlZd dlmZ ddlmZmZm	Z	m
Z
mZmZmZmZmZmZmZmZmZmZmZ ddgZ G d„ de«      Zd	e› d
e› de› de	› de› d�e_        dee   dee   dee   dee   dee   dee   dededededededededefd„Zdee   dee   dee   dee   dee   dee   dededededededededefd„Z e
e¬«      	 	 	 	 	 d!dee   dee   dee   dee   dee   dee   dee   dededededededededef d „«       Zy)"é    )ÚcastÚOptionalÚUnionN)ÚTensoré   )Ú_capturable_docÚ_default_to_fused_or_foreachÚ_differentiable_docÚ_disable_dynamo_if_unsupportedÚ_foreach_docÚ!_get_capturable_supported_devicesÚ_get_scalar_dtypeÚ
_get_valueÚ_maximize_docÚ_params_docÚ
_to_scalarÚ_use_grad_for_differentiableÚ_view_as_realÚ	OptimizerÚParamsTÚASGDÚasgdc                   óŽ   ‡ — e Zd Z	 	 	 	 	 	 	 	 	 ddedeeef   dededededee   ded	ed
efˆ fd„Z	ˆ fd„Z
d„ Zedd„«       Zˆ xZS )r   ÚparamsÚlrÚlambdÚalphaÚt0Úweight_decayÚforeachÚmaximizeÚdifferentiableÚ
capturablec                 óö   •— t        |t        «      r|j                  «       dk7  rt        d«      ‚d|k  st        d|› �«      ‚d|k  st        d|› �«      ‚t	        ||||||||	|
¬«	      }t
        ‰| �  ||«       y )Nr   zTensor lr must be 1-elementg        zInvalid learning rate: zInvalid weight_decay value: )	r   r   r   r   r   r    r!   r"   r#   )Ú
isinstancer   ÚnumelÚ
ValueErrorÚdictÚsuperÚ__init__)Úselfr   r   r   r   r   r   r    r!   r"   r#   ÚdefaultsÚ	__class__s               €úe/home/mcse/projects/srt_converter/srt-converter-venv/lib/python3.12/site-packages/torch/optim/asgd.pyr*   zASGD.__init__   s’   ø€ ô �bœ&Ô! b§h¡h£j°A¢oÜÐ:Ó;Ð;Ø�bŠyÜÐ6°r°dÐ;Ó<Ð<Ø�lÒ"ÜÐ;¸L¸>ÐJÓKÐKäØØØØØ%ØØØ)Ø!ô

ˆô 	‰Ñ˜ Õ*ó    c                 ó  •— t         ‰| �  |«       | j                  D �]f  }|j                  dd «       |j                  dd«       |j                  dd«       |j                  dd«       |d   D �]  }| j                  j                  |g «      }t        |«      dk7  sŒ/t        j                  |d   «      s;t        |d   «      }t        j                  |t        «       |j                  ¬	«      |d<   t        j                  |d
   «      s0t        j                  |d
   t        «       |j                  ¬	«      |d
<   t        j                  |d   «      rŒãt        j                  |d   t        «       |j                  ¬	«      |d<   �Œ �Œi y )Nr    r!   Fr"   r#   r   r   Ústep)ÚdtypeÚdeviceÚetaÚmu)r)   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetÚlenÚtorchÚ	is_tensorÚfloatÚtensorr   r3   )r+   r9   ÚgroupÚpÚp_stateÚstep_valr-   s         €r.   r6   zASGD.__setstate__?   sF  ø€ Ü‰Ñ˜UÔ#Ø×&Ñ&ó 	ˆEØ×Ñ˜Y¨Ô-Ø×Ñ˜Z¨Ô/Ø×ÑÐ-¨uÔ5Ø×Ñ˜\¨5Ô1Ø˜8‘_ó �ØŸ*™*Ÿ.™.¨¨BÓ/�Ü�w“< 1Ó$Ü Ÿ?™?¨7°6©?Ô;Ü#(¨°©Ó#9˜Ü*/¯,©,Ø$Ô,=Ó,?ÈÏÉô+˜ ™ô !Ÿ?™?¨7°5©>Ô:Ü).¯©Ø# E™NÔ2CÓ2EÈaÏhÉhô*˜ ™ô !Ÿ?™?¨7°4©=Õ9Ü(-¯©Ø# D™MÔ1BÓ1DÈQÏXÉXô)˜ ›òñ	r/   c                 óŽ  — d}|d   D �]¹  }	|	j                   €Œ|t        j                  |	«      z  }|j                  |	«       |	j                   j                  rt        d«      ‚|j                  |	j                   «       | j                  |	   }
t        |
«      dk(  r×t        j                  d|	j                  t        «       ¬«      |
d<   t        j                  t        |d   «      |	j                  t        «       ¬«      j                  «       j                  «       |
d	<   t        j                  d|	j                  t        «       ¬«      |
d
<   t        j                   |	t        j"                  ¬«      |
d<   |j                  |
d
   «       |j                  |
d   «       |j                  |
d	   «       |j                  |
d   «       �Œ¼ |S )NFr   z&ASGD does not support sparse gradientsr   © )r3   r2   r1   r   r4   r5   )Úmemory_formatÚax)Úgradr<   Ú
is_complexÚappendÚ	is_sparseÚRuntimeErrorr9   r;   Úzerosr3   r   Ú	as_tensorr   ÚcloneÚdetachÚonesÚ
zeros_likeÚpreserve_format)r+   r@   Úparams_with_gradÚgradsÚmusÚaxsÚetasÚstate_stepsÚhas_complexrA   r9   s              r.   Ú_init_groupzASGD._init_groupW   s{  € ØˆØ�x‘ó !	2ˆAØ�v‰vÑ!Øœu×/Ñ/°Ó2Ñ2�Ø ×'Ñ'¨Ô*Ø—6‘6×#Ò#Ü&Ð'OÓPÐPØ—‘˜QŸV™VÔ$àŸ
™
 1™�ä�u“: ’?Ü$)§K¡KØ 1§8¡8Ô3DÓ3Fô%�E˜&‘Mô Ÿ™Ü& u¨T¡{Ó3Ø#$§8¡8Ü"3Ó"5ô÷
 ™›ß™›ð ˜%‘Lô #(§*¡*Ø 1§8¡8Ô3DÓ3Fô#�E˜$‘Kô #(×"2Ñ"2Ø¬×)>Ñ)>ô#�E˜$‘Kð —
‘
˜5 ™;Ô'Ø—
‘
˜5 ™;Ô'Ø—‘˜E %™LÔ)Ø×"Ñ" 5¨¡=Ö1ðC!	2ðD Ðr/   c                 ób  — | j                  «        d}|�$t        j                  «       5   |«       }ddd«       | j                  D ][  }g }g }g }g }g }g }	| j	                  |||||||	«      }
t        ||||||	|d   |d   |d   |d   |d   |d   |d   |d	   |d
   |
¬«       Œ] |S # 1 sw Y   ŒuxY w)z°Perform a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r   r   r   r   r    r!   r"   r#   )
r   r   r   r   r   r    r!   r"   r#   rZ   )Ú _cuda_graph_capture_health_checkr<   Úenable_gradr7   r[   r   )r+   ÚclosureÚlossr@   rT   rU   rV   rW   rX   rY   rZ   s              r.   r1   z	ASGD.step}   s  € ð 	×-Ñ-Ô/àˆØÐÜ×"Ñ"Ó$ñ !Ù“y�÷!ð ×&Ñ&ò 	ˆEØ-/ÐØ"$ˆEØ "ˆCØ "ˆCØ!#ˆDØ(*ˆKà×*Ñ*ØÐ'¨°°S¸$ÀóˆKô Ø ØØØØØØ˜G‘nØ˜‘;Ø˜‘;Ø˜G‘nØ" >Ñ2Ø˜iÑ(Ø˜zÑ*Ø$Ð%5Ñ6Ø  Ñ.Ø'ö!ð	ð> ˆ÷E!ð !ús   ©B%Â%B.)	g{®Gáz„?g-Cëâ6?g      è?g    €„.Ar   NFFF©N)Ú__name__Ú
__module__Ú__qualname__r   r   r>   r   r   Úboolr*   r6   r[   r   r1   Ú__classcell__)r-   s   @r.   r   r      s´   ø„ ð $(ØØØØØ"&ØØ$Ø ñ+àð+ð �%˜�-Ñ ð+ð ð	+ð
 ð+ð ð+ð ð+ð ˜$‘ð+ð ð+ð ð+ð õ+ôBò0$ðL "ò-ó "ô-r/   zŸImplements Averaged Stochastic Gradient Descent.

    It has been proposed in `Acceleration of stochastic approximation by
    averaging`_.

    Args:
        am  
        lr (float, Tensor, optional): learning rate (default: 1e-2)
        lambd (float, optional): decay term (default: 1e-4)
        alpha (float, optional): power for eta update (default: 0.75)
        t0 (float, optional): point at which to start averaging (default: 1e6)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        z	
        z¿

    .. _Acceleration of stochastic approximation by averaging:
        https://meyn.ece.ufl.edu/wp-content/uploads/sites/77/archive/spm_files/Courses/ECE555-2011/555media/poljud92.pdf

    r   rU   rW   rV   rX   rY   r   r   r   r   r   r!   r"   r#   rZ   c       	   
      ó(  — t         j                  j                  «       st        |«      }t	        | «      D �]Ú  \  }}||   }|s|n| }||   }||   }||   }||   }t         j
                  j                  «       s“|r‘t        «       }|j                  j                  |j                  j                  cxk(  r3|j                  j                  cxk(  r|j                  j                  k(  rn n|j                  j                  |v sJ d|› d�«       ‚t        j                  |«      r?t        j                  |«      }t        j                  |«      }t        j                  |«      }|dz  }|
dk7  r|j                  ||
¬«      }|r,|j                  d||z  z
  «       |j                  ||d¬«       n6t        |«      }|j                  d||z  z
  «       |j!                  || ¬«       |s|j#                  «       dk7  r0|j!                  |j%                  |«      j                  |«      «       n|j'                  |«       |r`|j'                  |d||z  |z  z   |	z  z  «       |j'                  dt        j(                  ||z
  t        j*                  |«      «      z  «       �Œet        |«      }t        j,                  |d||z  |z  z   |	z  z  «      }|j'                  |«       t        j,                  dt/        d||z
  «      z  «      }|j'                  |«       �ŒÝ y )NúUIf capturable=True, params, mus, etas, and state_steps must be on supported devices: ú.r   r   ©r   éÿÿÿÿ©Úvalue)r<   ÚjitÚis_scriptingr   Ú	enumerateÚcompilerÚis_compilingr   r3   ÚtyperI   Úview_as_realÚaddÚmul_Úaddcmul_r   Úadd_ÚitemÚsubÚcopy_ÚmaximumÚ	ones_likerN   Úmax)r   rU   rW   rV   rX   rY   r   r   r   r   r   r!   r"   r#   rZ   ÚiÚparamrH   r5   rG   r4   Ústep_tÚcapturable_supported_devicesÚ	eta_valuer1   Únew_etaÚnew_mus                              r.   Ú_single_tensor_asgdr†   Å   s´  € ô$ �9‰9×!Ñ!Ô#Ü˜‹^ˆä˜fÓ%ó 7‰ˆˆ5Ø�Q‰xˆÙ#‰t¨$¨ˆØ�‰VˆØ�‰VˆØ�1‰gˆØ˜Q‘ˆô �~‰~×*Ñ*Ô,±Ü+LÓ+NÐ(à—‘×!Ñ!Ø—9‘9—>‘>ô&à—:‘:—?‘?ô&ð —=‘=×%Ñ%õ&ð —L‘L×%Ñ%Ð)EÑEð	ð)Ø)EÐ(FÀaðIó	ðFô ×Ñ˜EÔ"Ü×%Ñ% dÓ+ˆDÜ×&Ñ& uÓ-ˆEÜ×#Ñ# BÓ'ˆBð 	�!‰ˆà˜1ÒØ—8‘8˜E¨�8Ó6ˆDáØ�J‰J�q˜5 3™;‘Ô'Ø�N‰N˜4 ¨BˆNÕ/ä" 3›ˆIØ�J‰J�q˜5 9Ñ,Ñ,Ô-Ø�J‰J�t I :ˆJÔ.ñ ˜Ÿ™› ašØ�G‰G�E—I‘I˜b“M×&Ñ& rÓ*Õ+à�H‰H�UŒOáØ�I‰I�b˜Q ¨¡¨fÑ!4Ñ4¸Ñ>Ñ?Ô@Ø�H‰H�QœŸ™ v°¡{´E·O±OÀFÓ4KÓLÑLÖMä˜fÓ%ˆDÜ—o‘o b¨Q°¸±¸dÑ1BÑ-BÀuÑ,LÑ&MÓNˆGØ�I‰I�gÔÜ—_‘_ Q¬¨Q°°r±	Ó):Ñ%:Ó;ˆFØ�H‰H�VÖño7r/   c       	         ó\  ‡"— t        | «      dk(  ry |rJ d«       ‚t        j                  j                  «       s9|r7t	        d¬«      Š"t        ˆ"fd„t        | |||«      D «       «      sJ d‰"› d�«       ‚t        |«      }t        j                  | |||||g«      }|j                  «       D �]x  \  \  }}\  \  }}}}}}}t        t        t           |«      }t        t        t           |«      }t        t        t           |«      }t        t        t           |«      }t        t        t           |«      }t        t        t           |«      }|rt        |||«       |rt        j                  |«      }t        j                  j                  «       s=|d   j                   r.t        j"                  |t        j$                  dd	¬
«      d¬«       nt        j"                  |d«       |
dk7  rN|rt        j"                  |||
¬«       |}nt        j&                  |||
¬«      }t        j"                  |||¬«       nt        j&                  |||¬«      }t        j(                  |||d¬«       ~t        j*                  ||«      }t        j(                  |||«       ~|rót        j*                  ||«      }t        j,                  |d«       t        j.                  |«       t        j0                  ||«       ~t        j2                  ||«      } t        j4                  | |«       t        j"                  | d«       t        j6                  | |	«       t        j.                  | «       t        j4                  | |«       t        j0                  || «       �ŒÝ|D �!cg c](  }!t        j8                  |d||z  |!z  z   |	z  z  |¬
«      ‘Œ* } }!|D �!cg c]2  }!t        j8                  dt;        dt=        |!«      |z
  «      z  |¬
«      ‘Œ4 }}!t        j0                  || «       t        j0                  ||«       �Œ{ y c c}!w c c}!w )Nr   z#_foreach ops don't support autogradF)Úsupports_xlac              3   ó,  •K  — | ]‹  \  }}}}|j                   j                  |j                   j                  cxk(  xr5 |j                   j                  cxk(  xr |j                   j                  k(  nc xr |j                   j                  ‰v –— Œ� y ­wra   )r3   rs   )Ú.0rA   r5   r4   r1   r‚   s        €r.   ú	<genexpr>z%_multi_tensor_asgd.<locals>.<genexpr>0  so   øè ø€ ò 
ñ !��2�s˜Dð �H‰H�M‰M˜RŸY™YŸ^™^ÖR¨s¯z©z¯©ÖRÀ$Ç+Á+×BRÑBRÔRò >Ø—‘—‘Ð!=Ð=ó>ñ
ùs   ƒBBrh   ri   g      ð?Úcpu)r3   rj   r   rk   rl   )r;   r<   rq   rr   r   ÚallÚzipr   r   Ú"_group_tensors_by_device_and_dtypeÚitemsr   Úlistr   r   Ú_foreach_negÚis_cpuÚ_foreach_add_r?   Ú_foreach_addÚ_foreach_addcmul_Ú_foreach_subÚ_foreach_maximum_Ú_foreach_reciprocal_Ú_foreach_copy_Ú_foreach_mulÚ_foreach_mul_Ú_foreach_pow_rN   r~   r   )#r   rU   rW   rV   rX   rY   r   r   r   r   r   r!   r"   r#   rZ   Úgrouped_tensorsr3   Ú_Úgrouped_params_Úgrouped_grads_Úgrouped_axs_Úgrouped_mus_Úgrouped_etas_Úgrouped_state_steps_Úgrouped_paramsÚgrouped_gradsÚgrouped_axsÚgrouped_musÚgrouped_etasÚgrouped_state_stepsÚintermediateÚnew_musÚnew_etasr1   r‚   s#                                     @r.   Ú_multi_tensor_asgdr¯     sõ  ø€ ô$ ˆ6ƒ{�aÒØáÐDÐDÓDÐô �>‰>×&Ñ&Ô(©ZÜ'HØô(
Ð$ô ó 
ô %(¨°°T¸;Ó$Gô
ô 
ð 	
ð
 dð  eAð  dBð  BCð  Dó	
ð 
ô 
�B‹€Bä×BÑBØ	�˜˜S $¨Ð4ó€Oð 
×	Ñ	Ó	 óc7ñ 
	‰ˆ�ñ 
ñ	
ØØØØØØ à	äœd¤6™l¨OÓ<ˆÜœT¤&™\¨>Ó:ˆÜœ4¤™<¨Ó6ˆÜœ4¤™<¨Ó6ˆÜœD¤™L¨-Ó8ˆÜ"¤4¬¡<Ð1EÓFÐáÜ˜.¨-¸ÔEáÜ!×.Ñ.¨}Ó=ˆMô �~‰~×*Ñ*Ô,Ð1DÀQÑ1G×1NÒ1NÜ×ÑØ#¤U§\¡\°#¸eÔ%DÈCöô ×ÑÐ 3°QÔ7ð ˜1ÒÙÜ×#Ñ# M°>ÈÕVØ,‘ä$×1Ñ1Ø! >¸ô �ô ×Ñ ¨nÀEÖJä ×-Ñ-Ø˜~°UôˆLô 	×Ñ °¸lÐRTÕUØô ×)Ñ)¨.¸+ÓFˆÜ×Ñ ¨\¸;ÔGØñ ä×(Ñ(Ð)<¸bÓAˆGÜ×#Ñ# G¨SÔ1Ü×&Ñ& wÔ/Ü× Ñ  ¨gÔ6Øô ×)Ñ)Ð*=¸uÓEˆHÜ×Ñ ¨"Ô-Ü×Ñ ¨!Ô,Ü×Ñ ¨%Ô0Ü×&Ñ& xÔ0Ü×Ñ ¨"Ô-Ü× Ñ  ¨xÖ8ð 0öàô —‘  q¨5°2©:¸Ñ+<Ñ'<ÀÑ&FÑ GÐPVÖWðˆHð ð 0öàô —‘ ¤C¨¬:°dÓ+;¸bÑ+@Ó$AÑ AÈ&ÖQðˆGð ô × Ñ  ¨xÔ8Ü× Ñ  ¨gÖ6ñGc7ùòtùòs   Î-P$Î;7P))Úsingle_tensor_fnr    c                ó  — |€t        | |d¬«      \  }}|r)t        j                  j                  «       rt	        d«      ‚|r%t        j                  j                  «       st
        }nt        } || |||||||||||||	|
¬«       y)znFunctional API that performs asgd algorithm computation.

    See :class:`~torch.optim.ASGD` for details.
    NF)Ú	use_fusedz6torch.jit.script not supported with foreach optimizers)	r   r   r   r   r   r!   r"   r#   rZ   )r	   r<   rn   ro   rL   r¯   r†   )r   rU   rW   rV   rX   rY   r    r!   r"   r#   rZ   r   r   r   r   r   rŸ   Úfuncs                     r.   r   r   £  s�   € ð4 €Ü1Ø�N¨eô
‰
ˆˆ7ñ ”5—9‘9×)Ñ)Ô+ÜÐSÓTÐTá”u—y‘y×-Ñ-Ô/Ü!‰ä"ˆáØØØØØØØØØØØ!ØØ%ØØör/   )NFFFF)Útypingr   r   r   r<   r   Ú	optimizerr   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   Ú__all__r   Ú__doc__r‘   r>   re   r†   r¯   r   rE   r/   r.   ú<module>r¸      sï  ðç (Ñ (ã Ý ÷÷ ÷ ÷ ñ ð& �6Ð
€ôNˆ9ô Nðb	ð 
ˆð 	ð 
ˆð 	Ø	ˆð 	Ø	Ðð 	Ø	Ðð ð€„ð.LØ�‰LðLà�‰<ðLð 
ˆf‰ðLð 
ˆf‰ð	Lð
 ˆv‰,ðLð �f‘ðLð ðLð 	ðLð 	ðLð ðLð ðLð ðLð ðLð ðLð  ó!Lð^L7Ø�‰LðL7à�‰<ðL7ð 
ˆf‰ðL7ð 
ˆf‰ð	L7ð
 ˆv‰,ðL7ð �f‘ðL7ð ðL7ð 	ðL7ð 	ðL7ð ðL7ð ðL7ð ðL7ð ðL7ð ðL7ð  ó!L7ñ^  Ð1DÔEð #ØØ ØØñ6Ø�‰Lð6à�‰<ð6ð 
ˆf‰ð6ð 
ˆf‰ð	6ð
 ˆv‰,ð6ð �f‘ð6ð �d‰^ð6ð ð6ð ð6ð ð6ð ð6ð ð6ð  	ð!6ð" 	ð#6ð$ ð%6ð& ò'6ó Fñ6r/   