§
    �ŠtjÌœ  ã            .       ó  — d dl mZ d dlZd dlmZ ddlmZmZmZmZm	Z	m
Z
mZmZmZmZmZmZmZmZmZmZmZmZmZmZ ddgZ G d„ de¦  «        Zd	d
e› de
› de› de› de› de› d�z   e_        dee         dee         dee         dee         dee         dee         dedz  dedz  dededeez  deez  deez  dededededed ed!df(d"„Zdee         dee         dee         dee         dee         dee         dedz  dedz  dededeez  deez  deez  dededededed ed!df(d#„Z dee         dee         dee         dee         dee         dee         dedz  dedz  dededeez  deez  deez  dededededed ed!df(d$„Z! e	e¬%¦  «        	 	 	 	 	 	 	 	 d*dee         dee         dee         dee         dee         dee         d'edz  deded(edz  dedz  dedz  ded ededeez  deez  deez  dededed!df,d)„¦   «         Z"dS )+é    )ÚcastN)ÚTensoré   )Ú_capturable_docÚ_default_to_fused_or_foreachÚ_device_dtype_check_for_fusedÚ_differentiable_docÚ_disable_dynamo_if_unsupportedÚ_foreach_docÚ
_fused_docÚ!_get_capturable_supported_devicesÚ_get_scalar_dtypeÚ
_get_valueÚ_maximize_docÚ_params_docÚ_stack_if_compilingÚ
_to_scalarÚ_use_grad_for_differentiableÚ_view_as_realÚ
DeviceDictÚDeviceDtypeDictÚ	OptimizerÚParamsTÚAdamÚadamc                   óÆ   ‡ — e Zd Z	 	 	 	 	 ddddddddœded	eez  d
eeez  eez  f         dededededz  dededededz  deddfˆ fd„Zˆ fd„Z	d„ Z
edd„¦   «         Zˆ xZS )r   çü©ñÒMbP?©gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?ç:Œ0âŽyE>r   FN)ÚforeachÚmaximizeÚ
capturableÚdifferentiableÚfusedÚdecoupled_weight_decayÚparamsÚlrÚbetasÚepsÚweight_decayÚamsgradr    r!   r"   r#   r$   r%   Úreturnc                óJ  •— t          |t          ¦  «        r:|r|	st          d¦  «        ‚|                     ¦   «         dk    rt          d¦  «        ‚d|k    st          d|› �¦  «        ‚d|k    st          d|› �¦  «        ‚d|d         cxk    rdk     sn t          d	|d         › �¦  «        ‚d|d         cxk    rdk     sn t          d
|d         › �¦  «        ‚d|k    st          d|› �¦  «        ‚t          |d         t          ¦  «        rt          |d         t          ¦  «        sEt          |d         t          ¦  «        rt          |d         t          ¦  «        st          d¦  «        ‚t          |d         t          ¦  «        r@|	s|rt          d¦  «        ‚|d                              ¦   «         dk    rt          d¦  «        ‚t          |d         t          ¦  «        r@|	s|rt          d¦  «        ‚|d                              ¦   «         dk    rt          d¦  «        ‚t          t          t          |¦  «        ¦  «        }||||||||	|
||dœ}t          ¦   «          	                    ||¦  «         |r)|
rt          d¦  «        ‚d| _        |rt          d¦  «        ‚d S d S )NúElr as a Tensor is not supported for capturable=False and foreach=Truer   úTensor lr must be 1-elementç        zInvalid learning rate: zInvalid epsilon value: r   ç      ð?z#Invalid beta parameter at index 0: z#Invalid beta parameter at index 1: zInvalid weight_decay value: z0betas must be either both floats or both TensorszKbetas[0] as a Tensor is not supported for capturable=False and foreach=Truez!Tensor betas[0] must be 1-elementzKbetas[1] as a Tensor is not supported for capturable=False and foreach=Truez!Tensor betas[1] must be 1-element)r'   r(   r)   r*   r+   r!   r    r"   r#   r$   r%   z)`fused` does not support `differentiable`Tz0`fused` and `foreach` cannot be `True` together.)Ú
isinstancer   Ú
ValueErrorÚnumelÚfloatÚtupleÚmapr   ÚsuperÚ__init__ÚRuntimeErrorÚ_step_supports_amp_scaling)Úselfr&   r'   r(   r)   r*   r+   r    r!   r"   r#   r$   r%   ÚdefaultsÚ	__class__s                 €úN/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/torch/optim/adam.pyr9   zAdam.__init__#   s0  ø€ õ  �b�&Ñ!Ô!ð 	@Øð ˜zð Ý Ø[ñô ð ð �xŠx‰zŒz˜QŠˆÝ Ð!>Ñ?Ô?Ð?Ø�bŠyˆyÝÐ;°rÐ;Ð;Ñ<Ô<Ð<Ø�cŠzˆzÝÐ<°sÐ<Ð<Ñ=Ô=Ð=Ø�e˜A”hÐ$Ð$Ò$Ð$ Ò$Ð$Ð$Ð$ÝÐMÀ5ÈÄ8ÐMÐMÑNÔNÐNØ�e˜A”hÐ$Ð$Ò$Ð$ Ò$Ð$Ð$Ð$ÝÐMÀ5ÈÄ8ÐMÐMÑNÔNÐNØ�lÒ"Ð"ÝÐJ¸LÐJÐJÑKÔKÐKå˜˜aœ¥%Ñ(Ô(ð	QÝ-7¸¸a¼Å%Ñ-HÔ-Hð	Qå˜5 œ8¥VÑ,Ô,ð	Qå1;¸EÀ!¼HÅfÑ1MÔ1Mð	Qõ ÐOÑPÔPÐPÝ�e˜A”h¥Ñ'Ô'ð 	FØð  'ð Ý Øañô ð ð �QŒx�~Š~ÑÔ 1Ò$Ð$Ý Ð!DÑEÔEÐEÝ�e˜A”h¥Ñ'Ô'ð 	FØð  'ð Ý Øañô ð ð �QŒx�~Š~ÑÔ 1Ò$Ð$Ý Ð!DÑEÔEÐEÝ•c�* eÑ,Ô,Ñ-Ô-ˆð ØØØ(ØØ ØØ$Ø,ØØ&<ð
ð 
ˆõ 	‰Œ×Ò˜ Ñ*Ô*Ð*àð 		WØð PÝ"Ð#NÑOÔOÐOØ.2ˆDÔ+ð
 ð WÝ"Ð#UÑVÔVÐVð		Wð 		WðWð Wó    c                 ó$  •— t          ¦   «                              |¦  «         | j        D �]d}|                     dd¦  «         |                     dd¦  «         |                     dd ¦  «         |                     dd¦  «         |                     dd¦  «         |                     dd¦  «         |                     dd ¦  «        }|d	         D ]¾}| j                             |g ¦  «        }t          |¦  «        d
k    rŽt          j        |d         ¦  «        stt          |d         ¦  «        }|d         s|d         r*t          j
        |t          |¬¦  «        |j        ¬¦  «        n!t          j
        |t          ¦   «         ¬¦  «        |d<   Œ¿�Œfd S )Nr+   Fr!   r    r"   r#   r%   r$   r&   r   Ústep©Úis_fused©ÚdtypeÚdevice)rF   )r8   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetÚlenÚtorchÚ	is_tensorr5   Útensorr   rG   )r<   rK   Úgroupr$   ÚpÚp_stateÚstep_valr>   s          €r?   rH   zAdam.__setstate__s   s¦  ø€ Ý‰Œ×Ò˜UÑ#Ô#Ð#ØÔ&ð 	ñ 	ˆEØ×Ò˜Y¨Ñ.Ô.Ð.Ø×Ò˜Z¨Ñ/Ô/Ð/Ø×Ò˜Y¨Ñ-Ô-Ð-Ø×Ò˜\¨5Ñ1Ô1Ð1Ø×ÒÐ-¨uÑ5Ô5Ð5Ø×ÒÐ5°uÑ=Ô=Ð=Ø×$Ò$ W¨dÑ3Ô3ˆEØ˜8”_ð ð �Øœ*Ÿ.š.¨¨BÑ/Ô/�Ý�w‘<”< 1Ò$Ð$­U¬_¸WÀV¼_Ñ-MÔ-MÐ$Ý$ W¨V¤_Ñ5Ô5�Hð ! Ô.ðOð
 38¸´.ðO�œØ$Ý"3¸UÐ"CÑ"CÔ"CØ#$¤8ðñ ô ð õ #œ\¨(Õ:KÑ:MÔ:MÐNÑNÔNð ˜F‘Oøñ	ð	ð 	r@   c                 ó´  — d}|d         D �]K}	|	j         ��@|t          j        |	¦  «        z  }|                     |	¦  «         |	j         j        rt          d¦  «        ‚|                     |	j         ¦  «         | j        |	         }
t          |
¦  «        dk    rî|d         rt          |	¦  «         |d         s|d         r0t          j	        dt          |d         ¬¦  «        |	j        ¬	¦  «        n"t          j        d
t          ¦   «         d¬	¦  «        |
d<   t          j        |	t          j        ¬¦  «        |
d<   t          j        |	t          j        ¬¦  «        |
d<   |d         r#t          j        |	t          j        ¬¦  «        |
d<   |                     |
d         ¦  «         |                     |
d         ¦  «         |d         r|                     |
d         ¦  «         |d         r|
d         j        rt          d¦  «        ‚|d         r1t          j        |d         ¦  «        r|d         st          d¦  «        ‚|                     |
d         ¦  «         �ŒM|S )NFr&   zJAdam does not support sparse gradients, please consider SparseAdam insteadr   r$   r"   © rC   rE   r0   ÚcpurB   )Úmemory_formatÚexp_avgÚ
exp_avg_sqr+   Úmax_exp_avg_sqr#   zB`requires_grad` is not supported for `step` in differentiable moder    r'   r.   )ÚgradrN   Ú
is_complexÚappendÚ	is_sparser:   rK   rM   r   Úzerosr   rG   rP   Ú
zeros_likeÚpreserve_formatÚrequires_gradrO   )r<   rQ   Úparams_with_gradÚgradsÚexp_avgsÚexp_avg_sqsÚmax_exp_avg_sqsÚstate_stepsÚhas_complexrR   rK   s              r?   Ú_init_groupzAdam._init_group‹   s¨  € ð ˆØ�x”ð =	2ñ =	2ˆAØŒvÑ!Ø�uÔ/°Ñ2Ô2Ñ2�Ø ×'Ò'¨Ñ*Ô*Ð*Ø”6Ô#ð Ý&Ødñô ð ð —’˜QœVÑ$Ô$Ð$àœ
 1œ�å�u‘:”: ’?�?Ø˜W”~ð 9Ý5°aÑ8Ô8Ð8ð ! Ô.ðXð
 38¸´.ðX�œØÝ"3¸UÀ7¼^Ð"LÑ"LÔ"LØ#$¤8ðñ ô ð õ #œ\¨#Õ5FÑ5HÔ5HÐQVÐWÑWÔWð ˜&‘Mõ (-Ô'7Ø­Ô)>ð(ñ (ô (�E˜)Ñ$õ +0Ô*:Ø­Ô)>ð+ñ +ô +�E˜,Ñ'ð ˜YÔ'ð å27Ô2BØ­UÔ-Bð3ñ 3ô 3˜Ð.Ñ/ð —’  iÔ 0Ñ1Ô1Ð1Ø×"Ò" 5¨Ô#6Ñ7Ô7Ð7à˜Ô#ð DØ#×*Ò*¨5Ð1AÔ+BÑCÔCÐCØÐ)Ô*ð ¨u°V¬}Ô/Jð Ý&Ø\ñô ð ð ˜)Ô$ðåœ¨¨d¬Ñ4Ô4ðð " ,Ô/ðõ
 'Ø_ñô ð ð ×"Ò" 5¨¤=Ñ1Ô1Ð1ùØÐr@   c                 ó  — |                       ¦   «          d}|�5t          j        ¦   «         5   |¦   «         }ddd¦  «         n# 1 swxY w Y   | j        D ]®}g }g }g }g }g }g }	|d         \  }
}|                      |||||||	¦  «        }t          ||||||	f|d         ||
||d         |d         |d         |d         |d         |d	         |d
         |d         t          | dd¦  «        t          | dd¦  «        |d         dœŽ Œ¯|S )z°Perform a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr(   r+   r'   r*   r)   r!   r    r"   r#   r$   Ú
grad_scaleÚ	found_infr%   )r+   rj   Úbeta1Úbeta2r'   r*   r)   r!   r    r"   r#   r$   rm   rn   r%   )Ú'_accelerator_graph_capture_health_checkrN   Úenable_gradrI   rk   r   Úgetattr)r<   ÚclosureÚlossrQ   rd   re   rf   rg   rh   ri   ro   rp   rj   s                r?   rB   z	Adam.stepÖ   s®  € ð 	×4Ò4Ñ6Ô6Ð6àˆØÐÝÔ"Ñ$Ô$ð !ð !Ø�w‘y”y�ð!ð !ð !ñ !ô !ð !ð !ð !ð !ð !ð !øøøð !ð !ð !ð !ð Ô&ð )	ð )	ˆEØ-/ÐØ"$ˆEØ%'ˆHØ(*ˆKØ,.ˆOØ(*ˆKØ  œ>‰LˆE�5à×*Ò*ØØ ØØØØØñô ˆKõ Ø ØØØØØðð ˜iÔ(Ø'ØØØ˜”;Ø" >Ô2Ø˜%”LØ˜zÔ*Ø˜iÔ(Ø  Ô.Ø$Ð%5Ô6Ø˜G”nÝ" 4¨°tÑ<Ô<Ý! $¨°TÑ:Ô:Ø',Ð-EÔ'Fð+ð ð ð ð ð0 ˆs   ¬AÁAÁ
A)r   r   r   r   F©N)Ú__name__Ú
__module__Ú__qualname__r   r5   r   r6   Úboolr9   rH   rk   r   rB   Ú__classcell__)r>   s   @r?   r   r   "   s‰  ø€ € € € € ð "Ø7CØØØðNWð  $ØØ Ø$Ø!Ø',ðNWð NWð NWàðNWð �F‰NðNWð �U˜V‘^ U¨V¡^Ð3Ô4ð	NWð
 ðNWð ðNWð ðNWð ˜‘ðNWð ðNWð ðNWð ðNWð �d‰{ðNWð !%ðNWð 
ðNWð NWð NWð NWð NWð NWð`ð ð ð ð ð0Ið Ið IðV "ð9ð 9ð 9ñ "Ô!ð9ð 9ð 9ð 9ð 9r@   af  Implements Adam algorithm.

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \gamma \text{ (lr)}, \beta_1, \beta_2
                \text{ (betas)},\theta_0 \text{ (params)},f(\theta) \text{ (objective)}          \\
            &\hspace{13mm}      \lambda \text{ (weight decay)},  \: \textit{amsgrad},
                \:\textit{maximize},  \: \epsilon \text{ (epsilon)}                              \\
            &\textbf{initialize} :  m_0 \leftarrow 0 \text{ ( first moment)},
                v_0\leftarrow 0 \text{ (second moment)},\: v_0^{max}\leftarrow 0          \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\

            &\hspace{5mm}\textbf{if} \: \textit{maximize}:                                       \\
            &\hspace{10mm}g_t           \leftarrow   -\nabla_{\theta} f_t (\theta_{t-1})         \\
            &\hspace{5mm}\textbf{else}                                                           \\
            &\hspace{10mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})          \\
            &\hspace{5mm}\textbf{if} \: \lambda \neq 0                                           \\
            &\hspace{10mm} g_t \leftarrow g_t + \lambda  \theta_{t-1}                            \\
            &\hspace{5mm}m_t           \leftarrow   \beta_1 m_{t-1} + (1 - \beta_1) g_t          \\
            &\hspace{5mm}v_t           \leftarrow   \beta_2 v_{t-1} + (1-\beta_2) g^2_t          \\
            &\hspace{5mm}\widehat{m_t} \leftarrow   m_t/\big(1-\beta_1^t \big)                   \\
            &\hspace{5mm}\textbf{if} \: amsgrad                                                  \\
            &\hspace{10mm} v_t^{max} \leftarrow \mathrm{max}(v_{t-1}^{max},v_t)                  \\
            &\hspace{10mm}\widehat{v_t} \leftarrow v_t^{max}/\big(1-\beta_2^t \big)              \\
            &\hspace{5mm}\textbf{else}                                                           \\
            &\hspace{10mm}\widehat{v_t} \leftarrow   v_t/\big(1-\beta_2^t \big)                  \\
            &\hspace{5mm}\theta_t \leftarrow \theta_{t-1} - \gamma \widehat{m_t}/
                \big(\sqrt{\widehat{v_t}} + \epsilon \big)                                       \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to `Adam: A Method for Stochastic Optimization`_.
    z
    Args:
        a;  
        lr (float, Tensor, optional): learning rate (default: 1e-3). A tensor LR
            is not yet supported for all our implementations. Please use a float
            LR if you are not also specifying fused=True or capturable=True.
        betas (tuple[float | Tensor, float | Tensor], optional):
            coefficients used for computing running averages of gradient and
            its square. If a tensor is provided, must be 1-element. (default: (0.9, 0.999))
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        decoupled_weight_decay (bool, optional): if True, this optimizer is
            equivalent to AdamW and the algorithm will not accumulate weight
            decay in the momentum nor variance. (default: False)
        amsgrad (bool, optional): whether to use the AMSGrad variant of this
            algorithm from the paper `On the Convergence of Adam and Beyond`_
            (default: False)
        z	
        a=  
    .. Note::
        A prototype implementation of Adam and AdamW for MPS supports `torch.float32` and `torch.float16`.
    .. _Adam\: A Method for Stochastic Optimization:
        https://arxiv.org/abs/1412.6980
    .. _On the Convergence of Adam and Beyond:
        https://openreview.net/forum?id=ryQu7f-RZ

    r&   re   rf   rg   rh   ri   rm   rn   r+   rj   ro   rp   r'   r*   r)   r!   r"   r#   r%   r,   c          
      óŒ  — |€|�t          d¦  «        ‚t          j                             ¦   «         r�t	          |t
          ¦  «        st          dt          |¦  «        › �¦  «        ‚t	          |
t
          ¦  «        st          dt          |
¦  «        › �¦  «        ‚t	          |t
          ¦  «        st          dt          |¦  «        › �¦  «        ‚n-t          |¦  «        }t          |
¦  «        }
t          |¦  «        }t	          |
t          ¦  «        r|
j	        |
j
        f|
i}nd }t          | ¦  «        D �]�\  }}|s||         n||          }||         }||         }||         }t          j                             ¦   «         sK|rIt          ¦   «         }|j	        j        |j	        j        k    r|j	        j        |v st          d|› d�¦  «        ‚|dz  }|dk    r”|r|                     d||z  z
  ¦  «         nv|r]t	          |t          ¦  «        rH|j        r)|                     |                     ¦   «         |¦  «        }n/|                     ||¬	¦  «        }n|                     ||¬	¦  «        }t          j        |¦  «        rot          j        |¦  «        }t          j        |¦  «        }t          j        |¦  «        }|rt          j        ||         ¦  «        ||<   t          j        |¦  «        }|j	        }|�3|j
        }||f}||vr|
                     ||d
¬¦  «        ||<   ||         }n|
}|                     |d|z
  ¦  «         |r‹t	          |t          ¦  «        rv|j        r-|                     t          j        |¦  «        d|z
  ¬¦  «         np|                     |¦  «                             ||t3          t
          d|z
  ¦  «        ¬¦  «         n.|                     |¦  «                             ||d|z
  ¬¦  «         |s|�rÁ|}|r@t	          |
t          ¦  «        r+|
j        rd|
|                     ¦   «         z  z
  } nd|
|z  z
  } nd|
|z  z
  } |r@t	          |t          ¦  «        r+|j        rd||                     ¦   «         z  z
  }!nd||z  z
  }!nd||z  z
  }!|| z  }"|"                     ¦   «         }#|!                     ¦   «         }$|rŠ|r||                              ¦   «         }%n||         }%||                              t          j        |%|¦  «        ¦  «         ||                              ¦   «         |$|#z  z                       ||#z  ¦  «        }&n0|                     ¦   «         |$|#z  z                       ||#z  ¦  «        }&|r)|                     |                     ¦   «         |&¦  «         nÙ|                     ||&¦  «         nÂtA          |¦  «        }d|
|z  z
  } d||z  z
  }!|| z  }"|!dz  }$|rTt          j        ||         |||         ¬¦  «         ||                              ¦   «         |$z                       |¦  «        }&n*|                     ¦   «         |$z                       |¦  «        }&|                     ||&|" ¬¦  «         |r7t          j        | |         ¦  «        rt          j!        ||         ¦  «        ||<   �Œ‘d S )Nú,Expected grad_scale and found_inf to be Nonez#Expected lr to be a float, but got z&Expected beta1 to be a float, but got z&Expected beta2 to be a float, but got úIIf capturable=True, params and state_steps must be on supported devices: ú.r   r   ©ÚalphaT)rG   rF   Únon_blocking)Úweight)Úvalueç      à?)Úout)"ÚAssertionErrorrN   ÚjitÚis_scriptingr2   r5   Útyper   r   rG   rF   Ú	enumerateÚcompilerÚis_compilingr   Úmul_rc   Úaddcmul_ÚcloneÚaddr]   Úview_as_realÚtoÚlerp_Úsquarer   ÚnegÚsqrtÚcopy_ÚmaximumÚadd_Úaddcdiv_r   Úview_as_complex)'r&   re   rf   rg   rh   ri   rm   rn   r+   rj   ro   rp   r'   r*   r)   r!   r"   r#   r%   Ú
beta1_dictÚiÚparamr\   rY   rZ   Ústep_tÚcapturable_supported_devicesrG   rF   ÚkeyÚdevice_beta1rB   Úbias_correction1Úbias_correction2Ú	step_sizeÚstep_size_negÚbias_correction2_sqrtr[   Údenoms'                                          r?   Ú_single_tensor_adamrª   [  sj  € ð, Ð Ð!6ÝÐKÑLÔLÐLå„y×ÒÑÔð "õ ˜"�eÑ$Ô$ð 	SÝ Ð!QÅtÈBÁxÄxÐ!QÐ!QÑRÔRÐRÝ˜%¥Ñ'Ô'ð 	YÝ Ð!WÍ$ÈuÉ+Ì+Ð!WÐ!WÑXÔXÐXÝ˜%¥Ñ'Ô'ð 	YÝ Ð!WÍ$ÈuÉ+Ì+Ð!WÐ!WÑXÔXÐXð	Yõ ˜‰^Œ^ˆÝ˜5Ñ!Ô!ˆÝ˜5Ñ!Ô!ˆõ �%�Ñ Ô ð Ø/4¬|¸U¼[Ð.IÈ5Ð-Qˆ
ˆ
àˆ
å˜fÑ%Ô%ð ZKñ ZK‰ˆˆ5Ø'Ð6ˆu�QŒxˆx¨e°A¬h¨YˆØ˜1”+ˆØ  ”^ˆ
Ø˜Q”ˆõ Œ~×*Ò*Ñ,Ô,ð 	°ð 	Ý+LÑ+NÔ+NÐ(à”Ô! V¤]Ô%7Ò7Ð7Ø”LÔ%Ð)EÐEÐEå$ØÐ`|ÐÐÐñô ð ð
 	�!‰ˆà˜1ÒÐØ%ð ?à—
’
˜1˜r LÑ0Ñ0Ñ1Ô1Ð1Ð1ð "ð ?¥j°½vÑ&FÔ&Fð ?Ø#Ô1ð CØ#Ÿ}š}¨U¯[ª[©]¬]¸LÑIÔI˜˜à#Ÿxšx¨°\˜xÑBÔB˜˜àŸ8š8 E°˜8Ñ>Ô>�DåÔ˜EÑ"Ô"ð 	.ÝÔ% dÑ+Ô+ˆDÝÔ(¨Ñ1Ô1ˆGÝÔ+¨JÑ7Ô7ˆJØð LÝ%*Ô%7¸ÈÔ8JÑ%KÔ%K� Ñ"ÝÔ& uÑ-Ô-ˆEà”ˆàÐ!Ø”KˆEð ˜5�/ˆCØ˜*Ð$Ð$Ø"'§(¢(Ø!¨¸Tð #+ñ #ô #�
˜3‘ð ,6°c¬?ˆLˆLà ˆLð 	�Š�d˜A Ñ,Ñ-Ô-Ð-ð ð 	I�j¨µÑ7Ô7ð 	IØÔ"ð ð × Ò ¥¤¨dÑ!3Ô!3¸AÀ¹IÐ ÑFÔFÐFÐFà—’ Ñ&Ô&×/Ò/Ø˜$¥d­5°!°e±)Ñ&<Ô&<ð 0ñ ô ð ð ð �OŠO˜EÑ"Ô"×+Ò+¨D°$¸aÀ%¹iÐ+ÑHÔHÐHàð E	=˜ñ E	=ØˆDð ð 3¥*¨UµFÑ";Ô";ð 3ØÔ&ð 7Ø'(¨5°D·J²J±L´LÑ+@Ñ'@Ð$Ð$à'(¨5°$©;¡Ð$Ð$à#$ u¨d¡{¡?Ð ð ð 3¥*¨UµFÑ";Ô";ð 3ØÔ&ð 7Ø'(¨5°D·J²J±L´LÑ+@Ñ'@Ð$Ð$à'(¨5°$©;¡Ð$Ð$à#$ u¨d¡{¡?Ð àÐ-Ñ-ˆIØ%ŸMšM™OœOˆMà$4×$9Ò$9Ñ$;Ô$;Ð!àð ,à!ð 8Ø%4°QÔ%7×%=Ò%=Ñ%?Ô%?�N�Nà%4°QÔ%7�Nà Ô"×(Ò(­¬°~ÀzÑ)RÔ)RÑSÔSÐSð $ AÔ&×+Ò+Ñ-Ô-Ð1FÈÑ1VÑWß’$�s˜]Ñ*Ñ+Ô+ð �ð
 —O’OÑ%Ô%Ð)>ÀÑ)NÑOß’$�s˜]Ñ*Ñ+Ô+ð ð ð /Ø—’˜wŸ}š}™œ°Ñ6Ô6Ð6Ð6à—’˜w¨Ñ.Ô.Ð.Ð.å˜fÑ%Ô%ˆDà  5¨$¡;™ÐØ  5¨$¡;™ÐàÐ-Ñ-ˆIà$4°cÑ$9Ð!àð Nå”˜o¨aÔ0°*À/ÐRSÔBTÐUÑUÔUÐUð )¨Ô+×0Ò0Ñ2Ô2Ð5JÑJ×PÒPÐQTÑUÔU��à#ŸšÑ*Ô*Ð-BÑB×HÒHÈÑMÔM�à�NŠN˜7 E°)°ˆNÑ<Ô<Ð<ð ð 	K•uÔ'¨¨q¬	Ñ2Ô2ð 	KÝ!&Ô!6°ÀqÔ7IÑ!JÔ!JˆO˜AÑùðuZKð ZKr@   c          
      ó  ‡
‡‡‡+— t          | ¦  «        dk    rd S t          ‰t          ¦  «        r8|st          d¦  «        ‚‰                     ¦   «         dk    rt          d¦  «        ‚t          ‰
t          ¦  «        r8|st          d¦  «        ‚‰
                     ¦   «         dk    rt          d¦  «        ‚t          ‰t          ¦  «        r8|st          d¦  «        ‚‰                     ¦   «         dk    rt          d¦  «        ‚t          j                             ¦   «         sP|rNt          d	¬
¦  «        Š+t          ˆ+fd„t          | |d¬¦  «        D ¦   «         ¦  «        st          d‰+› d�¦  «        ‚|€|�t          d¦  «        ‚|rt          d¦  «        ‚t          ‰¦  «        Št          ‰
¦  «        Š
t          ‰¦  «        Št          j        | |||||g¦  «        }t          ‰
t          ¦  «        r!t!          ‰
j        ¦  «        dk    r	‰
j        ‰
ind }|                     ¦   «         D �]ä\  \  }}}}}}}t'          t(          t                   |¦  «        }t'          t(          t                   |¦  «        }t'          t(          t                   |¦  «        }t'          t(          t                   |¦  «        }t'          t(          t                   |¦  «        } |d         j        }!|�|!|vr‰
                     |!d¬¦  «        ||!<   |r||!         n‰
}"|	rH|r4t'          t(          t                   |¦  «        }#t-          |||||#¦  «         nt-          ||||¦  «         |rt          j        |¦  «        }t          j                             ¦   «         s9| d         j        r,t          j        | t          j        dd¬¦  «        d¬¦  «         nt          j        | d¦  «         |dk    rO|rt          j        |d‰|z  z
  ¦  «         n1|rt          j        |||¬¦  «         nt          j        |||¬¦  «        }t          j        ||t'          t<          d|"z
  ¦  «        ¦  «         t          j        |‰¦  «         t          ‰t          j        ¦  «        rt          j        |d‰z
  ¦  «        }$d}%n|}$d‰z
  }%t          j         ||$||%¦  «         ~~$|�r`t          j!        ‰
| ¦  «        }&t          j!        ‰| ¦  «        }'t          j"        |&d¦  «         t          j"        |'d¦  «         t          j#        |'¦  «         t          j$        |&‰¦  «         t          j%        |&¦  «         t          j&        |'¦  «         |&}(|'})|rJt'          t(          t                   |¦  «        }#t          j'        |#|¦  «         t          j(        |#¦  «        }*nt          j(        |¦  «        }*t          j$        |*|)¦  «         t          j        |*|¦  «         t          j$        |*|(¦  «         t          j)        |||*¦  «         �Œ ˆ
fd„| D ¦   «         }&ˆfd„| D ¦   «         }'tU          ˆfd„|&D ¦   «         ¦  «        }(d„ |'D ¦   «         })|rJt'          t(          t                   |¦  «        }#t          j'        |#|¦  «         t          j(        |#¦  «        }*nt          j(        |¦  «        }*t          j$        |*|)¦  «         t          j        |*|¦  «         t          j)        |||*|(¦  «         �Œæd S )Nr   r.   r   r/   zHbeta1 as a Tensor is not supported for capturable=False and foreach=TruezTensor beta1 must be 1-elementzHbeta2 as a Tensor is not supported for capturable=False and foreach=TruezTensor beta2 must be 1-elementF)Úsupports_xlac              3   ón   •K  — | ]/\  }}|j         j        |j         j        k    o|j         j        ‰v V — Œ0d S rv   )rG   rŠ   )Ú.0rR   rB   r¡   s      €r?   ú	<genexpr>z%_multi_tensor_adam.<locals>.<genexpr>_  s]   øè è € ð 
ð 
ñ ��4ð ŒHŒM˜Tœ[Ô-Ò-ð >Ø””Ð!=Ð=ð
ð 
ð 
ð 
ð 
ð 
r@   T)Ústrictr~   r   r}   z#_foreach ops don't support autogradrW   ©rG   r‚   r1   )rG   r€   c                 ó:   •— g | ]}d ‰t          |¦  «        z  z
  ‘ŒS ©r   ©r   )r®   rB   ro   s     €r?   ú
<listcomp>z&_multi_tensor_adam.<locals>.<listcomp>  ó8   ø€ ð  ð  ð  Ø26��E�Z¨Ñ-Ô-Ñ-Ñ-ð ð  ð  r@   c                 ó:   •— g | ]}d ‰t          |¦  «        z  z
  ‘ŒS r³   r´   )r®   rB   rp   s     €r?   rµ   z&_multi_tensor_adam.<locals>.<listcomp>  r¶   r@   c                 ó    •— g | ]
}‰|z  d z  ‘ŒS )éÿÿÿÿrV   )r®   Úbcr'   s     €r?   rµ   z&_multi_tensor_adam.<locals>.<listcomp>
  s!   ø€ Ð,WÐ,WÐ,WÀ¨b°2©g¸©^Ð,WÐ,WÐ,Wr@   c                 ó   — g | ]}|d z  ‘ŒS )r…   rV   )r®   rº   s     r?   rµ   z&_multi_tensor_adam.<locals>.<listcomp>  s   € Ð$HÐ$HÐ$H° R¨¡WÐ$HÐ$HÐ$Hr@   )+rM   r2   r   r:   r4   r3   rN   rŒ   r�   r   ÚallÚzipr‡   r   r   Ú"_group_tensors_by_device_and_dtypeÚstrrG   Úvaluesr   Úlistr“   r   Ú_foreach_negÚis_cpuÚ_foreach_add_rP   Ú_foreach_mul_Ú_foreach_addÚ_foreach_lerp_r5   Ú_foreach_mulÚ_foreach_addcmul_Ú_foreach_powÚ_foreach_sub_Ú_foreach_neg_Ú_foreach_div_Ú_foreach_reciprocal_Ú_foreach_sqrt_Ú_foreach_maximum_Ú_foreach_sqrtÚ_foreach_addcdiv_r   ),r&   re   rf   rg   rh   ri   rm   rn   r+   rj   ro   rp   r'   r*   r)   r!   r"   r#   r%   Úgrouped_tensorsr�   Údevice_params_Údevice_grads_Údevice_exp_avgs_Údevice_exp_avg_sqs_Údevice_max_exp_avg_sqs_Údevice_state_steps_Ú_Údevice_paramsÚdevice_gradsÚdevice_exp_avgsÚdevice_exp_avg_sqsÚdevice_state_stepsrG   r£   Údevice_max_exp_avg_sqsÚscaled_device_gradsr„   r¤   r¥   r¦   r¨   Úexp_avg_sq_sqrtr¡   s,             ```                              @r?   Ú_multi_tensor_adamrã   )  s  øøøø€ õ, ˆ6�{„{�aÒÐØˆå�"•fÑÔð <Øð 	ÝØWñô ð ð �8Š8‰:Œ:˜Š?ˆ?ÝÐ:Ñ;Ô;Ð;å�%�Ñ Ô ð ?Øð 	ÝØZñô ð ð �;Š;‰=Œ=˜AÒÐÝÐ=Ñ>Ô>Ð>å�%�Ñ Ô ð ?Øð 	ÝØZñô ð ð �;Š;‰=Œ=˜AÒÐÝÐ=Ñ>Ô>Ð>õ Œ>×&Ò&Ñ(Ô(ð ¨Zð Ý'HØð(
ñ (
ô (
Ð$õ ð 
ð 
ð 
ð 
õ ˜v {¸4Ð@Ñ@Ô@ð
ñ 
ô 
ñ 
ô 
ð 	õ
 !Ø{Ð\xÐ{Ð{Ð{ñô ð ð Ð Ð!6ÝÐKÑLÔLÐLàð DÝÐBÑCÔCÐCå	�B‰Œ€BÝ�uÑÔ€EÝ�uÑÔ€EåÔBØ	�˜ +¨ÀÐLñô €Oõ �e�VÑ$Ô$ð	Ý),¨U¬\Ñ):Ô):¸eÒ)CÐ)Cð 
Œ�uÐÐàð ð ×"Ò"Ñ$Ô$ðañ añ 		ñ 	ØØØØØØØÝ�T¥&œ\¨>Ñ:Ô:ˆÝ�D¥œL¨-Ñ8Ô8ˆÝ�t¥Fœ|Ð-=Ñ>Ô>ˆÝ!¥$¥v¤,Ð0CÑDÔDÐÝ!¥$¥v¤,Ð0CÑDÔDÐà˜qÔ!Ô(ˆØÐ! f°JÐ&>Ð&>Ø!&§¢°Àd Ñ!KÔ!KˆJ�vÑà-7ÐB�z &Ô)Ð)¸Uˆð ð 	Øð Ý)-­dµ6¬lÐ<SÑ)TÔ)TÐ&ÝØ!Ø Ø#Ø&Ø*ñô ð ð õ Ø! <°ÐBTñô ð ð ð 	<Ý Ô-¨lÑ;Ô;ˆLõ Œ~×*Ò*Ñ,Ô,ð 	7Ð1CÀAÔ1FÔ1Mð 	7ÝÔØ"¥E¤L°¸UÐ$CÑ$CÔ$CÈ3ðñ ô ð ð õ ÔÐ 2°AÑ6Ô6Ð6à˜1ÒÐØ%ð 
åÔ# M°1°r¸LÑ7HÑ3HÑIÔIÐIÐIð ð ÝÔ'¨°mÈ<ÐXÑXÔXÐXÐXå#(Ô#5Ø$ m¸<ð$ñ $ô $�Lõ 	ÔØ˜\­4µ°q¸<Ñ7GÑ+HÔ+Hñ	
ô 	
ð 	
õ 	ÔÐ.°Ñ6Ô6Ð6õ �e�Uœ\Ñ*Ô*ð 	Ý"'Ô"4°\À1ÀuÁ9Ñ"MÔ"MÐØˆEˆEà".ÐØ˜‘IˆEåÔØÐ 3°\À5ñ	
ô 	
ð 	
ð
 Øð ñ B	Ý$Ô1°%Ð9KÑLÔLÐÝ$Ô1°%Ð9KÑLÔLÐåÔÐ 0°!Ñ4Ô4Ð4ÝÔÐ 0°!Ñ4Ô4Ð4åÔÐ 0Ñ1Ô1Ð1õ ÔÐ 0°"Ñ5Ô5Ð5ÝÔ&Ð'7Ñ8Ô8Ð8åÔ Ð!1Ñ2Ô2Ð2ð
 )ˆIØ$4Ð!àð JÝ)-­dµ6¬lÐ<SÑ)TÔ)TÐ&åÔ'Ð(>Ð@RÑSÔSÐSõ #(Ô"5Ð6LÑ"MÔ"M��å"'Ô"5Ð6HÑ"IÔ"I�åÔ Ð1FÑGÔGÐGÝÔ °Ñ5Ô5Ð5ÝÔ °Ñ;Ô;Ð;õ Ô# M°?ÀOÑTÔTÐTÑTð ð  ð  ð  Ø:Lð ñ  ô  Ðð ð  ð  ð  Ø:Lð ñ  ô  Ðõ ,Ð,WÐ,WÐ,WÐ,WÐFVÐ,WÑ,WÔ,WÑXÔXˆIà$HÐ$HÐ7GÐ$HÑ$HÔ$HÐ!àð JÝ)-­dµ6¬lÐ<SÑ)TÔ)TÐ&åÔ'Ð(>Ð@RÑSÔSÐSõ #(Ô"5Ð6LÑ"MÔ"M��å"'Ô"5Ð6HÑ"IÔ"I�åÔ Ð1FÑGÔGÐGÝÔ °Ñ5Ô5Ð5ÝÔ#ØØØØñ	ô ð ñ ðyað ar@   c                óÆ  — | sd S |rt          d¦  «        ‚t          |
¦  «        }
t          |¦  «        }|�	|j        |ini }|�	|j        |ini }t          |t          ¦  «        r!t          |j        ¦  «        dk    r	|j        |ind }t          j        | |||||g¦  «        }|                     ¦   «         D �]«\  \  }}\  \  }}}}}}}t          t          t                   |¦  «        }t          t          t                   |¦  «        } t          t          t                   |¦  «        }!t          t          t                   |¦  «        }"t          t          t                   |¦  «        }#d\  }$}%|�+|                     ||                     |d¬¦  «        ¦  «        }$|�+|                     ||                     |d¬¦  «        ¦  «        }%|�&||vr"|                     |d¬¦  «        ||<   ||         }t          j        |#d¦  «         |st          j        nt          j        }& |&|| |!|"||#|||
|||||$|%¬¦  «         |%�&t          j        |#|%gt%          |#¦  «        z  ¦  «         �Œ­d S )	Nz9Adam with fused=True does not support differentiable=TruerW   )NNT)r‚   r±   r   )	r+   r'   ro   rp   r*   r)   r!   rm   rn   )r:   r   rG   r2   r   r¿   r   r¾   Úitemsr   rÁ   rJ   r“   rN   rÄ   Ú_fused_adam_Ú_fused_adamw_rË   rM   )'r&   re   rf   rg   rh   ri   rm   rn   r+   rj   ro   rp   r'   r*   r)   r!   r"   r#   r%   Úgrad_scale_dictÚfound_inf_dictÚlr_dictrÓ   rG   rÚ   rÔ   rÕ   rÖ   r×   rà   rÙ   rÛ   rÜ   rÝ   rÞ   rß   Údevice_grad_scaleÚdevice_found_infÚfuncs'                                          r?   Ú_fused_adamrî   "  sÑ  € ð, ð ØˆØð XÝÐVÑWÔWÐWå�uÑÔ€EÝ�uÑÔ€Eð ,6Ð+AˆÔ	˜JÐ'Ð'Àrð ð *3Ð)>ˆÔ	˜9Ð%Ð%ÀBð õ & b­&Ñ1Ô1ÐWµc¸"¼)±n´nÈÒ6MÐ6MˆŒ�BˆˆÐSWð õ  ÔBØ	�˜ +¨ÀÐLñô €Oð 
×	Ò	Ñ	 Ô	 ð4ñ 4ñ 
	‰ˆ�ñ 
ñ	
ØØØØØ"Øà	å�T¥&œ\¨>Ñ:Ô:ˆÝ�D¥œL¨-Ñ8Ô8ˆÝ�t¥Fœ|Ð-=Ñ>Ô>ˆÝ!¥$¥v¤,Ð0CÑDÔDÐÝ!¥$¥v¤,Ð0CÑDÔDÐà.8Ñ+ÐÐ+ØÐ!Ø /× :Ò :Ø˜
Ÿš f¸4˜Ñ@Ô@ñ!ô !Ðð Ð Ø-×8Ò8Ø˜	Ÿš V¸$˜Ñ?Ô?ñ ô  Ðð Ð 6°Ð#8Ð#8Ø Ÿeše¨6À˜eÑEÔEˆG�F‰OØ˜”ˆBÝÔÐ.°Ñ2Ô2Ð2Ø)?ÐX�uÔ!Ð!ÅUÔEXˆàˆØØØØØ"ØØØØØØ%ØØØ(Ø&ð	
ñ 	
ô 	
ð 	
ð" Ð'ÝÔØ"Ð%5Ð$6½Ð=OÑ9PÔ9PÑ$Pñô ð ùðe4ð 4r@   )Úsingle_tensor_fnFr    r$   c                ó¨  — |	€2|€0t          | |d¬¦  «        \  }}|rt          |t          ¦  «        r|sd}|	€d}	|€d}t          j                             ¦   «         s(t          d„ |D ¦   «         ¦  «        st          d¦  «        ‚|r-t          j         	                    ¦   «         rt          d¦  «        ‚|	r-t          j         	                    ¦   «         rt          d¦  «        ‚|	r&t          j         	                    ¦   «         st          }n/|r&t          j         	                    ¦   «         st          }nt          } || |||||f|||||||||||
||dœŽ dS )	znFunctional API that performs Adam algorithm computation.

    See :class:`~torch.optim.Adam` for details.
    NF)Ú	use_fusedc              3   óJ   K  — | ]}t          |t          j        ¦  «        V — Œd S rv   )r2   rN   r   )r®   Úts     r?   r¯   zadam.<locals>.<genexpr>·  s?   è è € ð 5ð 5Ø()�
�1•e”lÑ#Ô#ð5ð 5ð 5ð 5ð 5ð 5r@   zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsz6torch.jit.script not supported with foreach optimizersz4torch.jit.script not supported with fused optimizers)r+   rj   ro   rp   r'   r*   r)   r!   r"   r#   rm   rn   r%   )r   r2   r   rN   rŒ   r�   r¼   r:   rˆ   r‰   rî   rã   rª   )r&   re   rf   rg   rh   ri   r    r"   r#   r$   rm   rn   rj   r%   r+   ro   rp   r'   r*   r)   r!   rÚ   rí   s                          r?   r   r   †  sÒ  € ðF €}˜˜Ý1Ø�N¨eð
ñ 
ô 
‰
ˆˆ7ð ð 	•z "¥fÑ-Ô-ð 	°jð 	ØˆGØ€}ØˆØ€Øˆõ Œ>×&Ò&Ñ(Ô(ð 
µð 5ð 5Ø-8ð5ñ 5ô 5ñ 2ô 2ð 
õ Ø^ñ
ô 
ð 	
ð ð U•5”9×)Ò)Ñ+Ô+ð UÝÐSÑTÔTÐTØð S•”×'Ò'Ñ)Ô)ð SÝÐQÑRÔRÐRàð #•U”Y×+Ò+Ñ-Ô-ð #ÝˆˆØ	ð #�œ×/Ò/Ñ1Ô1ð #Ý!ˆˆå"ˆà€DØØØØØØðð ØØØØØ!ØØØØ%ØØØ5ð'ð ð ð ð ð r@   )NFFNNNFF)#Útypingr   rN   r   Ú	optimizerr   r   r   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   Ú__all__r   Ú__doc__rÁ   rz   r5   rª   rã   rî   r   rV   r@   r?   ú<module>rø      s¶  ðà Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð0 �6Ð
€ðnð nð nð nð nˆ9ñ nô nð nðd$ðJà	ðð ð$ 
ð%ð ð& 
ð'ð ð( 
ð)ð ð* 
ð+ð ð, 
ð-ð ð ñKCð „ðPKKØ�ŒLðKKà�Œ<ðKKð �6ŒlðKKð �f”ð	KKð
 ˜&”\ðKKð �f”ðKKð ˜‘ðKKð ˜‰}ðKKð ðKKð ðKKð �6‰>ðKKð �6‰>ðKKð 	�‰ðKKð ðKKð  
ð!KKð" ð#KKð$ ð%KKð& ð'KKð( !ð)KKð* 
ð+KKð KKð KKð KKð\vØ�ŒLðvà�Œ<ðvð �6Œlðvð �f”ð	vð
 ˜&”\ðvð �f”ðvð ˜‘ðvð ˜‰}ðvð ðvð ðvð �6‰>ðvð �6‰>ðvð 	�‰ðvð ðvð  
ð!vð" ð#vð$ ð%vð& ð'vð( !ð)vð* 
ð+vð vð vð vðraØ�ŒLðaà�Œ<ðað �6Œlðað �f”ð	að
 ˜&”\ðað �f”ðað ˜‘ðað ˜‰}ðað ðað ðað �6‰>ðað �6‰>ðað 	�‰ðað ðað  
ð!að" ð#að$ ð%að& ð'að( !ð)að* 
ð+að að að aðH  ÐÐ1DÐEÑEÔEð  ØØ ØØ $Ø#ØØ#(ð!Wð WØ�ŒLðWà�Œ<ðWð �6ŒlðWð �f”ð	Wð
 ˜&”\ðWð �f”ðWð �D‰[ðWð ðWð ðWð �$‰;ðWð ˜‘ðWð ˜‰}ðWð ðWð  !ð!Wð$ ð%Wð& �6‰>ð'Wð( �6‰>ð)Wð* 	�‰ð+Wð, ð-Wð. 
ð/Wð0 ð1Wð2 
ð3Wð Wð Wñ FÔEðWð Wð Wr@   