§
    ŠŠtj{j ã            $       óÖ  — d dl Z d dlZd dlZd dlmZmZ d dlmZ d dlm	Z	 d dl
Z
d dlZ
d dlmZmZ d dlmZ d dlmZmZ g d¢Z G d	„ d
e¦  «        Zde
j        fd„Zde
j        fd„Zd„ Zdedee
j        df         fd„Z	 d{dedee
j        df         fd„Z	 d|de
j        ee
j                 z  dedee
j                 fd„Zd„ Z d„ Z!	 d}de"ee
j        df                  fd„Z# e	de$¬¦  «        d~d „¦   «         Z%d!„ Z&	 d|de"e
j                 fd"„Z'd#e(e)e"e
j                 f         dee
j        df         fd$„Z*	 d|d%e
j        d&e
j        dz  de
j        fd'„Z+dd€d(„Z,	 d�dee
j        df         fd)„Z-ddd*œdeee
j        df         df         fd+„Z.d,„ Z/d|d-„Z0d.„ Z1d/„ Z2d0„ Z3	 d|de"e
j                 fd1„Z4d2„ Z5d3„ Z6deee
j        df         eef         fd4„Z7d5Z8	 d‚dee
j        df         fd6„Z9d7„ Z: e	d8e$¬¦  «        dƒd:„¦   «         Z;d;„ Z<de"e"e
j        dz                    fd<„Z=de"e"e
j        dz                    fd=„Z>defd>„Z?d€d?„Z@defd@„ZAdA„ ZBdB C                    ¦   «         ZDdCZE	 d|dD„ZFdefdE„ZGdefdF„ZHdefdG„ZIdH„ ZJdefdI„ZKdJ„ ZLdK„ ZM	 	 d„deNfdL„ZOdM„ ZPdN„ ZQdO„ ZRdP„ ZSdddddQœdR„ZTdS„ ZUdT„ ZVdU„ ZWd|dV„ZXdW„ ZYdX„ ZZdY C                    ¦   «         Z[dZ„ Z\d[„ Z]d\„ Z^dd]œd^„Z_dddddQœd_„Z`d`daddbd9dbdddddbdddcœddedef         deedfeadgeadheadiedjeadkedledmednedoedpedqedredz  def ds„Zbdt„ Zc	 dd`dadddbd9dbddddbddduœddedef         deedvedz  dfeadgeadheadwediedjeadkedledmedxedyedqedredef"dz„ZddS )…é    N)ÚCallableÚIterable)Úproduct)Ú
deprecated)Ú_vmapÚvmap)Úis_tensor_like)Ú_TensorOrOptionalTensorsÚ_TensorOrTensors)Ú	gradcheckÚgradgradcheckÚGradcheckErrorÚget_numerical_jacobianÚget_analytical_jacobianÚ)get_numerical_jacobian_wrt_specific_inputc                   ó   — e Zd ZdZdS )r   z<Error raised by :func:`gradcheck` and :func:`gradgradcheck`.N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__© ó    úV/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/torch/autograd/gradcheck.pyr   r      s   € € € € € ØGÐGÐGÐGr   r   Úobjc                 ól   — | j         t          j        t          j        t          j        t          j        hv S ©N)ÚlayoutÚtorchÚ
sparse_csrÚ
sparse_cscÚ
sparse_bsrÚ
sparse_bsc©r   s    r   Ú_is_sparse_compressed_tensorr$   #   s.   € ØŒ:ÝÔÝÔÝÔÝÔð	ð ð r   c                 óF   — t          | ¦  «        p| j        t          j        u S r   )r$   r   r   Ú
sparse_coor#   s    r   Ú_is_sparse_any_tensorr'   ,   s   € Ý'¨Ñ,Ô,ÐN°´
½eÔ>NÐ0NÐNr   c                 óp   — t          | ¦  «        o'|                      ¦   «         p|                      ¦   «         S r   )r	   Úis_floating_pointÚ
is_complexr#   s    r   Ú_is_float_or_complex_tensorr+   0   s0   € Ý˜#ÑÔÐP C×$9Ò$9Ñ$;Ô$;Ð$O¸s¿~º~Ñ?OÔ?OÐPr   Úinput_tensorsÚreturn.c                 ó>   ‡— ˆfd„| D ¦   «         }t          |¦  «        S )Nc                 ó¦   •— g | ]M}t          |¦  «        ¯|j        ¯|                     |                     ¦   «         ‰ft          j        ¬ ¦  «        ‘ŒNS )©r   )r+   Úrequires_gradÚ	new_zerosÚnumelr   Ústrided)Ú.0ÚtÚnumel_outputs     €r   ú
<listcomp>z3_allocate_jacobians_with_inputs.<locals>.<listcomp><   sb   ø€ ð ð ð àÝ& qÑ)Ô)ðð /0¬oðØ	�Š�Q—W’W‘Y”Y Ð-µe´mˆÑDÔDðð ð r   )Útuple)r,   r7   Úouts    ` r   Ú_allocate_jacobians_with_inputsr;   4   s9   ø€ ðð ð ð àðñ ô €Cõ
 �‰:Œ:Ðr   Úoutput_tensorsc                 ób   ‡‡— ||t           j        dœŠˆˆfd„| D ¦   «         }t          |¦  «        S )N)ÚdtypeÚdevicer   c                 ór   •— g | ]3}t          |¦  «        ¯ |j        ‰|                     ¦   «         ffi ‰¤Ž‘Œ4S r   )r+   r2   r3   )r5   r6   Únumel_inputÚoptionss     €€r   r8   z4_allocate_jacobians_with_outputs.<locals>.<listcomp>L   sX   ø€ ð ð ð àÝ& qÑ)Ô)ðØˆŒ�[ !§'¢'¡)¤)Ð,Ð8Ð8°Ð8Ð8ðð ð r   )r   r4   r9   )r<   rA   r>   r?   r:   rB   s    `   @r   Ú _allocate_jacobians_with_outputsrC   D   sR   øø€ ð ¨½5¼=ÐIÐI€Gðð ð ð ð àðñ ô €Cõ
 �‰:Œ:Ðr   FÚxÚonly_requiring_gradc              #   óð   K  — t          | ¦  «        r| j        s|s| V — d S d S t          | t          j        j        ¦  «        r0t          | t          ¦  «        s| D ]}t          ||¦  «        E d {V —† Œd S d S d S r   )r	   r1   Ú
isinstanceÚcollectionsÚabcr   ÚstrÚ_iter_tensors)rD   rE   Úelems      r   rK   rK   T   sÇ   è è € õ �aÑÔð @àŒ?ð 	Ð"5ð 	ØˆGˆGˆGˆGˆGð	ð 	å	�A•{”Ô/Ñ	0Ô	0ð @½ÀAÅsÑ9KÔ9Kð @Øð 	@ð 	@ˆDÝ$ TÐ+>Ñ?Ô?Ð?Ð?Ð?Ð?Ð?Ð?Ð?Ð?ð@ð @ð @ð @ð	@ð 	@r   c                 ó:	  — t          | t          t          f¦  «        r+ t          | ¦  «        t	          t
          | ¦  «        ¦  «        S t          | ¦  «        r| j        t          j	        t          j
        hv r| S | j        t          j        u �rü| j        }|                      ¦   «         j        }t          j        | j        d |                      ¦   «         …         t          j        |¬¦  «        }|                     ¦   «                              ¦   «                              |¬¦  «        }t          j        |                     ¦   «         g| j        |                      ¦   «         d …         ¢R | j        |¬¦  «        }|                      ¦   «                              ¦   «         }|                     ¦   «         dk    r�|                     ¦   «         }|                     ¦   «                              t          j        |||¬¦  «                             d¦  «        ¦  «                             d¦  «        }|                      ¦   «         ||<   t          j!        ||| j        ¦  «         "                    d¦  «         #                    | j$        ¦  «        S tK          | ¦  «        �rà| j        t          j&        t          j'        hv r!|                       ¦   «         j        dd…         nd }	| j        t          j(        t          j&        hv r|  )                    ¦   «         n|  *                    ¦   «         }
t          |                      ¦   «          +                    t          j        ¬¦  «        ¦  «         +                    | j        |	¬¦  «        }|                      ¦   «                              ¦   «         tY          d|                      ¦   «         j        d         ¦  «        z  }|
                     ¦   «         |
j        d	         z  }|                     ¦   «         tY          d||z  ¦  «        z  }|| -                    ¦   «         k    r.t]          | j        › d
|› d| -                    ¦   «         › �¦  «        ‚| #                    | j$        ¦  «        S t_          | ¦  «        rta          | j        ¦  «        ‚| S )N©r>   r?   ©r>   r   é   Té   r0   )r   Ú	blocksizeéÿÿÿÿz densify failed: expected nnz=z	 but got )1rG   Úlistr9   ÚtypeÚmapÚ_densifyr	   r   r   r4   Ú_mkldnnr&   r?   Ú_indicesr>   ÚonesÚshapeÚ
sparse_dimÚint8Únonzeror6   ÚtoÚzerosr3   ÚdetachÚcoalesceÚstrideÚindicesÚmulÚtensorÚ	unsqueezeÚsumÚvaluesÚsparse_coo_tensorÚ_coalesced_Úrequires_grad_r1   r$   r!   r"   r   Úcrow_indicesÚccol_indicesÚ	to_sparseÚmaxÚ_nnzÚAssertionErrorr'   ÚNotImplementedError)rD   r?   Úindices_dtypeÚtmprd   ri   Úx_coalescedrc   Úflat_indicesrR   Úcompressed_indicesÚrÚdense_numelÚbatch_numelÚsparse_numels                  r   rW   rW   `   sä  € õ �!•d�E�]Ñ#Ô#ð 7,Ø�t�A‰wŒw•s�8 QÑ'Ô'Ñ(Ô(Ð(Ý˜AÑÔð 5, !¤(­u¬}½e¼mÐ.LÐ"LÐ"LØˆØ	
Œ•UÔ%Ð	%Ñ	%Ø”ˆØŸ
š
™œÔ*ˆÝŒj˜œÐ!1 1§<¢<¡>¤>Ð!1Ô2½%¼*ÈVÐTÑTÔTˆØ—+’+‘-”-—/’/Ñ#Ô#×&Ò&¨]Ð&Ñ;Ô;ˆÝ”Ø�YŠY‰[Œ[Ð5˜1œ7 1§<¢<¡>¤>Ð#3Ð#3Ô4Ð5Ð5¸Q¼WÈVð
ñ 
ô 
ˆð —h’h‘j”j×)Ò)Ñ+Ô+ˆØ×ÒÑÔ Ò"Ð"Ø—Z’Z‘\”\ˆFà×#Ò#Ñ%Ô%ß’Ý”L ¨}ÀVÐLÑLÔL×VÒVØñô ñô ÷
 ’�Q‘”ð ð $/×#5Ò#5Ñ#7Ô#7ˆF�<Ñ åÔ# G¨V°Q´WÑ=Ô=ßŠ[˜ÑÔßŠ^˜AœOÑ,Ô,ð	
õ
 
& aÑ	(Ô	(ñ ,ð Œx�EÔ,­eÔ.>Ð?Ð?Ð?ð �HŠH‰JŒJÔ˜Q˜q˜SÔ!Ð!àð 	ð Œx�EÔ,­eÔ.>Ð?Ð?Ð?ð �NŠNÑÔÐà—’Ñ!Ô!ð 	õ �Q—X’X‘Z”Z×)Ò)µÔ1AÐ)ÑBÔBÑCÔC×MÒMØ”8 yð Nñ 
ô 
ˆð —h’h‘j”j×&Ò&Ñ(Ô(­C°°1·8²8±:´:Ô3CÀAÔ3FÑ,GÔ,GÑGˆØ(×.Ò.Ñ0Ô0Ð4FÔ4LÈRÔ4PÑPˆØ—w’w‘y”y¥C¨¨;¸Ñ+DÑ$EÔ$EÑEˆØ˜1Ÿ6š6™8œ8Ò#Ð#Ý Ø”8Ð\Ð\¸<Ð\Ð\ÐRS×RXÒRXÑRZÔRZÐ\Ð\ñô ð ð ×Ò ¤Ñ0Ô0Ð0Ý	˜qÑ	!Ô	!ð ,Ý! !¤(Ñ+Ô+Ð+Ø€Hr   c           
   #   ó8  ‡‡K  — t          | ¦  «        �rUd„ }|                      ¦   «         }t          |                      ¦   «         ¦  «        }| j        t
          j        u r<|                      ¦   «                              ¦   «         }|  	                    ¦   «         }�n	| j        t
          j
        u rat          j        |                      ¦   «         |                      ¦   «         ¦  «                             ¦   «         }|                      ¦   «         }�n•| j        t
          j        u rct          j        |                      ¦   «         |                      ¦   «         d¬¦  «                             ¦   «         }|                      ¦   «         }�n| j        t
          j        u �ro|                      ¦   «         }|                     ¦   «         dd…         }t          j        |                      ¦   «         |                      ¦   «         ¦  «                             |d         |d         z  d¦  «                             t          j        || j        ¬¦  «                             dd¦  «        ¦  «                             t          j        t          j        t          j        || j        ¬¦  «        ¦  «        ¦  «                             d|¦  «        ¦  «                             ¦   «         }|                     dd¦  «        }|                     d¦  «        }�nœ| j        t
          j        u �rp|                      ¦   «         }|                     ¦   «         dd…         }t          j        |                      ¦   «         |                      ¦   «         d¬¦  «                             |d         |d         z  d¦  «                             t          j        || j        ¬¦  «                             dd¦  «        ¦  «                             t          j        t          j        t          j        || j        ¬¦  «        ¦  «        ¦  «                             d|¦  «        ¦  «                             ¦   «         }|                     dd¦  «        }|                     d¦  «        }nt?          d	| j        › d
�¦  «        ‚ ||¦  «        Š|j         }tC          |¦  «        D ]¡}||         }	tE          d„ |                     ¦   «         dd …         D ¦   «         Ž D ]i}
||          #                    ¦   «         t          |
¦  «        z   ŠtI          ˆˆfd„tC          tK          |¦  «        ¦  «        D ¦   «         ¦  «        }|	|
|fV — ŒjŒ¢d S | j        t
          j&        k    rUtO          tE          d„ |                      ¦   «         D ¦   «         Ž ¦  «        D ] \  }}
|  (                    ¦   «         }||
|fV — Œ!d S | j         } tO          tE          d„ |                      ¦   «         D ¦   «         Ž ¦  «        D ]\  }}
| |
|fV — Œd S )Nc                 ó’   — t          | ¦  «        }d}dg|z  }t          t          |¦  «        ¦  «        D ]}|||<   || |         z  }Œ|S )NrP   r   )ÚlenÚreversedÚrange)ÚsizeÚdimru   rc   Úis        r   Ú
get_stridez _iter_tensor.<locals>.get_stride±   sW   € Ý�d‘)”)ˆCØˆCØ�S˜3‘YˆFÝ�e C™jœjÑ)Ô)ð ð �Ø��q‘	Ø�t˜A”w‘��ØˆMr   T)Ú	transposerP   rQ   r   ©r?   é   z_iter_tensor for z inputc                 ó,   — g | ]}t          |¦  «        ‘ŒS r   ©r�   ©r5   Úms     r   r8   z _iter_tensor.<locals>.<listcomp>ô   s   € Ð"IÐ"IÐ"I°¥5¨¡8¤8Ð"IÐ"IÐ"Ir   c              3   ó:   •K  — | ]}‰|         ‰|         z  V — Œd S r   r   )r5   Úkrd   Úx_strides     €€r   ú	<genexpr>z_iter_tensor.<locals>.<genexpr>ö   s0   øè è € ÐQÐQ¸˜G AœJ¨°!¬Ñ4ÐQÐQÐQÐQÐQÐQr   c                 ó,   — g | ]}t          |¦  «        ‘ŒS r   rŠ   r‹   s     r   r8   z _iter_tensor.<locals>.<listcomp>ù   ó   € Ð/RÐ/RÐ/R¸Qµ°a±´Ð/RÐ/RÐ/Rr   c                 ó,   — g | ]}t          |¦  «        ‘ŒS r   rŠ   r‹   s     r   r8   z _iter_tensor.<locals>.<listcomp>  r’   r   ))r'   rq   rT   r‚   r   r   r&   rY   r6   Ú_valuesr   Ú _convert_indices_from_csr_to_coorm   Úcol_indicesri   r    rn   Úrow_indicesr!   Úrepeat_interleaveÚmul_rf   r?   ÚreshapeÚadd_ÚstackÚwhererZ   ÚrepeatÚflattenr"   rs   Údatar�   r   Útolistrh   r   rX   Ú	enumerateÚto_dense)Úx_tensorr…   Úx_nnzÚx_sizeÚ	x_indicesÚx_valuesÚx_block_valuesÚx_blocksizer„   Úx_valueÚx_idxÚd_idxÚx_tensor_denserd   r�   s                @@r   Ú_iter_tensorr¯   ž   s£  øøè è € õ" ˜XÑ&Ô&ñ S)ð	ð 	ð 	ð —’‘”ˆÝ�h—m’m‘o”oÑ&Ô&ˆØŒ?�eÔ.Ð.Ð.Ø ×)Ò)Ñ+Ô+×-Ò-Ñ/Ô/ˆIØ×'Ò'Ñ)Ô)ˆH‰HØŒ_¥Ô 0Ð0Ð0ÝÔ>Ø×%Ò%Ñ'Ô'¨×)=Ò)=Ñ)?Ô)?ñô çŠa‰cŒcð ð  —’Ñ(Ô(ˆH‰HØŒ_¥Ô 0Ð0Ð0ÝÔ>Ø×%Ò%Ñ'Ô'¨×)=Ò)=Ñ)?Ô)?È4ðñ ô çŠa‰cŒcð ð  —’Ñ(Ô(ˆH‰HØŒ_¥Ô 0Ð0Ñ0Ø%Ÿ_š_Ñ.Ô.ˆNØ(×-Ò-Ñ/Ô/°°!°Ô4ˆKåÔ6Ø×)Ò)Ñ+Ô+¨X×-AÒ-AÑ-CÔ-Cñô ÷ #Ò" ;¨q¤>°KÀ´NÑ#BÀAÑFÔFß’•e”l ;°x´ÐGÑGÔG×OÒOÐPQÐSTÑUÔUÑVÔVß’Ý”KÝœ¥E¤J¨{À8Ä?Ð$SÑ$SÔ$SÑTÔTñô ç’f˜Q Ñ&Ô&ñô ÷
 ’‘”ð ð &×-Ò-¨a°Ñ3Ô3ˆHØ—M’M !Ñ$Ô$ˆE‰EØŒ_¥Ô 0Ð0Ñ0Ø%Ÿ_š_Ñ.Ô.ˆNØ(×-Ò-Ñ/Ô/°°!°Ô4ˆKåÔ6Ø×)Ò)Ñ+Ô+¨X×-AÒ-AÑ-CÔ-CÈtðñ ô ÷ #Ò" ;¨q¤>°KÀ´NÑ#BÀAÑFÔFß’•e”l ;°x´ÐGÑGÔG×OÒOÐPQÐSTÑUÔUÑVÔVß’Ý”KÝœ¥E¤J¨{À8Ä?Ð$SÑ$SÔ$SÑTÔTñô ç’f˜Q Ñ&Ô&ñô ÷
 ’‘”ð ð &×-Ò-¨a°Ñ3Ô3ˆHØ—M’M !Ñ$Ô$ˆEˆEå%Ð&Q¸(¼/Ð&QÐ&QÐ&QÑRÔRÐRØ�:˜fÑ%Ô%ˆà”=ˆÝ�u‘”ð 	,ð 	,ˆAØ˜q”kˆGÝ Ð"IÐ"I°X·]²]±_´_ÀQÀRÀRÔ5HÐ"IÑ"IÔ"IÐJð ,ð ,�Ø# Aœ,×-Ò-Ñ/Ô/µ$°u±+´+Ñ=�ÝÐQÐQÐQÐQÐQ½eÅCÈÁKÄKÑ>PÔ>PÐQÑQÔQÑQÔQ�Ø˜u eÐ+Ð+Ð+Ð+Ð+ð,ð	,ð 	,ð 
Œ�EœMÒ	)Ð	)Ý%¥gÐ/RÐ/RÀ(Ç-Â-Á/Ä/Ð/RÑ/RÔ/RÐ&SÑTÔTð 	/ð 	/‰LˆE�5ð &×.Ò.Ñ0Ô0ˆNØ  %¨Ð.Ð.Ð.Ð.Ð.ð		/ð 	/ð ”=ˆÝ%¥gÐ/RÐ/RÀ(Ç-Â-Á/Ä/Ð/RÑ/RÔ/RÐ&SÑTÔTð 	)ð 	)‰LˆE�5Ø˜E 5Ð(Ð(Ð(Ð(Ð(ð	)ð 	)r   çü©ñÒMbP?c                 óV  — g }|€t           | t          |¦  «        Ž ¦  «        }|s(t          d„ |D ¦   «         ¦  «        rt          d¦  «        ‚|€|}d„ t          |¦  «        D ¦   «         }t	          t          |d¦  «        |¦  «        D ]\  }}	|t          | |	|||||¬¦  «        gz  }Œ |S )aÉ  Compute the numerical Jacobian of `fn(inputs)` with respect to `target`.

    If not specified, targets are the input. Returns M * N Jacobians where N is the
    number of tensors in target that require grad and M is the number of non-integral
    outputs.

    Args:
        fn: the function to compute the jacobian for
        inputs: inputs to `fn`
        outputs: provide precomputed outputs to avoid one extra invocation of fn
        target: the Tensors wrt whom Jacobians are calculated (default=`inputs`)
        eps: the magnitude of the perturbation during finite differencing
             (default=`1e-3`)
        is_forward_ad: if this numerical jacobian is computed to be checked wrt
                       forward AD gradients (this is used for error checking only)

    Returns:
        A list of M N-tuples of tensors

    Note that `target` may not even be part of `input` to `fn`, so please be
    **very careful** in this to not clone `target`.
    Nc              3   ó>   K  — | ]}|                      ¦   «         V — Œd S r   ©r*   ©r5   Úos     r   r�   z*_get_numerical_jacobian.<locals>.<genexpr>!  s*   è è € Ð AÐ A°A §¢¡¤Ð AÐ AÐ AÐ AÐ AÐ Ar   zsExpected output to be non-complex. get_numerical_jacobian no longer supports functions that return complex outputs.c                 óD   — g | ]\  }}t          |¦  «        ¯|j        ¯|‘ŒS r   ©r	   r1   )r5   r„   Úas      r   r8   z+_get_numerical_jacobian.<locals>.<listcomp>(  sC   € ð ð ð Ùˆa�­>¸!Ñ+<Ô+<ðØABÄðØ	ðð ð r   T)ÚinputÚis_forward_ad)Ú	_as_tupleÚanyÚ
ValueErrorr¢   ÚziprK   r   )
ÚfnÚinputsÚoutputsÚtargetÚepsrº   Ú	jacobiansÚinp_indicesÚinpÚinp_idxs
             r   Ú_get_numerical_jacobianrÈ     s  € ð2 13€IØ€Ý˜B˜B¥	¨&Ñ 1Ô 1Ð2Ñ3Ô3ˆØð 
�SÐ AÐ A¸Ð AÑ AÔ AÑAÔAð 
ÝðEñ
ô 
ð 	
ð €~Øˆðð Ý Ñ'Ô'ðñ ô €Kõ �M¨&°$Ñ7Ô7¸ÑEÔEð 
ð 
‰ˆˆWØÝ5ØØØØØØØ+ðñ ô ð

ñ 
	
ˆ	ˆ	ð Ðr   aA  `get_numerical_jacobian` was part of PyTorch's private API and not meant to be exposed. We are deprecating it and it will be removed in a future version of PyTorch. If you have a specific use for this or feature request for this to be a stable API, please file us an issue at https://github.com/pytorch/pytorch/issues/new)Úcategoryç      ð?c                 ó�   ‡ — |dk    rt          d¦  «        ‚ˆ fd„}t          ||d||¦  «        }t          d„ |D ¦   «         ¦  «        S )aÈ  Compute the numerical Jacobian for a given fn and its inputs.

    This is a Deprecated API.

    Args:
        fn: the function to compute the Jacobian for (must take inputs as a tuple)
        inputs: input to `fn`
        target: the Tensors wrt whom Jacobians are calculated (default=`input`)
        eps: the magnitude of the perturbation during finite differencing
             (default=`1e-3`)
        grad_out: defaults to 1.0.

    Returns:
        A list of Jacobians of `fn` (restricted to its first output) with respect to
        each input or target, if provided.

    Note that `target` may not even be part of `input` to `fn`, so please be
    **very careful** in this to not clone `target`.
    rÊ   zaExpected grad_out to be 1.0. get_numerical_jacobian no longer supports values of grad_out != 1.0.c                  ó   •—  ‰| ¦  «        S r   r   )Úinpsr¿   s    €r   Úfn_pack_inpsz,get_numerical_jacobian.<locals>.fn_pack_inps^  s   ø€ Øˆr�$‰xŒxˆr   Nc              3   ó&   K  — | ]}|d          V — ŒdS )r   Nr   )r5   Újacobian_for_each_outputs     r   r�   z)get_numerical_jacobian.<locals>.<genexpr>c  s)   è è € ÐWÐWÐ1IÐ)¨!Ô,ÐWÐWÐWÐWÐWÐWr   )r½   rÈ   r9   )r¿   rÀ   rÂ   rÃ   Úgrad_outrÎ   rÄ   s   `      r   r   r   :  sr   ø€ ð: 	�CŠˆåð2ñ
ô 
ð 	
ð
ð ð ð ð õ (¨°f¸dÀFÈCÑPÔP€IåÐWÐWÈYÐWÑWÔWÑWÔWÐWr   c                 ó
  ‡‡‡— t          |¦  «        rÖ|j        |j        k    rt          d|j        › d|j        › �¦  «        ‚|                     ¦   «         |                     ¦   «         k    rAt          d|                     ¦   «         › d|                     ¦   «         › d|j        › �¦  «        ‚|                     ¦   «         }|                     ¦   «         }|                     ¦   «         }|                     ¦   «         }|                     ||z
  ¦  «          | ¦   «         }|                     ||z   ¦  «          | ¦   «         }|                     |¦  «         ˆˆfd„Št          ˆfd„t          ||¦  «        D ¦   «         ¦  «        S )Nz6Expected entry and v to have the same layout, but got ú and z3Expected entry and v to have the same nnz, but got z with entry shape c                 ó€   •—  ‰| |¦  «         || z
  d‰z  z  }|                      ¦   «                              d¦  «        S )Nrˆ   rS   )ra   rš   )r¸   ÚbÚretÚnbhd_checks_fnÚnorm_vs      €€r   Úcomputez,_compute_numerical_gradient.<locals>.compute„  sC   ø€ Øˆ�q˜!ÑÔÐØ�1‰u˜˜V™Ñ$ˆØ�zŠz‰|Œ|×#Ò# BÑ'Ô'Ð'r   c              3   ó6   •K  — | ]\  }} ‰||¦  «        V — Œd S r   r   )r5   r¸   rÕ   rÙ   s      €r   r�   z._compute_numerical_gradient.<locals>.<genexpr>‰  s1   øè è € Ð=Ð=¡6 A q��˜˜A‘”Ð=Ð=Ð=Ð=Ð=Ð=r   )r$   r   rr   rq   r[   ri   ra   ÚcloneÚcopy_r9   r¾   )	r¿   ÚentryÚvrØ   r×   ÚorigÚoutaÚoutbrÙ   s	      ``   @r   Ú_compute_numerical_gradientrâ   f  sš  øøø€ õ $ EÑ*Ô*ð ð Œ<˜1œ8Ò#Ð#Ý ØfÈÌÐfÐfÐ\]Ô\dÐfÐfñô ð ð �:Š:‰<Œ<˜1Ÿ6š6™8œ8Ò#Ð#Ý ð2ÀeÇjÂjÁlÄlð 2ð 2ÐYZ×Y_ÒY_ÑYaÔYað 2ð 2Ø$)¤Kð2ð 2ñô ð ð
 —’‘”ˆØ�HŠH‰JŒJˆð —’‘”ˆà�;Š;‰=Œ=€DØ	‡K‚K��q‘ÑÔÐØˆ2‰4Œ4€DØ	‡K‚K��q‘ÑÔÐØˆ2‰4Œ4€DØ	‡K‚K�ÑÔÐð(ð (ð (ð (ð (ð (õ
 Ð=Ð=Ð=Ð=­S°°t©_¬_Ð=Ñ=Ô=Ñ=Ô=Ð=r   c                 óø  — g } | t          |t          ¦  «        r|d         n|¦  «        }|rŽt          |t          ¦  «        r | |d         dz  ¦  «        n | |dz  ¦  «        }t          ||¦  «        D ]E\  }}|                     ¦   «         rt	          d¦  «        ‚||dz  z   }	|                     |	¦  «         ŒFn?|D ]<}|s#|                     ¦   «         rt	          d¦  «        ‚|                     |¦  «         Œ=|S )Nr   rP   ù              ð?z-Expected ds_dx to be real-valued, not complexz.Expected ds_dx to be real-valued, not complex.)rG   r9   r¾   r*   rr   Úappend)
Újvp_fnÚdeltaÚinput_is_complexrº   ÚjvpsÚ	ds_dx_tupÚ	ds_dy_tupÚds_dxÚds_dyÚconj_w_ds
             r   Ú*_compute_numerical_jvps_wrt_specific_inputrï   Œ  s6  € ð  "€DØ�¥:¨eµUÑ#;Ô#;ÐF�u˜Q”x�xÀÑGÔG€Iàð å%/°µuÑ%=Ô%=ÐUˆFˆF�5˜”8˜b‘=Ñ!Ô!Ð!À6À6È%ÐRTÉ*ÑCUÔCUð 	õ   	¨9Ñ5Ô5ð 	"ð 	"‰LˆE�5Ø×ÒÑ!Ô!ð VÝ$Ð%TÑUÔUÐUà˜u r™zÑ)ˆHØ�KŠK˜Ñ!Ô!Ð!Ð!ð	"ð ð 	ð 	ˆEØ ð W U×%5Ò%5Ñ%7Ô%7ð WÝ$Ð%UÑVÔVÐVØ�KŠK˜ÑÔÐÐØ€Kr   Újacobians_colsc                 óÄ   — t          |||j        j        r|j        nd ¬¦  «        }t          |¦  «        D ]*\  }}|                      ¦   «         D ]\  }}||         ||<   ŒŒ+|S ©NrO   )rC   r>   r*   r¢   Úitems)	rð   rÁ   r¹   r3   rÄ   r„   ÚjacobianrŽ   rÞ   s	            r   Ú_combine_jacobian_colsrõ   ©  s…   € õ
 1Ø�¨U¬[Ô-CÐM˜eœk˜kÈðñ ô €Iõ ! Ñ+Ô+ð ð ‰ˆˆ8Ø"×(Ò(Ñ*Ô*ð 	ð 	‰DˆAˆqØ˜Aœ$ˆH�Q‰KˆKð	àÐr   r¹   Úmaybe_perturbed_inputc                 óŽ   — | j         t          j        k    r|�|                     ¦   «         S | S t	          | ¦  «        r|r|�|S | S | S r   )r   r   rX   Ú	to_mkldnnr'   )r¹   rö   Ú	fast_modes      r   Ú_prepare_inputrú   ·  sb   € ð
 „|•u”}Ò$Ð$à Ð,Ø(×2Ò2Ñ4Ô4Ð4àˆLÝ	˜uÑ	%Ô	%ð 
Øð 	Ð.Ð:ð )Ð(àˆLð ˆr   c                 óò   — |�d|› d�nd}| j         |j         k    r&t          d|› d|› d| j         › d|j         › d�	¦  «        ‚| j        |j        k    r&t          d	|› d|› d
| j        › d|j        › d�	¦  «        ‚d S )Nz	on index ú Ú zPExpected `func` to return outputs with the same shape when inputs are perturbed zby z, but got: shapes rÓ   ú.zPExpected `func` to return outputs with the same dtype when inputs are perturbed z, but got: dtypes )r[   rr   r>   )Úoutput1Úoutput2rÃ   ÚidxÚon_indexs        r   Ú#_check_outputs_same_dtype_and_shaper  Ï  sî   € ð &) _Ð!˜3Ð!Ð!Ð!Ð!¸"€HØ„}˜œÒ%Ð%Ýð<Ø*2ð<ð <Ø7:ð<ð <à”}ð<ð <à+2¬=ð<ð <ð <ñ
ô 
ð 	
ð
 „}˜œÒ%Ð%Ýð<Ø*2ð<ð <Ø7:ð<ð <à”}ð<ð <à+2¬=ð<ð <ð <ñ
ô 
ð 	
ð &Ð%r   c                 óœ  — i }|€||         n|}|j         st          d¦  «        ‚t          |¦  «        D ]u\  }}	}
t          | |||¦  «        }||	         }t	          j        t          |	|¬¦  «        }t          ||||¦  «        }t          ||| 	                    ¦   «         |¦  «        ||
<   Œvt          ||||                     ¦   «         ¦  «        S )Nz)Expected input to have requires_grad=True)r  rÃ   )r1   rr   r¯   Ú_with_prepare_inputsÚ	functoolsÚpartialr  Ú_get_numerical_jvp_fnrï   r*   rõ   r3   )r¿   Ú	input_idxrÀ   rÁ   rÃ   r¹   rº   Újacobian_colsrD   r  r­   Ú
wrapped_fnÚinput_to_perturbr×   ræ   s                  r   r   r   á  só   € ð 46€MØ!& ˆF�9ÔÐ°E€EØÔð JÝÐHÑIÔIÐIÝ% eÑ,Ô,ð 
ð 
‰ˆˆ3�Ý)¨"¨f°iÀÑCÔCˆ
Ø˜Sœ6ÐÝ"Ô*Ý/°S¸cð
ñ 
ô 
ˆõ 'ØÐ(¨#¨~ñ
ô 
ˆõ  JØ�C˜Ÿš™œ¨ñ 
ô  
ˆ�eÑÐõ " -°¸%ÀÇÂÁÄÑOÔOÐOr   )Úcheck_grad_dtypesÚall_uc          	      óx  ‡— t           j        j        }t          d„ |D ¦   «         ¦  «        }t	          d„ |D ¦   «         ¦  «        rt          d¦  «        ‚|rt          ˆfd„|D ¦   «         ¦  «        }nt          ˆfd„|D ¦   «         ¦  «        }|                     ¦   «         5  g }g }	|D ]¹}
t          |
¦  «        r“|
j        rŒ|
j	        t           j
        k    rt          d¦  «        ‚|                     |
                     ¦   «         t          j        |
¦  «        ¦  «        }
|                     |                     |
¦  «        d         ¦  «         |	                     |
¦  «         Œº|�rot!          t#          ||¦  «        ¦  «        D �]N\  }\  }}|                     |                     |¦  «        ¦  «         t)           | |	Ž ¦  «        }t+          t,          |¦  «        }t!          |¦  «        D ]Ò\  }}|                     |¦  «        \  }}|r;|�9|                     ¦   «         |                     ¦   «         k    rt1          d	¦  «        ‚||         |                              d
¦  «         |€!||         |                              ¦   «          Œž||         |                              |                     d¦  «        ¦  «         ŒÓ|                     ¦   «          �ŒP�nOt!          |¦  «        D �]>\  }}t!          t9          d„ |                     ¦   «         D ¦   «         Ž ¦  «        D �]\  }}d||<   t)           | |	Ž ¦  «        }t+          t,          |¦  «        }t!          |¦  «        D ]½\  }}|                     |¦  «        \  }}|r;|�9|                     ¦   «         |                     ¦   «         k    rt1          d	¦  «        ‚|€'||         |         |                              ¦   «          Œƒ||         |         |                              |                     d¦  «        ¦  «         Œ¾d||<   �Œ�Œ@ddd¦  «         n# 1 swxY w Y   |S )a«  Compute the analytical Jacobian using forward mode AD of `fn(inputs)` using forward mode AD with respect to `target`.

    Return N * M Jacobians where N is the number of tensors in target that require grad and
    M is the number of non-integral outputs.
    Contrary to other functions here, this function requires "inputs" to actually be used by the function.
    The computed value is expected to be wrong if the function captures the inputs by side effect instead of
    using the passed ones (many torch.nn tests do this).

    Args:
        fn: the function to compute the jacobian for
        inputs: inputs to `fn`
        outputs: provide precomputed outputs to avoid one extra invocation of fn
        check_grad_dtypes: if True, will check that the gradient dtype are valid
        all_u (optional): if provided, the Jacobian will be right multiplied with this vector

    Returns:
        A tuple of M N-tuples of tensors
    c              3   óF   K  — | ]}t          |¦  «        ¯|j        ¯|V — Œd S r   r·   ©r5   r„   s     r   r�   z6_get_analytical_jacobian_forward_ad.<locals>.<genexpr>  s6   è è € ÐUÐU ­~¸aÑ/@Ô/@ÐUÀQÄ_ÐU˜!ÐUÐUÐUÐUÐUÐUr   c              3   ó>   K  — | ]}|                      ¦   «         V — Œd S r   r³   r  s     r   r�   z6_get_analytical_jacobian_forward_ad.<locals>.<genexpr>  s*   è è € Ð
1Ð
1˜aˆ1�<Š<‰>Œ>Ð
1Ð
1Ð
1Ð
1Ð
1Ð
1r   zJExpected inputs to be non-complex for _get_analytical_jacobian_forward_ad.c              3   ó8   •K  — | ]}t          ‰d ¦  «        V — ŒdS )rP   N)rC   ©r5   r„   rÁ   s     €r   r�   z6_get_analytical_jacobian_forward_ad.<locals>.<genexpr>  s?   øè è € ð 
ð 
Ø=>Õ,¨W°aÑ8Ô8ð
ð 
ð 
ð 
ð 
ð 
r   c              3   ó\   •K  — | ]&}t          ‰|                     ¦   «         ¦  «        V — Œ'd S r   )rC   r3   r  s     €r   r�   z6_get_analytical_jacobian_forward_ad.<locals>.<genexpr>   sG   øè è € ð 
ð 
ØEFÕ,¨W°a·g²g±i´iÑ@Ô@ð
ð 
ð 
ð 
ð 
ð 
r   ú7MKLDNN inputs are not support for forward AD gradcheck.rP   Nz'Forward AD gradient has dtype mismatch.r   rS   c                 ó,   — g | ]}t          |¦  «        ‘ŒS r   rŠ   r‹   s     r   r8   z7_get_analytical_jacobian_forward_ad.<locals>.<listcomp>O  s   € Ð?Ð?Ð?¨1�e A™hœhÐ?Ð?Ð?r   rÊ   ç        )r   ÚautogradÚ
forward_adr9   r¼   r½   Ú
dual_levelr	   r1   r   rX   Ú	make_dualra   Ú
zeros_likerå   Úunpack_dualr¢   r¾   rÜ   Úview_asr»   Úfilterr+   r*   r   Úsqueeze_Úzero_rš   r   r‚   )r¿   rÀ   rÁ   r  r  ÚfwADÚtensor_inputsrÄ   Úfw_gradsÚdual_inputsrÆ   r„   Úfw_gradÚuÚraw_outputsÚdual_outputsÚindex_oÚd_oÚvalÚresÚlin_idxÚgrad_idxs     `                   r   Ú#_get_analytical_jacobian_forward_adr1  ü  sÔ  ø€ õ, Œ>Ô$€DåÐUÐU VÐUÑUÔUÑUÔU€Må
Ð
1Ð
1 =Ð
1Ñ
1Ô
1Ñ1Ô1ð 
ÝØXñ
ô 
ð 	
ð ð 
Ýð 
ð 
ð 
ð 
ØBOð
ñ 
ô 
ñ 
ô 
ˆ	ˆ	õ ð 
ð 
ð 
ð 
ØJWð
ñ 
ô 
ñ 
ô 
ˆ	ð 
�ŠÑ	Ô	ð ?,ð ?,ØˆØˆØð 	$ð 	$ˆCÝ˜cÑ"Ô"ð 	: sÔ'8ð 	:Ø”:¥¤Ò.Ð.Ý$ØQñô ð ð —n’n S§Z¢Z¡\¤\µ5Ô3CÀCÑ3HÔ3HÑIÔI�ð —’ × 0Ò 0°Ñ 5Ô 5°aÔ 8Ñ9Ô9Ð9Ø×Ò˜sÑ#Ô#Ð#Ð#àñ /	,õ $-­S°¸5Ñ-AÔ-AÑ#BÔ#Bð  ñ  ‘�‘<�G˜QØ—’˜aŸiši¨Ñ0Ô0Ñ1Ô1Ð1Ý'¨¨¨KÐ(8Ñ9Ô9�Ý%Õ&AÀ;ÑOÔO�Ý$-¨lÑ$;Ô$;ð Eð E‘L�G˜SØ#×/Ò/°Ñ4Ô4‘H�C˜à)ðXà˜OØŸNšNÑ,Ô,°·²Ñ0@Ô0@Ò@Ð@å,Ð-VÑWÔWÐWð ˜a”L Ô)×2Ò2°1Ñ5Ô5Ð5Ø�{Ø! !œ WÔ-×3Ò3Ñ5Ô5Ð5Ð5à! !œ WÔ-×3Ò3°C·K²KÀ±O´OÑDÔDÐDÐDØ—’‘”�‘ñ' õ, (¨Ñ1Ô1ð ,ñ ,‘
��7Ý)2ÝÐ?Ð?°·²±´Ð?Ñ?Ô?Ð@ñ*ô *ð ,ñ ,Ñ%�G˜Xð ),�G˜HÑ%Ý"+¨B¨B°Ð,<Ñ"=Ô"=�KÝ#)Õ*EÀ{Ñ#SÔ#S�LÝ(1°,Ñ(?Ô(?ð Rð R™˜ Ø#'×#3Ò#3°CÑ#8Ô#8™˜˜Sà-ðà # Ø #§¢Ñ 0Ô 0°C·N²NÑ4DÔ4DÒ DÐ Då"0Ø Iñ#ô #ð ð ˜;Ø% aœL¨Ô1°'Ô:×@Ò@ÑBÔBÐBÐBà% aœL¨Ô1°'Ô:×@Ò@ÀÇÂÈRÁÄÑQÔQÐQÐQØ(+�G˜HÑ%Ñ%ñ+,ðU?,ð ?,ð ?,ñ ?,ô ?,ð ?,ð ?,ð ?,ð ?,ð ?,ð ?,øøøð ?,ð ?,ð ?,ð ?,ðB Ðs   Â!NP/Ð/P3Ð6P3c                 ó°   — | j         t          j        k    r|                      ¦   «         }n+t	          | ¦  «        r|                      ¦   «         }n| j        }|S r   )r   r   rX   r£   r'   rÛ   r    )r¹   r  s     r   Ú_get_input_to_perturbr3  h  sW   € ð „|•u”}Ò$Ð$à Ÿ>š>Ñ+Ô+ÐÐÝ	˜uÑ	%Ô	%ð &ð !Ÿ;š;™=œ=ÐÐà œ:ÐØÐr   c                 ó"   ‡ ‡‡‡‡— ˆˆ ˆˆˆfd„}|S )Nc                  óÄ   •— t          ˆˆˆfd„t          t          ‰¦  «        ¦  «        D ¦   «         ¦  «        } t          d„ t           ‰| Ž ¦  «        D ¦   «         ¦  «        S )Nc              3   ór   •K  — | ]1\  }}t          |¦  «        rt          ||‰k    r‰nd ‰¦  «        n|V — Œ2d S r   )r	   rú   )r5   r„   r¸   rù   r	  r  s      €€€r   r�   z;_with_prepare_inputs.<locals>.wrapped_fn.<locals>.<genexpr>{  sl   øè è € ð 
ð 
ñ ��1õ ˜aÑ Ô ð�N˜1°!°y².°.Ð.Ð.ÀdÈIÑVÔVÐVàð
ð 
ð 
ð 
ð 
ð 
r   c              3   ó>   K  — | ]}|                      ¦   «         V — Œd S r   )rÛ   )r5   r¸   s     r   r�   z;_with_prepare_inputs.<locals>.wrapped_fn.<locals>.<genexpr>�  s*   è è € Ð<Ð< 1�Q—W’W‘Y”YÐ<Ð<Ð<Ð<Ð<Ð<r   )r9   r¢   r»   )rÆ   rù   r¿   r	  r  rÀ   s    €€€€€r   r  z(_with_prepare_inputs.<locals>.wrapped_fnz  s�   ø€ Ýð 
ð 
ð 
ð 
ð 
ð 
õ "¥)¨FÑ"3Ô"3Ñ4Ô4ð	
ñ 
ô 
ñ 
ô 
ˆõ Ð<Ð<­	°"°"°c°(Ñ(;Ô(;Ð<Ñ<Ô<Ñ<Ô<Ð<r   r   )r¿   rÀ   r	  r  rù   r  s   ````` r   r  r  x  s<   øøøøø€ ð=ð =ð =ð =ð =ð =ð =ð =ð =ð Ðr   c                 ó   ‡ ‡‡‡— ˆˆˆˆ fd„}|S )Nc                 ó*   •— t          ‰‰| ‰‰¦  «        S r   )râ   )rç   rÃ   r  r×   r  s    €€€€r   ræ   z%_get_numerical_jvp_fn.<locals>.jvp_fnˆ  s!   ø€ Ý*ØÐ(¨%°°nñ
ô 
ð 	
r   r   )r  r  rÃ   r×   ræ   s   ```` r   r  r  †  s5   øøøø€ ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð
 €Mr   c                 ó  — t          | t          ¦  «        rLt          | d         ¦  «        s6| d                              |¦  «        | d                              |¦  «        fS n$t          | ¦  «        s|                      |¦  «        S | S ©Nr   rP   )rG   r9   r'   rš   )r(  r[   s     r   Ú_reshape_tensor_or_tupler<  �  s�   € å�!•UÑÔð $Ý$ Q q¤TÑ*Ô*ð 	>Ø�a”D—L’L Ñ'Ô'¨¨1¬¯ª°eÑ)<Ô)<Ð=Ð=ð	>õ % QÑ'Ô'ð 	$Ø—9’9˜UÑ#Ô#Ð#Ø€Hr   c                 ób   — t          | t          ¦  «        r|| d         z  || d         z  fS || z  S r;  )rG   r9   )r(  rŽ   s     r   Ú_mul_tensor_or_tupler>  ›  s8   € Ý�!•UÑÔð Ø�A�a”D‘˜!˜a œd™(Ð#Ð#à�1‰uˆr   c                 óB  — ||         }t          |¦  «        }t          | |||d¦  «        }t          j        t          |¬¦  «        }	t          ||||	¦  «        }
t          ||j        ¦  «        }t          ||¦  «        }t          |
|| 
                    ¦   «         |¦  «        S )NT)rÃ   )r3  r  r  r  r  r  r<  r[   r>  rï   r*   )r¿   r	  rÀ   r(  rÃ   rº   r¹   r  r  r×   ræ   s              r   Ú%_get_numerical_jvp_wrt_specific_inputr@  ¢  s¦   € ð �9Ô€EÝ,¨UÑ3Ô3ÐÝ% b¨&°)Ð=MÈtÑTÔT€JÝÔ&Õ'JÐPSÐTÑTÔT€NÝ" :Ð/?ÀÀnÑUÔU€FÝ  Ð$4Ô$:Ñ;Ô;€AÝ˜Q Ñ$Ô$€AÝ5Ø��5×#Ò#Ñ%Ô% }ñô ð r   c           	      ój  — g }t          ||¦  «        D �]\  }	}
t          | |	||
||¦  «        }g }t          |¦  «        }t          |¦  «        t          |¦  «        k    r/t	          dt          |¦  «        › dt          |¦  «        › �¦  «        ‚t          ||¦  «        D ]*\  }}t          |¦  «        r|                     |¦  «         Œ*Œ+|�Rg }t          ||¦  «        D ](\  }}|                     t          ||¦  «        ¦  «         Œ)|                     |¦  «         �Œ	|                     |¦  «         �Œ |S )Nz>Expected all_Ju and func_out to have the same length, but got rÓ   )r¾   r@  r»   r   rr   r+   rå   Ú_dot_with_type_promotion)r¿   rÀ   rÅ   Úfunc_outr  Úall_vrÃ   rº   Úreduced_jacobiansrÇ   r(  Úall_JuÚfiltered_JuÚJuÚoutputÚjacobian_scalarsrÞ   s                    r   Ú_get_numerical_vJurK  ±  sx  € ð 35ÐÝ˜+ uÑ-Ô-ð 2ñ 2‰
ˆ�Ý6Ø�˜  C¨ñ
ô 
ˆð ˆÝ˜XÑ&Ô&ˆÝˆv‰;Œ;�#˜h™-œ-Ò'Ð'Ý ð=Ý˜v™;œ;ð=ð =Ý-0°©]¬]ð=ð =ñô ð õ ˜f hÑ/Ô/ð 	ð 	‰JˆB�Ý*¨6Ñ2Ô2ð Ø×"Ò" 2Ñ&Ô&Ð&Ð&ð ØÐØ35ÐÝ˜U KÑ0Ô0ð Ið I‘��2Ø ×'Ò'Õ(@ÀÀBÑ(GÔ(GÑHÔHÐHÐHØ×$Ò$Ð%5Ñ6Ô6Ð6Ñ6à×$Ò$ [Ñ1Ô1Ð1Ñ1ØÐr   c                 óÂ   — t          | |¦  «        D ]M\  }}|                     ¦   «         dk    r0||z
                       ¦   «                              ¦   «         |k    r dS ŒNdS )Nr   FT)r¾   r3   Úabsrp   )Új1Új2ÚatolÚj1_xÚj2_xs        r   Ú_check_jacobians_equalrS  Ò  se   € õ ˜"˜b‘k”kð ð ‰
ˆˆdØ�:Š:‰<Œ<˜1ÒÐ $¨¡+×!2Ò!2Ñ!4Ô!4×!8Ò!8Ñ!:Ô!:¸TÒ!AÐ!AØ�5�5øØˆ4r   c           
      ó.  — t          ||¦  «        }t          t          |d¦  «        ¦  «        }d}d}t          | ¦  «        D �]O\  }}||         }	||         }
t          |¦  «        D �](\  }}|�-|                     ¦   «         |	                     ¦   «         k    rd}n|�|j        |	j        k    rd}|€|
d d …|f                              ¦   «          Œj|j        t          j	        k    r| 
                    ¦   «         n|}|
d d …|f                              ¦   «         |                     ¦   «         k    rCt          d|                     ¦   «         › d|
d d …|f                              ¦   «         › �¦  «        ‚|                     d¦  «        |
d d …|f<   �Œ*�ŒQ|||fS )NTFz%Expected out_jacobian column to have z elements, but got rS   )r;   rT   rK   r¢   r‚   r>   r"  r   r   r4   r£   r3   rr   rš   )Úlist_of_list_of_tensorsrÀ   Únumel_outputsÚout_jacobiansÚdiff_input_listÚcorrect_grad_sizesÚcorrect_grad_typesr„   Útensor_listrÆ   Úout_jacobianÚjrf   Údenses                 r   Ú_stack_and_check_tensorsr_  Û  sÉ  € õ
 4°F¸MÑJÔJ€MÝ�=¨°Ñ6Ô6Ñ7Ô7€OØÐØÐÝ#Ð$;Ñ<Ô<ð 7ñ 7‰ˆˆ;Ø˜aÔ ˆØ$ QÔ'ˆÝ" ;Ñ/Ô/ð 	7ñ 	7‰IˆAˆvØÐ! f§k¢k¡m¤m°s·x²x±z´zÒ&AÐ&AØ%*Ð"Ð"ØÐ#¨¬¸¼	Ò(AÐ(AØ%*Ð"Øˆ~Ø˜Q˜Q˜Q ˜TÔ"×(Ò(Ñ*Ô*Ð*Ð*à-3¬]½e¼mÒ-KÐ-K˜ŸšÑ)Ô)Ð)ÐQW�Ø    1 Ô%×+Ò+Ñ-Ô-°·²±´Ò>Ð>Ý(ð@ÀÇÂÁÄð @ð @Ø#/°°°°1°Ô#5×#;Ò#;Ñ#=Ô#=ð@ð @ñô ð ð &+§]¢]°2Ñ%6Ô%6�˜Q˜Q˜Q ˜TÑ"Ñ"ñ	7ð Ð,Ð.@Ð@Ð@r   aù  

NOTE: If your op relies on non-deterministic operations i.e., it is listed here:
https://pytorch.org/docs/stable/generated/torch.use_deterministic_algorithms.html
this failure might be expected.

If you are adding a new operator, please file an issue and then use one of the
workarounds. The workaround depends on how your test invokes gradcheck/gradgradcheck.
If the test
- manually invokes gradcheck/gradgradcheck, then call gradcheck/gradgradcheck
  with `nondet_tol=<tol>` as a keyword argument.
- is OpInfo-based (e.g., in test_ops_gradients.py), then modify the OpInfo for the test
  to have `gradcheck_nondet_tol=<tol>`.
- is a Module test (e.g., in common_nn.py), then modify the corresponding
  module_test entry to have `gradcheck_nondet_tol=<tol>`
c                 ó’  ‡‡— t          t          | d¦  «        ¦  «        Šˆˆfd„}|rGt          |‰                     ¦   «         |¦  «        }t          |‰                     ¦   «         |¦  «        }nDt	          |‰                     ¦   «         ¦  «        }t	          |‰                     ¦   «         ¦  «        }|s‰                     ¦   «         nd}	t          || |	¦  «        \  }
}}t          || |	¦  «        \  }}}t          |
||¦  «        }|s|rt          d¦  «        ‚|st          d¦  «        ‚|st          d|› d�t          z   ¦  «        ‚|
S )NTc                 óL   •— t           j                             ‰‰| dd¬¦  «        S ©NT©Úretain_graphÚallow_unused©r   r  Úgrad©Úgrad_outputrX  rI  s    €€r   Úvjp_fnz5_check_analytical_jacobian_attributes.<locals>.vjp_fn  ó.   ø€ ÝŒ~×"Ò"Ø�O [¸tÐRVð #ñ 
ô 
ð 	
r   rP   zGradient has dtype mismatchz&Analytical gradient has incorrect sizezÖBackward is not reentrant, i.e., running backward with same input and grad_output multiple times gives different values, although analytical gradient matches numerical gradient.The tolerance for nondeterminism was rþ   )
rT   rK   Ú(_get_analytical_vjps_wrt_specific_outputrÛ   Ú!_compute_analytical_jacobian_rowsr3   r_  rS  r   ÚFAILED_NONDET_MSG)rÀ   rI  Ú
nondet_tolr  rù   rÞ   rj  Úvjps1Úvjps2Úoutput_numelÚ
jacobians1Útypes_okÚsizes_okÚ
jacobians2Ú_Ú	reentrantrX  s    `              @r   Ú%_check_analytical_jacobian_attributesry  
  s‹  øø€ õ �=¨°Ñ6Ô6Ñ7Ô7€Oð
ð 
ð 
ð 
ð 
ð 
ð ð JÝ8¸ÀÇÂÁÄÐQRÑSÔSˆÝ8¸ÀÇÂÁÄÐQRÑSÔSˆˆå1°&¸&¿,º,¹.¼.ÑIÔIˆÝ1°&¸&¿,º,¹.¼.ÑIÔIˆà)2Ð9�6—<’<‘>”>�>¸€LÝ%=Øˆv�|ñ&ô &Ñ"€J�˜(õ 0°°v¸|ÑLÔLÑ€J��1Ý& z°:¸zÑJÔJ€Iàð <Ð)ð <ÝÐ:Ñ;Ô;Ð;Øð GÝÐEÑFÔFÐFØð 
ÝðBð 5?ðBð Bð Bõ EVñVñ
ô 
ð 	
ð Ðr   c           
      óª  — g }t          ||¦  «        D �]>\  }}t          | |||d|¬¦  «        }	g }
t          |	|¦  «        D ]û\  }}|j                             d¦  «        }|                     ¦   «         r t          j        |                     ¦   «         ¦  «        }|                     dd¦  «        }|                     dd¦  «        }|
 	                    | 
                    |d         ¦  «        d| 
                    |d         ¦  «        z  z   ¦  «         ŒÓ|
 	                    | 
                    |¦  «        ¦  «         Œü| 	                    |
¦  «         �Œ@|S )NT)rù   rÞ   r   rS   rP   rä   )r¾   ry  ÚTÚsqueezer*   r   Úview_as_realÚresolve_conjÚselectrå   Údot)rÀ   rÁ   ro  r  rD  r  rE  rI  rÞ   Úall_vJrJ  ÚvJr(  ÚtvÚtrÚtis                   r   Ú!_get_analytical_vJu_backward_moder†  6  sM  € ð 35ÐÝ˜ %Ñ(Ô(ð 3ñ 3‰	ˆ�Ý6Ø�F˜JÐ(9ÀTÈQð
ñ 
ô 
ˆð 02ÐÝ˜ Ñ'Ô'ð 
	3ð 
	3‰EˆB�ð ”—’˜a‘”ˆBØ�}Š}‰Œð 3ÝÔ'¨¯ªÑ(9Ô(9Ñ:Ô:�Ø—Y’Y˜r 1Ñ%Ô%�Ø—Y’Y˜r 1Ñ%Ô%�Ø ×'Ò'¨¯ª¨q°¬t©¬°r¸B¿FºFÀ1ÀQÄ4¹L¼LÑ7HÑ(HÑIÔIÐIÐIà ×'Ò'¨¯ª¨q©	¬	Ñ2Ô2Ð2Ð2Ø× Ò Ð!1Ñ2Ô2Ð2Ñ2ØÐr   aB  `get_analytical_jacobian` was part of PyTorch's private API and not meant to be exposed. We are deprecating it and it will be removed in a future version of PyTorch. If you have a specific use for this or feature request for this to be a stable API, please file us an issue at https://github.com/pytorch/pytorch/issues/newr  c                 óî  ‡‡— |dk    rt          d¦  «        ‚‰                     ¦   «         rt          d¦  «        ‚t          t          | d¦  «        ¦  «        Šˆˆfd„}t	          |‰                     ¦   «         ¦  «        }t	          |‰                     ¦   «         ¦  «        }‰                     ¦   «         }t          || |¦  «        \  }}	}
t          || |¦  «        \  }}}t          |||¦  «        }|||
|	fS )NrÊ   zbExpected grad_out to be 1.0. get_analytical_jacobian no longer supports values of grad_out != 1.0.ztExpected output to be non-complex. get_analytical_jacobian no longer supports functions that return complex outputs.Tc                 óL   •— t           j                             ‰‰| dd¬¦  «        S rb  rf  rh  s    €€r   rj  z'get_analytical_jacobian.<locals>.vjp_fng  rk  r   )	r½   r*   rT   rK   rm  rÛ   r3   r_  rS  )rÀ   rI  ro  rÑ   rj  rp  rq  rr  rs  rt  ru  rv  rw  rx  rX  s    `            @r   r   r   N  s  øø€ ð 	�CŠˆåð2ñ
ô 
ð 	
ð ×ÒÑÔð 
ÝðEñ
ô 
ð 	
õ �=¨°Ñ6Ô6Ñ7Ô7€Oð
ð 
ð 
ð 
ð 
ð 
õ .¨f°f·l²l±n´nÑEÔE€EÝ-¨f°f·l²l±n´nÑEÔE€Eà—<’<‘>”>€LÝ%=Øˆv�|ñ&ô &Ñ"€J�˜(õ 0°°v¸|ÑLÔLÑ€J��1Ý& z°:¸zÑJÔJ€Ià�y (¨HÐ4Ð4r   c                 ó^   — t          | ||         t          d¦  «        d¬¦  «        }||         S )NÚinfF)ro  r  )ry  Úfloat)rÀ   rÁ   r	  Ú
output_idxrÄ   s        r   Ú_get_analytical_jacobianr�  z  s:   € õ 6Ø�˜
Ô#µ°e±´ÐPUðñ ô €Ið �YÔÐr   c           	      óâ  — t          j        |t           j        ¬¦  «        }|                     d¦  «        }g }t	          |                     ¦   «         ¦  «        D ]•}|                     ¦   «          d||<    | |¦  «        }t          |¦  «        D ]_\  }}|dk    r|                     g ¦  «         ||xx         t          |t           j
        ¦  «        r|                     ¦   «         nd gz  cc<   Œ`Œ–|S )N©Úmemory_formatrS   rÊ   r   )r   r  Úlegacy_contiguous_formatÚviewr�   r3   r"  r¢   rå   rG   ÚTensorrÛ   )	rj  Úsample_outputÚgrad_out_baseÚflat_grad_outÚjacobians_rowsr]  Úgrad_inputsr„   Úd_xs	            r   rm  rm  ƒ  s  € õ Ô$Ø¥UÔ%Cðñ ô €Mð "×&Ò& rÑ*Ô*€Mà68€NÝ�=×&Ò&Ñ(Ô(Ñ)Ô)ð 	ð 	ˆØ×ÒÑÔÐØˆ�aÑØ�f˜]Ñ+Ô+ˆÝ Ñ,Ô,ð 	ð 	‰FˆAˆsØ�AŠvˆvØ×%Ò% bÑ)Ô)Ð)Ø˜1ÐÐÔÝ)¨#­u¬|Ñ<Ô<ÐF�—	’	‘”�À$ð"ñ ÐÐÑÐð	ð Ðr   c                 ód   —  | |                      |j        ¦  «        ¦  «        }d„ |D ¦   «         }|S )Nc                 óp   — g | ]3}t          |t          j        ¦  «        r|                     ¦   «         nd g‘Œ4S r   )rG   r   r“  rÛ   )r5   Úvjps     r   r8   z<_get_analytical_vjps_wrt_specific_output.<locals>.<listcomp>¢  sE   € ð -ð -ð -ØEH�
 3­¬Ñ5Ô5Ð	?ˆ�Š‰Œˆ¸4Ð@ð-ð -ð -r   )rš   r[   )rj  r”  rÞ   r˜  Úvjpss        r   rl  rl  ž  sG   € ð �&˜Ÿš =Ô#6Ñ7Ô7Ñ8Ô8€Kð-ð -ØLWð-ñ -ô -€Dð €Kr   c           	      óz  — d}t          | ¦  «        D �]\  }}t          |¦  «        �r |j        rù|j        t          j        k    s/|j        t          j        k    st          j        d|› d�d¬¦  «         |j	        r| 
                    ¦   «         }n&t          |¦  «        r|                     ¦   «         }n|}|j        t          j        ur^t          d„ t!          |                     ¦   «         |                     ¦   «         ¦  «        D ¦   «         ¦  «        st'          d|› d�¦  «        ‚d	}�Œ|st)          d
¦  «        ‚d	S )NFzInput #z± requires gradient and is not a double precision floating point or complex. This check will likely fail if all the inputs are not of double precision floating point or complex. rˆ   ©Ú
stacklevelc              3   ó4   K  — | ]\  }}|d k    p|dk    V — ŒdS )r   rP   Nr   )r5   ÚstÚszs      r   r�   z _check_inputs.<locals>.<genexpr>¾  sG   è è € ð ð á˜˜Bð ˜’FÐ%˜b Ašgðð ð ð ð ð r   zThe záth input has a dimension with stride 0. gradcheck only supports inputs that are non-overlapping to be able to compute the numerical gradients correctly. You should call .contiguous on the input before passing it to gradcheck.Tzjgradcheck expects at least one input tensor to require gradient, but none of them have requires_grad=True.)r¢   r	   r1   r>   r   Úfloat64Ú
complex128ÚwarningsÚwarnÚ	is_sparser”   r$   ri   r   rX   Úallr¾   rc   r‚   ÚRuntimeErrorr½   )Útupled_inputsÚany_input_requiring_gradr  rÆ   Úcontents        r   Ú_check_inputsr®  ¨  sŠ  € à$ÐÝ˜mÑ,Ô,ð ,ñ ,‰ˆˆSÝ˜#ÑÔñ 	, 3Ô#4ð 	,Ø”I¥¤Ò.Ð.°#´)½uÔ?OÒ2OÐ2OÝ”ðJ˜cð Jð Jð Jð  !ðñ ô ð ð Œ}ð ØŸ+š+™-œ-��Ý-¨cÑ2Ô2ð ØŸ*š*™,œ,��à�ð Œ~¥U¤]Ð2Ð2Ýð ð å"% g§n¢nÑ&6Ô&6¸¿º¹¼Ñ"GÔ"Gðñ ô ñ ô ð 	õ 'ðS˜sð Sð Sð Sñô ð ð (,Ð$ùà#ð 
Ýð8ñ
ô 
ð 	
ð ˆ4r   c                 ó¦   — t          d„ | D ¦   «         ¦  «        rt          d¦  «        ‚t          d„ | D ¦   «         ¦  «        rt          d¦  «        ‚d S )Nc              3   óh   K  — | ]-}t          |t          j        ¦  «        ¯t          |¦  «        V — Œ.d S r   )rG   r   r“  r'   ©r5   r6   s     r   r�   z!_check_outputs.<locals>.<genexpr>Ó  s<   è è € Ð
TÐ
T¨½
À1ÅeÄlÑ8SÔ8SÐ
TÕ  Ñ#Ô#Ð
TÐ
TÐ
TÐ
TÐ
TÐ
Tr   zySparse output is not supported at gradcheck yet. Please call to_dense(masked_grad=...) on the output of fn for gradcheck.c              3   ót   K  — | ]3}t          |t          j        ¦  «        ¯|j        t          j        k    V — Œ4d S r   )rG   r   r“  r   rX   r±  s     r   r�   z!_check_outputs.<locals>.<genexpr>Ú  s<   è è € Ð
UÐ
U¨½ÀAÅuÄ|Ñ9TÔ9TÐ
Uˆ1Œ8•u”}Ò$Ð
UÐ
UÐ
UÐ
UÐ
UÐ
Ur   zyMKLDNN output is not supported at gradcheck yet. Please call to_dense(masked_grad=...) on the output of fn for gradcheck.)r¼   r½   )rÁ   s    r   Ú_check_outputsr³  Ò  s~   € Ý
Ð
TÐ
T¨WÐ
TÑ
TÔ
TÑTÔTð 
õ ðWñ
ô 
ð 	
õ Ð
UÐ
U¨gÐ
UÑ
UÔ
UÑUÔUð 
ÝðWñ
ô 
ð 	
ð
ð 
r   c                ó¶   — t          | ||||¬¦  «        }|D ]A}|D ]<}t          j        |d¦  «                             ¦   «         dk    rt	          d¦  «        ‚Œ=ŒBdS )N©rÃ   rº   r   ú3Numerical gradient for function expected to be zeroT)rÈ   r   Únerh   r   )ÚfuncrÀ   rC  rÃ   rº   Újacobians_all_inputs_outputsÚ%jacobians_all_outputs_and_fixed_inputrô   s           r   Ú _check_no_differentiable_outputsr»  á  s�   € õ
 $;Øˆf�h C°}ð$ñ $ô $Ð ð 2Nð ð Ð-Ø=ð 	ð 	ˆHÝŒx˜ !Ñ$Ô$×(Ò(Ñ*Ô*¨QÒ.Ð.Ý$ØIñô ð ð /ð	ð
 ˆ4r   c                 ó0  — t          ||¦  «        D ]„\  }}t          | ||||¦  «        }	|	D ]i}
|
                     ¦   «         dk    rŒ|
t          j        |
¦  «        z
                       ¦   «                              ¦   «         |k    rt          d¦  «        ‚ŒjŒ…dS )Nr   r¶  T)r¾   r@  r3   r   r  rM  rp   r   )r¸  rC  Ú
all_inputsÚinputs_indicesr  rÃ   ro  rÇ   r(  ré   Újvps              r   Ú%_check_no_differentiable_outputs_fastrÀ  ò  s·   € õ ˜.¨%Ñ0Ô0ð ð ‰
ˆ�Ý4°T¸7ÀJÐPQÐSVÑWÔWˆØð 	ð 	ˆCØ�yŠy‰{Œ{˜aÒÐØØ•eÔ& sÑ+Ô+Ñ+×0Ò0Ñ2Ô2×6Ò6Ñ8Ô8¸:ÒEÐEÝ$ØIñô ð ð Fð	ð ˆ4r   a�  
gradcheck or gradgradcheck failed while testing batched gradient computation.
This could have been invoked in a number of ways (via a test that calls
gradcheck/gradgradcheck directly or via an autogenerated test).

If you are adding a new operator, please file an issue and then use one of the
workarounds. The workaround depends on how your test invokes gradcheck/gradgradcheck.
If the test
- manually invokes gradcheck/gradgradcheck, then call gradcheck/gradgradcheck
  with `check_batched_grad=False` as a keyword argument.
- is OpInfo-based (e.g., in test_ops_gradients.py), then modify the OpInfo for the test
  to have `check_batched_grad=False` and/or `check_batched_gradgrad=False`.

If you're modifying an existing operator that supports batched grad computation,
or wish to make a new operator work with batched grad computation, please read
the following.

To compute batched grads (e.g., jacobians, hessians), we vmap over the backward
computation. The most common failure case is if there is a 'vmap-incompatible
operation' in the backward pass. Please see
NOTE: [How to write vmap-compatible backward formulas]
in the codebase for an explanation of how to fix this.
a  
gradcheck failed while testing batched gradient computation with forward-mode AD.
This test is enabled automatically when both `check_batched_grad=True`
and `check_forward_ad=True`, but can be disabled in the following ways
dependong on how the test was invoked (via a test that calls gradcheck
directly or via an autogenerated test).

If you are adding a new operator, please file an issue and then use one of the
workarounds. The workaround depends on how your test invokes gradcheck/gradgradcheck.
If the test
- manually invokes gradcheck/gradgradcheck, then call gradcheck/gradgradcheck
  with `check_batched_forward_grad=False` as a keyword argument.
- is OpInfo-based (e.g., in test_ops_gradients.py), then modify the OpInfo for the test
  to have `check_batched_forward_grad=False`
c                 óf   — d| › d|› d|rt           nt          › d|› d|› d�                     ¦   «         S )Nz
For output z and input z:

z

Got:
z

Expected:
ú
)ÚFAILED_BATCHED_GRAD_MSG_FWD_ADÚFAILED_BATCHED_GRAD_MSGÚstrip)rŒ  r	  r.  Úexprº   s        r   Ú!_get_failed_batched_grad_test_msgrÇ  *  so   € ð
Øð
ð 
Ø#,ð
ð 
ð $1ÐMÕÐÕ6Mð
ð 
ð ð
ð 
ð ð
ð 
ð 
÷ 
‚E�G„Gð
r   c                 ó  ‡ ‡‡‡	‡
‡— t           j        j        Š	t          ‰t          ¦  «        st          d¦  «        ‚t          ‰¦  «        D �]4\  Š
Št          ‰¦  «        r‰j        sŒdt           j	        fˆˆ ˆ	ˆ
ˆfd„Št          ‰¦  «        sŒCˆfd„t          d¦  «        D ¦   «         }ˆfd„|D ¦   «         }d„ t          |Ž D ¦   «         }	  t          ‰¦  «        t          j        |¦  «        ¦  «        }n-# t          $ r }t!          d|› d	t"          › �¦  «        |‚d }~ww xY wt          t          ||¦  «        ¦  «        D ]>\  Š
\  }}t          j        ||¦  «        rŒt!          t'          ‰
‰
||d
¬¦  «        ¦  «        ‚�Œ6d
S )NzExpected inputs to be a tupleÚtangentc           	      óR  •‡— ‰
                      ¦   «         5  ‰
                     ‰                     ¦   «         | ¦  «        Št          ˆˆfd„t	          ‰¦  «        D ¦   «         ¦  «        }t           ‰	|Ž ¦  «        }g }|D ]�}|€Œ‰
                     |¦  «        \  }}|�|                     |¦  «         Œ5|                     t          j	        g |j
        |j        ¬¦  «                             |j        ¦  «        ¦  «         Œ‚t          |¦  «        cd d d ¦  «         S # 1 swxY w Y   d S )Nc              3   óx   •K  — | ]4\  }}|‰k    r‰n$t          |¦  «        r|                     ¦   «         n|V — Œ5d S r   )r	   ra   )r5   r  rÆ   Údualr	  s      €€r   r�   z=_test_batched_grad_forward_ad.<locals>.jvp.<locals>.<genexpr>F  sh   øè è € ð )ð )ñ !˜˜Sð ˜iÒ'Ð'ð �Då*8¸Ñ*=Ô*=ÐF˜#Ÿ*š*™,œ,˜,À3ð)ð )ð )ð )ð )ð )r   rN   )r  r  ra   r9   r¢   r»   r  rå   r   r`   r>   r?   Úexpandr[   )rÉ  Úinputs_with_dualr*  rÖ   Údual_outputÚ
primal_outÚtangent_outrÌ  Úcurrent_inputr¸  r#  r	  rÀ   s          @€€€€€r   r¿  z*_test_batched_grad_forward_ad.<locals>.jvpC  sš  øø€ Ø—’Ñ"Ô"ð "ð "Ø—~’~ m×&:Ò&:Ñ&<Ô&<¸gÑFÔF�Ý#(ð )ð )ð )ð )ð )õ %.¨fÑ$5Ô$5ð	)ñ )ô )ñ $ô $Ð õ  )¨¨Ð/?Ð)@ÑAÔA�Ø�Ø#/ð ð �KØ"Ð*Ø Ø.2×.>Ò.>¸{Ñ.KÔ.KÑ+�J Ø"Ð.ØŸ
š
 ;Ñ/Ô/Ð/Ð/àŸ
š
Ý!œKØ "¨*Ô*:À:ÔCTðñ ô ç$šf ZÔ%5Ñ6Ô6ñô ð ð õ
 ˜S‘z”zð-"ð "ð "ð "ñ "ô "ð "ð "ð "ð "ð "ð "øøøð "ð "ð "ð "ð "ð "s   —C8DÄD Ä#D c                 ó8   •— g | ]}t          j        ‰¦  «        ‘ŒS r   ©r   Ú
randn_like)r5   rw  rÒ  s     €r   r8   z1_test_batched_grad_forward_ad.<locals>.<listcomp>_  s$   ø€ ÐFÐFÐF¸•EÔ$ ]Ñ3Ô3ÐFÐFÐFr   rˆ   c                 ó&   •— g | ]} ‰|¦  «        ‘ŒS r   r   )r5   r6   r¿  s     €r   r8   z1_test_batched_grad_forward_ad.<locals>.<listcomp>`  s!   ø€ Ð-Ð-Ð-˜q�C�C˜‘F”FÐ-Ð-Ð-r   c                 ó6   — g | ]}t          j        |¦  «        ‘ŒS r   ©r   rœ   ©r5   Úshardss     r   r8   z1_test_batched_grad_forward_ad.<locals>.<listcomp>a  s"   € ÐEÐEÐE¨F•E”K Ñ'Ô'ÐEÐEÐEr   ú(While computing batched gradients, got: ú

T©rº   )r   r  r  rG   r9   rr   r¢   r	   r1   r“  r+   r�   r¾   r   rœ   rª  r   rÃ  ÚallcloserÇ  )r¸  rÀ   ÚtangentsÚexpectedÚresultÚexr.  rÆ  rÒ  r#  r	  r¿  s   ``      @@@@r   Ú_test_batched_grad_forward_adrã  :  s÷  øøøøøø€ ÝŒ>Ô$€DÝ�f�eÑ$Ô$ð >ÝÐ<Ñ=Ô=Ð=å$-¨fÑ$5Ô$5ð 3ñ 3Ñ ˆ	�=Ý˜}Ñ-Ô-ð 	°-Ô2Mð 	Øð	"�œð 	"ð 	"ð 	"ð 	"ð 	"ð 	"ð 	"ð 	"ð 	"ð 	"õ2 +¨=Ñ9Ô9ð 	ØàFÐFÐFÐF½UÀ1¹X¼XÐFÑFÔFˆØ-Ð-Ð-Ð- HÐ-Ñ-Ô-ˆØEÐEµc¸8°nÐEÑEÔEˆð	Ø•U˜3‘Z”Z¥¤¨HÑ 5Ô 5Ñ6Ô6ˆFˆFøÝð 	ð 	ð 	å Øc¸2ÐcÐcÕCaÐcÐcñô àðøøøøð	øøøõ &/­s°6¸8Ñ/DÔ/DÑ%EÔ%Eð 	ð 	Ñ!ˆI‘z˜˜SÝŒ~˜c 3Ñ'Ô'ð ØÝ Ý1Ø˜y¨#¨sÀ$ðñ ô ñô ð ñ	ð ˆ4s   Ã*C6Ã6
D Ä DÄD c           	      ó2  ‡‡
‡‡— t          t          | d¦  «        ¦  «        Š
t          j        t          j        j        ‰‰
dd¬¦  «        Šˆ
ˆfd„Šˆfd„t          d¦  «        D ¦   «         }ˆfd„|D ¦   «         }d„ t          |Ž D ¦   «         }t          j
        ¦   «         5  t          j        dd	¬
¦  «         t          j        dd¬
¦  «         	  t          ‰¦  «        t	          j        |¦  «        ¦  «        }n-# t          $ r }t          d|› dt           › �¦  «        |‚d }~ww xY w	 d d d ¦  «         n# 1 swxY w Y   t#          t          ||¦  «        ¦  «        D ]<\  }\  }}	t	          j        ||	¦  «        rŒt          t'          ||||	¦  «        ¦  «        ‚dS )NTrc  c                 ól   •—  ‰| ¦  «        }t          d„ t          |‰¦  «        D ¦   «         ¦  «        }|S )Nc              3   ó–   K  — | ]D\  }}|�|n8t          j        g |j        |j        ¬¦  «                             |j        ¦  «        V — ŒEd S )NrN   )r   r`   r>   r?   rÍ  r[   )r5   rg  rÆ   s      r   r�   z2_test_batched_grad.<locals>.vjp.<locals>.<genexpr>‡  sn   è è € ð 
ð 
ñ ��cð Ðð ˆDå”˜R s¤y¸¼ÐDÑDÔD×KÒKÈCÌIÑVÔVð
ð 
ð 
ð 
ð 
ð 
r   )r9   r¾   )rÞ   ÚresultsrX  rg  s     €€r   rœ  z_test_batched_grad.<locals>.vjp…  sN   ø€ Ø�$�q‘'”'ˆÝð 
ð 
õ ! ¨/Ñ:Ô:ð	
ñ 
ô 
ñ 
ô 
ˆð ˆr   c                 ó8   •— g | ]}t          j        ‰¦  «        ‘ŒS r   rÔ  )r5   rw  rI  s     €r   r8   z&_test_batched_grad.<locals>.<listcomp>�  s$   ø€ Ð?Ð?Ð?°•EÔ$ VÑ,Ô,Ð?Ð?Ð?r   rˆ   c                 ó&   •— g | ]} ‰|¦  «        ‘ŒS r   r   )r5   ÚgOrœ  s     €r   r8   z&_test_batched_grad.<locals>.<listcomp>‘  s!   ø€ Ð/Ð/Ð/˜B���B‘”Ð/Ð/Ð/r   c                 ó6   — g | ]}t          j        |¦  «        ‘ŒS r   rØ  rÙ  s     r   r8   z&_test_batched_grad.<locals>.<listcomp>’  s"   € ÐAÐAÐA¨•”˜FÑ#Ô#ÐAÐAÐAr   ÚignorezThere is a performance drop)ÚmessagezPlease use `torch.vmap`rÛ  rÜ  )rT   rK   r  r  r   r  rg  r�   r¾   r¦  Úcatch_warningsÚfilterwarningsr   rœ   rª  r   rÄ  r¢   rÞ  rÇ  )r¹   rI  rŒ  Úgrad_outputsrà  rá  râ  r	  r.  rÆ  rX  rg  rœ  s    `        @@@r   Ú_test_batched_gradrñ  v  s0  øøøø€ õ �=¨°Ñ5Ô5Ñ6Ô6€OÝÔÝŒÔØØØØðñ ô €Dðð ð ð ð ð ð @Ð?Ð?Ð?µe¸A±h´hÐ?Ñ?Ô?€Là/Ð/Ð/Ð/ ,Ð/Ñ/Ô/€HØAÐAµ#°x°.ÐAÑAÔA€Hõ 
Ô	 Ñ	"Ô	"ð ð ÝÔ Ð2OÐPÑPÔPÐPÝÔ Ð2KÐLÑLÔLÐLð		Ø•T˜#‘Y”Y�uœ{¨<Ñ8Ô8Ñ9Ô9ˆFˆFøÝð 	ð 	ð 	õ
 !Ø\¸2Ð\Ð\ÕCZÐ\Ð\ñô àðøøøøð	øøøð ð	ð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð õ "+­3¨v°xÑ+@Ô+@Ñ!AÔ!Að 
ð 
Ñˆ	‘:�C˜ÝŒ>˜#˜sÑ#Ô#ð 	ØÝÝ-¨j¸)ÀSÈ#ÑNÔNñ
ô 
ð 	
ð ˆ4s6   Â-D2Ã*C8Ã7D2Ã8
D"ÄDÄD"Ä"D2Ä2D6Ä9D6c           
      ó>  — t          t          |d¦  «        ¦  «        }|st          d¦  «        ‚t          j                             | |d„ | D ¦   «         d¬¦  «        }t          ||¦  «        D �]¯\  }}|€Œ	t          |t          j        ¦  «        �r¥|j	        t          j
        k    �r�|j	        |j	        k    r?t          dt          |j	        ¦  «        z   dz   t          |j	        ¦  «        z   dz   ¦  «        ‚t          |¦  «        �rt          |j	        ¦  «                             dd	¦  «                             d
d	¦  «        }|                     ¦   «         |                     ¦   «         k    r<t          d|› d|                     ¦   «         › d|                     ¦   «         › �¦  «        ‚|                     ¦   «         |                     ¦   «         k    r<t          d|› d|                     ¦   «         › d|                     ¦   «         › �¦  «        ‚|                     ¦   «         }|                     ¦   «         }|r7t          j        |t          j        |¦  «        ¦  «        st          d¦  «        ‚n6|                     d¦  «                             ¦   «         st          d¦  «        ‚|j        |j        k    rt          d¦  «        ‚|j        |j        k    rt          d¦  «        ‚|                     ¦   «         |                     ¦   «         k    rt          d¦  «        ‚�Œ±dS )NTú(no Tensors requiring grad found in inputc                 óN   — g | ]"}t          j        |t           j        ¬ ¦  «        ‘Œ#S ©r�  ©r   r  r‘  r´   s     r   r8   z5_test_backward_mul_by_grad_output.<locals>.<listcomp>µ  s;   € ð 	
ð 	
ð 	
àõ Ô˜Q­eÔ.LÐMÑMÔMð	
ð 	
ð 	
r   ©re  zgrad is incorrect layout (z is not ú)ztorch.rý   Ú_coozgrad is z& tensor, but has incorrect sparse_dim z, expected z% tensor, but has incorrect dense_dim z&backward not multiplied by grad_outputr   zgrad is incorrect typezgrad is incorrect devicezgrad is incorrect size)rT   rK   r   r   r  rg  r¾   rG   r“  r   r4   rJ   r'   Úreplacer\   Ú	dense_dimr£   rÞ  r  Úeqr©  r>   r?   r‚   )rÁ   rÀ   ÚmaskedrX  Úgrads_inputÚgiÚdiÚsparse_kinds           r   Ú!_test_backward_mul_by_grad_outputr  ­  s)  € å*.­}¸VÀTÑ/JÔ/JÑ*KÔ*K€OØð IÝÐGÑHÔHÐHÝ”.×%Ò%ØØð	
ð 	
àð	
ñ 	
ô 	
ð ð &ñ ô €Kõ �k ?Ñ3Ô3ð $;ñ $;‰ˆˆBØˆ:ØÝ�b�%œ,Ñ'Ô'ñ 	¨B¬I½¼Ò,FÑ,FØŒy˜BœIÒ%Ð%Ý$Ø0Ý˜"œ)‘n”nñ%à ñ!õ ˜"œ)‘n”nñ%ð ñ	ñô ð õ % RÑ(Ô(ñ Ý! "¤)™nœn×4Ò4°X¸rÑBÔB×JÒJÈ6ÐSUÑVÔV�Ø—=’=‘?”? b§m¢m¡o¤oÒ5Ð5Ý(ðJ ;ð Jð JØŸMšM™OœOðJð JØ8:¿º¹¼ðJð Jñô ð ð —<’<‘>”> R§\¢\¡^¤^Ò3Ð3Ý(ðH ;ð Hð HØŸLšL™NœNðHð HØ79·|²|±~´~ðHð Hñô ð ð —’‘”ˆBØ—’‘”ˆBØð 	KÝ”> "¥eÔ&6°rÑ&:Ô&:Ñ;Ô;ð OÝ$Ð%MÑNÔNÐNðOà—’�q‘”—’‘”ð 	KÝ Ð!IÑJÔJÐJØŒ8�r”xÒÐÝ Ð!9Ñ:Ô:Ð:ØŒ9˜œ	Ò!Ð!Ý Ð!;Ñ<Ô<Ð<Ø�7Š7‰9Œ9˜Ÿš™	œ	Ò!Ð!Ý Ð!9Ñ:Ô:Ð:ñ "àˆ4r   c                 óÖ  — t           j        j        }t          |¦  «        \  }}t	          ||d¬¦  «        \  }}}|                     ¦   «         5  g }	g }
t          ¦   «         }t          |¦  «        D ]Ñ\  }}t          |¦  «        r¨|j	        r¡|j
        t           j        k    rt          d¦  «        ‚|                     |                     ¦   «         t          j        |¦  «        ¦  «        }|	                     |                     |¦  «        d         ¦  «         |                     |¦  «         |
                     |¦  «         ŒÒt'          |	|¦  «        D ]-\  }}|                     |                     |¦  «        ¦  «         Œ.t          |¦  «        D �]@\  }}||vrŒ|
|         }|                     |                     ¦   «         t          j        |¦  «        ¦  «        |
|<   t-           | |
Ž ¦  «        }t/          t0          |¦  «        }|                     ¦   «         |
|<   t-           | |
Ž ¦  «        }t/          t0          |¦  «        }||
|<   t          t'          ||¦  «        ¦  «        D ]h\  }\  }}|                     |¦  «        \  }}|                     |¦  «        \  }}|�.|�,t          j        ||¦  «        st5          d|d|dd|d	|¦	  «	        ‚Œi�ŒB	 d d d ¦  «         n# 1 swxY w Y   dS )
NT©Úuse_forward_adr  rP   z2Mismatch in tangent values for output with index: z when input: z! has an undefined tangent value. z Got: z but expected: )r   r  r  Ú_get_inp_tensorsÚ_make_vectorsr  Úsetr¢   r	   r1   r   rX   r½   r  ra   r  rå   r  Úaddr¾   rÜ   r  r»   r   r+   rÞ  r   )r¸  rÁ   rÀ   r#  Ú_inp_tensors_idxÚinp_tensorsÚ_all_vr  Ú_all_u_denser%  r&  Útensor_indicesr„   rÆ   r'  r(  r  Údual_inp_objr)  Údual_outputs1Údual_outputs2r+  Úd_o1Úd_o2Ú_val1Úres1Ú_val2Úres2s                               r   Ú_test_undefined_forward_moder  ã  sI  € ÝŒ>Ô$€Då$4°VÑ$<Ô$<Ñ!Ð�kÝ"/Ø�W¨Tð#ñ #ô #Ñ€FˆE�<ð 
�ŠÑ	Ô	ð 7ð 7ØˆØˆÝ™œˆÝ Ñ'Ô'ð 	$ð 	$‰FˆAˆsÝ˜cÑ"Ô"ð 
& sÔ'8ð 
&Ø”:¥¤Ò.Ð.Ý$ØQñô ð ð —n’n S§Z¢Z¡\¤\µ5Ô3CÀCÑ3HÔ3HÑIÔI�ð —’ × 0Ò 0°Ñ 5Ô 5°aÔ 8Ñ9Ô9Ð9Ø×"Ò" 1Ñ%Ô%Ð%Ø×Ò˜sÑ#Ô#Ð#Ð#å˜h¨Ñ.Ô.ð 	.ð 	.‰JˆG�QØ�MŠM˜!Ÿ)š) GÑ,Ô,Ñ-Ô-Ð-Ð-å! &Ñ)Ô)ð "	ñ "	‰HˆC�Ø˜.Ð(Ð(ØØ& sÔ+ˆLð  $Ÿ~š~¨c¯jªj©l¬l½EÔ<LÈSÑ<QÔ<QÑRÔRˆK˜ÑÝ# D D¨+Ð$6Ñ7Ô7ˆKÝ"Õ#>ÀÑLÔLˆMð  #Ÿzšz™|œ|ˆK˜ÑÝ# D D¨+Ð$6Ñ7Ô7ˆKÝ"Õ#>ÀÑLÔLˆMð  ,ˆK˜Ñå)2µ3°}ÀmÑ3TÔ3TÑ)UÔ)Uð ð Ñ%�™˜$ Ø"×.Ò.¨tÑ4Ô4‘��tØ"×.Ò.¨tÑ4Ô4‘��tà˜¨¨Ý œ>¨$°Ñ5Ô5ð Ý,ØPØ#Ø+ØØ?Ø$Ø Ø-Ø ñ
ô 
ð 
øñð%"	ð+7ð 7ð 7ñ 7ô 7ð 7ð 7ð 7ð 7ð 7ð 7øøøð 7ð 7ð 7ð 7ðp ˆ4s   ÁJKËK"Ë%K"c                 óØ  ‡‡‡‡— t          t          |d¦  «        ¦  «        Š‰st          d¦  «        ‚d„ Šˆˆfd„Šd„ t           | |Ž ¦  «        D ¦   «         g}t	          |d         ¦  «        dk    r_t          t	          |¦  «        ¦  «        D ]BŠt           | |Ž ¦  «        }|                     ˆfd„t          |¦  «        D ¦   «         ¦  «         ŒCt          ˆfd	„|D ¦   «         ¦  «        S )
NTró  c                  ó2   — t          j        dd¬¦  «         d S )Na   Backwards compatibility: New undefined gradient support checking feature is enabled by default, but it may break existing callers of this function. If this is true for you, you can call this function with "check_undefined_grad=False" to disable the featurerˆ   rŸ  )r¦  r§  r   r   r   Úwarn_bc_breakingz7_test_undefined_backward_mode.<locals>.warn_bc_breaking+  s0   € ÝŒðPð ð	
ñ 	
ô 	
ð 	
ð 	
ð 	
r   c                 óP  •— d„ | D ¦   «         }	 t           j                             | ‰|d¬¦  «        }n,# t          $ r} ‰¦   «          t	          d¦  «        |‚d }~ww xY w|D ]D}|�@|                     d¦  «                             ¦   «         s ‰¦   «          t	          d¦  «        ‚ŒEdS )Nc                 óN   — g | ]"}t          j        |t           j        ¬ ¦  «        ‘Œ#S rõ  rö  r´   s     r   r8   zW_test_undefined_backward_mode.<locals>.check_undefined_grad_support.<locals>.<listcomp>5  s;   € ð 
ð 
ð 
àõ Ô˜Q­eÔ.LÐMÑMÔMð
ð 
ð 
r   Tr÷  z™Expected backward function to handle undefined output grads. Please look at "Notes about undefined output gradients" in "tools/autograd/derivatives.yaml"r   zºExpected all input grads to be undefined or zero when all output grads are undefined or zero. Please look at "Notes about undefined output gradients" in "tools/autograd/derivatives.yaml")r   r  rg  rª  r   rü  r©  )Úoutput_to_checkÚgrads_outputrþ  Úerÿ  rX  r  s        €€r   Úcheck_undefined_grad_supportzC_test_undefined_backward_mode.<locals>.check_undefined_grad_support4  só   ø€ ð
ð 
à$ð
ñ 
ô 
ˆð
	Ýœ.×-Ò-Ø °,ÈTð .ñ ô ˆKˆKøõ ð 	ð 	ð 	ØÐÑÔÐÝ ð4ñô ð ð	øøøøð	øøøð ð 	ð 	ˆBØ�¨¯ª¨q©¬¯ª©¬�Ø Ð Ñ"Ô"Ð"Ý$ð8ñô ð øð
 ˆts   �#3 ³
A½AÁAc                 ó˜   — g | ]G}t          |t          j        ¦  «        ¯t          j        j                             ¦   «         |¦  «        ‘ŒHS r   )rG   r   r“  Ú_CÚ
_functionsÚUndefinedGradr´   s     r   r8   z1_test_undefined_backward_mode.<locals>.<listcomp>Q  sS   € ð 	
ð 	
ð 	
àå˜!�Uœ\Ñ*Ô*ð		
ÝŒHÔ×-Ò-Ñ/Ô/°Ñ2Ô2ð	
ð 	
ð 	
r   r   rP   c                 ó|   •— g | ]8\  }}|‰k    r+t          j        j                             ¦   «         |¦  «        n|‘Œ9S r   )r   r#  r$  r%  )r5   r  rµ   Úundef_grad_idxs      €r   r8   z1_test_undefined_backward_mode.<locals>.<listcomp>^  s[   ø€ ð ð ð ñ ˜˜Qð ˜nÒ,Ð,õ ”HÔ'×5Ò5Ñ7Ô7¸Ñ:Ô:Ð:àðð ð r   c              3   ó.   •K  — | ]} ‰|¦  «        V — Œd S r   r   )r5   rI  r!  s     €r   r�   z0_test_undefined_backward_mode.<locals>.<genexpr>f  s/   øè è € ÐSÐS¸Ð+Ð+¨FÑ3Ô3ÐSÐSÐSÐSÐSÐSr   )	rT   rK   r   Ú_differentiable_outputsr   r�   rå   r¢   r©  )	r¸  rÁ   rÀ   Úoutputs_to_checkr  r!  rX  r'  r  s	        @@@@r   Ú_test_undefined_backward_moder+  &  sR  øøøø€ Ý*.­}¸VÀTÑ/JÔ/JÑ*KÔ*K€OØð IÝÐGÑHÔHÐHð
ð 
ð 
ðð ð ð ð ð ð:	
ð 	
å,¨T¨T°6¨]Ñ;Ô;ð	
ñ 	
ô 	
ðÐõ Ð˜AÔÑÔ !Ò#Ð#Ý#¥C¨¡L¤LÑ1Ô1ð 		ð 		ˆNÝ5°d°d¸F°mÑDÔDˆOØ×#Ò#ðð ð ð õ #,¨OÑ"<Ô"<ð	ñ ô ñô ð ð õ ÐSÐSÐSÐSÐBRÐSÑSÔSÑSÔSÐSr   c                 ó~   — t          | t          ¦  «        r| S t          | t          ¦  «        rt          | ¦  «        S | fS r   )rG   r9   rT   ©rD   s    r   r»   r»   i  s=   € Ý�!•UÑÔð ØˆÝ	�A•tÑ	Ô	ð Ý�Q‰xŒxˆàˆtˆr   c                 óN   — t          d„ t          | ¦  «        D ¦   «         ¦  «        S )Nc              3   ó(   K  — | ]}|j         ¯	|V — Œd S r   ©r1   r´   s     r   r�   z*_differentiable_outputs.<locals>.<genexpr>s  s)   è è € Ð<Ð<�q¨A¬OÐ<�Ð<Ð<Ð<Ð<Ð<Ð<r   )r9   r»   r-  s    r   r)  r)  r  s%   € ÝÐ<Ð<�I a™LœLÐ<Ñ<Ô<Ñ<Ô<Ð<r   c                 ó”   — | o|o||v }|o|o||v }|rdnd}	|rdnd}
|s|sdnd|	› d|
› d�}|rd	nd}|d
|› d|d›d|d›d|› d| › d�z   S )NÚ	imaginaryÚrealrÀ   rÁ   rý   zWhile considering the z part of complex z only, zcomputed with forward mode z	Jacobian zmismatch for output Údz with respect to input z,
numerical:z
analytical:rÂ  r   )Ú
analyticalÚ	numericalrŒ  r	  Úcomplex_indicesÚ	test_imagrº   Úout_is_complexÚinp_is_complexÚpartÚelementÚprefixÚmodes                r   Ú_get_notallclose_msgr?  v  sñ   € ð Ð	ÐQ ÐQ°JÀ/Ð4Qð ð #ÐW ÐW¸9ÈÐ;W€NØ#Ð/ˆ;ˆ;¨€DØ'Ð6ˆhˆh¨Y€Gð ð	NØ"0ð	NˆˆàM dÐMÐM¸WÐMÐMÐMð ð
 -:ÐAÐ(Ð(¸r€Dàð<�dð <ð <°
Ð
eð <ð <ÐU^Ð
eð <ð <Øð<ð <Ø-7ð<ð <ð <ñ	<ðr   c                 ó.   — t          t          | Ž ¦  «        S r   )rT   r¾   )Úmatrix_of_tensorss    r   Ú
_transposerB  ’  s   € å•Ð&Ð'Ñ(Ô(Ð(r   c                 ó`   — d„ } || t           j        ¦  «         || t           j        ¦  «        fS )Nc                 ó   ‡ ‡— ˆ ˆfd„}|S )Nc                  ó^   •— t           ‰| Ž ¦  «        }t          ˆfd„|D ¦   «         ¦  «        S )Nc              3   óZ   •K  — | ]%}|                      ¦   «         r ‰|¦  «        n|V — Œ&d S r   r³   )r5   rµ   Úfn_to_applys     €r   r�   zU_real_and_imag_output.<locals>.apply_to_c_outs.<locals>.wrapped_fn.<locals>.<genexpr>�  s<   øè è € ÐOÐOÀQ¨1¯<ª<©>¬>Ð@˜˜ Q™œ˜¸qÐOÐOÐOÐOÐOÐOr   )r»   r9   )rÀ   Úoutsr¿   rG  s     €€r   r  zB_real_and_imag_output.<locals>.apply_to_c_outs.<locals>.wrapped_fn›  s9   ø€ Ý˜R˜R ˜[Ñ)Ô)ˆDÝÐOÐOÐOÐOÈ$ÐOÑOÔOÑOÔOÐOr   r   )r¿   rG  r  s   `` r   Úapply_to_c_outsz._real_and_imag_output.<locals>.apply_to_c_outsš  s0   øø€ ð	Pð 	Pð 	Pð 	Pð 	Pð 	Pð Ðr   )r   r3  Úimag)r¿   rI  s     r   Ú_real_and_imag_outputrK  —  s:   € ðð ð ð ˆ?˜2�uœzÑ*Ô*¨O¨O¸BÅÄ
Ñ,KÔ,KÐKÐKr   c                 óN   ‡‡— ˆˆfd„} || d„ ¦  «        } || d„ ¦  «        }||fS )Nc                 ó   •‡ ‡— ˆˆ ˆˆfd„}|S )Nc                  ó†   •— t          | ¦  «        }‰D ]} ‰||         ‰|         ¦  «        ||<   Œt           ‰|Ž ¦  «        S r   ©rT   r»   )rÀ   Ú
new_inputsÚshould_be_complexÚcomplex_inp_indicesr¿   rG  r«  s      €€€€r   r  zA_real_and_imag_input.<locals>.apply_to_c_inps.<locals>.wrapped_fnª  s_   ø€ Ý˜f™œˆJØ%8ð ð Ð!Ø0;°ØÐ0Ô1°=ÐARÔ3Sñ1ô 1�
Ð,Ñ-Ð-õ ˜R˜R ˜_Ñ-Ô-Ð-r   r   )r¿   rG  r  rR  r«  s   `` €€r   Úapply_to_c_inpsz-_real_and_imag_input.<locals>.apply_to_c_inps©  s5   øøø€ ð	.ð 	.ð 	.ð 	.ð 	.ð 	.ð 	.ð 	.ð Ðr   c                 ó   — | |j         dz  z   S ©Nrä   )rJ  ©rÆ   rß   s     r   ú<lambda>z&_real_and_imag_input.<locals>.<lambda>´  s   € °C¸$¼)Àb¹.Ñ4H€ r   c                 ó   — |j         | dz  z   S rU  )r3  rV  s     r   rW  z&_real_and_imag_input.<locals>.<lambda>µ  s   € °D´IÀÀbÁÑ4H€ r   r   )r¿   rR  r«  rS  Úreal_fnÚimag_fns    ``   r   Ú_real_and_imag_inputr[  ¤  sY   øø€ ð
	ð 	ð 	ð 	ð 	ð 	ð ˆo˜bÐ"HÐ"HÑIÔI€GØˆo˜bÐ"HÐ"HÑIÔI€GØ�GÐÐr   c                 ób  — d„ t          |¦  «        D ¦   «         }t          d„ t          |¦  «        D ¦   «         ¦  «        }|
r{|rft          |¦  «        \  }} ||Ž }t	          |¦  «        } | ||||||||||d¬¦  «          ||Ž }t	          |¦  «        } | ||||||||||¬¦
  «
         n | |||||||||¦	  «	         |	rîd„ t          |¦  «        D ¦   «         }|r©t          |||¦  «        \  }}d„ |D ¦   «         } ||Ž }t	          |¦  «        } | ||||||||||dd¬¦  «         d	„ |D ¦   «         } ||Ž }t	          |¦  «        } | ||||||||||d¬
¦  «         |r$t          |||¦  «         t          |||¦  «         d S d S  | |||||||||d¬¦
  «
         |rt          |||¦  «         d S d S d S )Nc                 ó@   — g | ]\  }}|                      ¦   «         ¯|‘ŒS r   r³   )r5   r„   rµ   s      r   r8   z(_gradcheck_real_imag.<locals>.<listcomp>È  s)   € ÐNÐNÐN¡  A¸q¿|º|¹~¼~ÐN˜1ÐNÐNÐNr   c              3   ó>   K  — | ]}|                      ¦   «         V — Œd S r   r³   r´   s     r   r�   z'_gradcheck_real_imag.<locals>.<genexpr>É  s*   è è € Ð MÐ M°A §¢¡¤Ð MÐ MÐ MÐ MÐ MÐ Mr   T)r7  r8  )r7  c                 ó^   — g | ]*\  }}t          |¦  «        ¯|                     ¦   «         ¯(|‘Œ+S r   )r	   r*   )r5   r„   rÆ   s      r   r8   z(_gradcheck_real_imag.<locals>.<listcomp>ú  sN   € ð 
ð 
ð 
á��3Ý˜cÑ"Ô"ð
ð (+§~¢~Ñ'7Ô'7ð
Øð
ð 
ð 
r   c                 óf   — g | ].}t          |¦  «        r|                     ¦   «         r|j        n|‘Œ/S r   )r	   r*   rJ  ©r5   rÆ   s     r   r8   z(_gradcheck_real_imag.<locals>.<listcomp>  óJ   € ð ð ð àõ +¨3Ñ/Ô/ÐM°C·N²NÑ4DÔ4DÐM�”�È#ðð ð r   )r7  r8  r  c                 óf   — g | ].}t          |¦  «        r|                     ¦   «         r|j        n|‘Œ/S r   )r	   r*   r3  ra  s     r   r8   z(_gradcheck_real_imag.<locals>.<listcomp>  rb  r   )r7  r  r  )r¢   r¼   r»   rK  r)  r[  r  )Úgradcheck_fnr¸  rC  r«  rÁ   rÃ   ÚrtolrP  r  Úcheck_forward_adÚcheck_backward_adro  Úcheck_undefined_gradÚcomplex_out_indicesÚhas_any_complex_outputrY  rZ  Úimag_func_outÚimag_outputsÚreal_func_outÚreal_outputsrR  Úimag_inputsÚdiff_imag_func_outÚreal_inputsÚdiff_real_func_outs                             r   Ú_gradcheck_real_imagrs  ¹  sA  € ð OÐN­°7Ñ);Ô);ÐNÑNÔNÐÝ Ð MÐ M½À8Ñ9LÔ9LÐ MÑ MÔ MÑMÔMÐØð -Ø!ð ,	Ý4°TÑ:Ô:ÑˆG�Wà#˜G ]Ð3ˆMÝ2°=ÑAÔAˆLØˆLØØØØØØØØ!ØØ 3Øðñ ô ð ð $˜G ]Ð3ˆMÝ2°=ÑAÔAˆLØˆLØØØØØØØØ!ØØ 3ðñ ô ð ð ð ˆLØØØØØØØØ!Øñ
ô 
ð 
ð ð DKð
ð 
å# MÑ2Ô2ð
ñ 
ô 
Ðð
 ð >	KÝ3ØÐ)¨=ñ ô  ÑˆG�Wðð à(ðñ ô ˆKð $˜G [Ð1ˆMÝ!8¸Ñ!GÔ!GÐØˆLØØØØ"ØØØØ!ØØ 3ØØ#ðñ ô ð ðð à(ðñ ô ˆKð $˜G [Ð1ˆMÝ!8¸Ñ!GÔ!GÐØˆLØØØØ"ØØØØ!ØØ 3Ø#ðñ ô ð ð $ð RÝ,¨W°mÀ[ÑQÔQÐQÝ,¨W°mÀ[ÑQÔQÐQÐQÐQðRð Rð ˆLØØØØØØØØ!ØØ#ðñ ô ð ð $ð KÝ,¨T°7¸MÑJÔJÐJÐJÐJðIDKð DKðFKð Kr   )r  r7  r8  rý  c	                ó^  — t          |¦  «        }|st          | ||||	¬¦  «        S |r|nt          |¦  «        }t          t	          | ||||	¬¦  «        ¦  «        }d„ t          ||¦  «        D ¦   «         }|	r™t          | |||¬¦  «        }t          |¦  «        D ]u\  }}t          |¦  «        D ]`\  }}||         |         }t          || 	                    |j
        ¦  «        ||¦  «        s#t          t          |||||
|d¬¦  «        ¦  «        ‚ŒaŒvnžt          |¦  «        D ]Ž\  }}t          ||||¦  «        }t          t          |||         ¦  «        ¦  «        D ]S\  }\  }}t          || 	                    |j
        ¦  «        ||¦  «        s!t          t          |||||
|¦  «        ¦  «        ‚ŒTŒ�dS )Nrµ  c                 ó&   — g | ]\  }}|j         ¯|‘ŒS r   r0  )r5   rµ   Únjs      r   r8   z#_slow_gradcheck.<locals>.<listcomp>c  s#   € ÐMÐMÐM™˜˜2¸Q¼_ÐM�ÐMÐMÐMr   )r  TrÝ  )r»   r»  rW   rB  rÈ   r¾   r1  r¢   Ú_allclose_with_type_promotionr_   r?   r   r?  ry  )r¸  rC  r«  rÁ   rÃ   re  rP  r  ro  r  r7  r8  rý  Útupled_inputs_numericalr6  Úanalytical_forwardr„   Ú	n_per_outr]  Únr¸   rµ   r5  s                          r   Ú_slow_gradcheckr|  @  s6  € õ  ˜Ñ"Ô"€HØð 
Ý/Ø�- ¨sÀ.ð
ñ 
ô 
ð 	
ð 06ÐR˜m˜m½8ÀMÑ;RÔ;RÐåÝØØ#ØØØ(ð	
ñ 	
ô 	
ñô €Ið NÐM¥ X¨yÑ!9Ô!9ÐMÑMÔM€IØð Ý@Ø�- Ð=Nð
ñ 
ô 
Ðõ & iÑ0Ô0ð 	ð 	‰LˆAˆyÝ! )Ñ,Ô,ð ð ‘��1Ø& qÔ)¨!Ô,�Ý4°Q¸¿º¸Q¼X¹¼ÈÈdÑSÔSð Ý(Ý,Ø˜q ! Q¨¸ÐRVðñ ô ñô ð ððð	õ ˜gÑ&Ô&ð 		ð 		‰DˆAˆqÝ>Ø˜q *Ð.?ñô ˆJõ '¥s¨:°yÀ´|Ñ'DÔ'DÑEÔEð ð ‘	�‘6�A�qÝ4°Q¸¿º¸Q¼X¹¼ÈÈdÑSÔSð Ý(Ý,¨Q°°1°a¸È)ÑTÔTñô ð ððð ˆ4r   c                 ó  — |                       ¦   «         dk    s|                      ¦   «         dk    r9t          d|                       ¦   «         › d|                      ¦   «         › �¦  «        ‚| |z                       ¦   «         S )NrP   z0Expected u and v to be 1D tensors, but got dims rÓ   )rƒ   rr   rh   )r(  rÞ   s     r   rB  rB  �  sl   € Ø‡u‚u�w„w�!‚|€|�q—u’u‘w”w !’|�|ÝØV¸q¿uºu¹w¼wÐVÐVÈQÏUÊUÉWÌWÐVÐVñ
ô 
ð 	
ð �‰E�;Š;‰=Œ=Ðr   c                 óÆ   — t          j        | j        |j        ¦  «        }|                      |¬¦  «        } |                     |¬¦  «        }t          j        | |||¦  «        S rò   )r   Úpromote_typesr>   r_   rÞ  )r¸   rÕ   re  rP  Úpromoted_types        r   rw  rw  ‰  sT   € ÝÔ'¨¬°´Ñ9Ô9€MØ	�Š�=ˆÑ!Ô!€AØ	�Š�=ˆÑ!Ô!€AÝŒ>˜!˜Q  dÑ+Ô+Ð+r   c                 óv   — | t           j        k    rt           j        S | t           j        k    rt           j        S | S r   )r   r¥  r¤  Ú	complex64Úfloat32rO   s    r   Ú_to_real_dtyper„  �  s2   € Ø•Ô Ò Ð ÝŒ}ÐØ	•%”/Ò	!Ð	!ÝŒ}Ðàˆr   c                 ó˜  — | j         t          j        k    rå|                      ¦   «         }|rt	          | j        ¦  «        n| j        }t          j        |                     ¦   «         |¬¦  «                             || j	        ¬¦  «         
                    |j        ¦  «        }||                     ¦   «         z  }t          j        |                      ¦   «         ||                      ¦   «         | j	        ¬¦  «        }�nÏt!          | ¦  «        �rI| j         t          j        t          j        hv r)|                      ¦   «         |                      ¦   «         }}n(|                      ¦   «         |                      ¦   «         }}|                      ¦   «         }|rt	          | j        ¦  «        n| j        }t          j        |                     ¦   «         |¬¦  «                             || j	        ¬¦  «         
                    |j        ¦  «        }||                     ¦   «         z  }t          j        ||||                      ¦   «         | j         | j	        ¬¦  «        }nv|rt	          | j        ¦  «        n| j        }t          j        |                      ¦   «         |¬¦  «                             || j	        ¬¦  «        }||                     ¦   «         z  }|S )N)Ú	generatorrN   r‡   )r   r?   )r   r   r&   r”   r„  r>   Úrandr3   r_   r?   r’  r[   Únormrj   rY   r‚   r$   r   r!   rm   r–   rn   r—   ri   Úsparse_compressed_tensor)	rD   r†  Údowncast_complexr¨   r>   ri   Úvecrx   Úplain_indicess	            r   Ú_vec_from_tensorr�  ™  sI  € ð 	„x•5Ô#Ò#Ð#ð —9’9‘;”;ˆØ+;ÐH•˜qœwÑ'Ô'Ð'ÀÄˆåŒJ�x—~’~Ñ'Ô'°9Ð=Ñ=Ô=ßŠR�e A¤HˆRÑ-Ô-ßŠT�(”.Ñ!Ô!ð 	ð
 	�&—+’+‘-”-ÑˆÝÔ% a§j¢j¡l¤l°F¸A¿FºF¹H¼HÈQÌXÐVÑVÔVˆ‰Ý	% aÑ	(Ô	(ñ ØŒ8�Ô(­%Ô*:Ð;Ð;Ð;Ø01·²Ñ0@Ô0@À!Ç-Â-Á/Ä/ ÐÐà01·²Ñ0@Ô0@À!Ç-Â-Á/Ä/ ÐØ—8’8‘:”:ˆØ+;ÐH•˜qœwÑ'Ô'Ð'ÀÄˆåŒJ�x—~’~Ñ'Ô'°9Ð=Ñ=Ô=ßŠR�e A¤HˆRÑ-Ô-ßŠT�(”.Ñ!Ô!ð 	ð
 	�&—+’+‘-”-ÑˆÝÔ,ØØØØ�FŠF‰HŒHØ”8Ø”8ð
ñ 
ô 
ˆˆð ,<ÐH•˜qœwÑ'Ô'Ð'ÀÄˆÝŒj˜Ÿš™œ¨iÐ8Ñ8Ô8×;Ò;Ø ¤ð <ñ 
ô 
ˆð 	ˆs�xŠx‰zŒzÑˆØ€Jr   c                 ód   — d„ t          | ¦  «        D ¦   «         }d„ |D ¦   «         d„ |D ¦   «         fS )Nc                 óH   — g | ]\  }}t          |¦  «        ¯|j        ¯||f‘Œ S r   r·   )r5   r„   r6   s      r   r8   z$_get_inp_tensors.<locals>.<listcomp>È  sJ   € ð ð ð áˆAˆqÝ˜!ÑÔðð "#¤ðØ	
ˆAˆðð ð r   c                 ó   — g | ]
}|d          ‘ŒS )r   r   ©r5   Útups     r   r8   z$_get_inp_tensors.<locals>.<listcomp>Í  s   € Ð*Ð*Ð*�sˆC�ŒFÐ*Ð*Ð*r   c                 ó   — g | ]
}|d          ‘ŒS )rP   r   r‘  s     r   r8   z$_get_inp_tensors.<locals>.<listcomp>Í  s   € Ð,KÐ,KÐ,K¸¨S°¬VÐ,KÐ,KÐ,Kr   )r¢   )r«  Úinp_idx_tups     r   r  r  Ç  sR   € ðð å˜mÑ,Ô,ðñ ô €Kð
 +Ð*˜kÐ*Ñ*Ô*Ð,KÐ,K¸{Ð,KÑ,KÔ,KÐKÐKr   c                 ó¤  — |€dn|                      ¦   «         }t          |t          ¦  «        rl|d         |d         }}|                      ¦   «         }|                      ¦   «         }| t          |¦  «        t          |¦  «        z   z  t          |¦  «        z  }|S |                      ¦   «         }	| t          |	¦  «        z  t          |¦  «        z  }
|
S )NrÊ   r   rP   )rh   rG   r9   r‹  )rP  r(  rÞ   Úsum_vÚurÚuiÚsum_urÚsum_uiÚcomplex_modified_atolÚsum_uÚmodified_atols              r   Ú_adjusted_atolrž  Ð  s³   € ð$ �9ˆCˆC !§%¢%¡'¤'€Eå�!•UÑÔð %à�1”�q˜”tˆBˆØ—’‘”ˆØ—’‘”ˆØ $­¨f©¬½¸f¹¼Ñ(EÑ FÍÈuÉÌÑ UÐØ$Ð$ð �EŠE‰GŒG€EØ�5 ™<œ<Ñ'­%°©,¬,Ñ6€MØÐr   a§  
Fast gradcheck failed but element-wise differences are small. This means that the
test might've passed in slow_mode!

If you are adding a new operator, please file an issue and then use one of the
workarounds. The workaround depends on how your test invokes gradcheck/gradgradcheck:

If the test
- manually invokes gradcheck/gradgradcheck, then call gradcheck/gradgradcheck
  with `fast_mode=False` as a keyword argument.
- is OpInfo-based (e.g., in test_ops_gradients.py), then modify the OpInfo for the test
  to have `gradcheck_fast_mode=False`
- is a Module test (e.g., in common_nn.py), then modify the corresponding
  module_test entry to have `gradcheck_fast_mode=False`
c	                 óŽ  ‡ ‡‡‡— t          ‰ ‰|||¬¦  «        ‰         ‰         }	|r4ˆ ˆˆˆfd„}
t          |
‰‰         f|‰         f¦  «        d         d         }nt          ‰|‰‰¦  «        }|	|z
                       ¦   «                              ¦   «         }t          j        ||	||¦  «        }d|	› d|› d|› d�}|r
|t          z  }|S )Nrµ  c                 ó\   •— t          ‰¦  «        }| |‰<   t           ‰|Ž ¦  «        ‰         S r   rO  )rÆ   rP  r¸  r	  rŒ  r«  s     €€€€r   Únew_fnz,_run_slow_mode_and_get_error.<locals>.new_fn  s4   ø€ Ý˜mÑ,Ô,ˆJØ$'ˆJ�yÑ!Ý˜T˜T :Ð.Ñ/Ô/°
Ô;Ð;r   r   a  
The above quantities relating the numerical and analytical jacobians are computed 
in fast mode. See: https://github.com/pytorch/pytorch/issues/53876 for more background 
about fast mode. Below, we recompute numerical and analytical jacobians in slow mode:

Numerical:
 z
Analytical:
z1

The max per-element difference (slow mode) is: z.
)rÈ   r1  r�  rM  rp   r   rÞ  ÚFAST_FAIL_SLOW_OK_MSG)r¸  r«  rÁ   r	  rŒ  re  rP  rÃ   rº   Úslow_numericalr¡  Úslow_analyticalÚslow_max_diffÚslow_allcloseÚmsgs   `` ``          r   Ú_run_slow_mode_and_get_errorr¨    sU  øøøø€ õ -Øˆm˜W¨#¸]ðñ ô àôàô€Nð ð 
ð	<ð 	<ð 	<ð 	<ð 	<ð 	<ð 	<ð 	<õ
 >Ø�] 9Ô-Ð/°'¸*Ô2EÐ1Gñ
ô 
à
ôàôˆˆõ 3Ø˜7 I¨zñ
ô 
ˆð
 $ oÑ5×:Ò:Ñ<Ô<×@Ò@ÑBÔB€Må”N ?°NÀDÈ$ÑOÔO€Mð	Mð 'ð	Mð 	Mð (ð		Mð 	Mð
 ;Hð	Mð 	Mð 	Mð ð ð %àÕ$Ñ$ˆØ€Jr   c                 ór   — t          | ¦  «        r'|                      ¦   «                              d¦  «        S | S )NrS   )r'   r£   rš   )rf   s    r   Ú_to_flat_dense_if_sparserª  +  s5   € Ý˜VÑ$Ô$ð Ø�ŠÑ Ô ×(Ò(¨Ñ,Ô,Ð,àˆr   c                ó¾  ‡‡— t          j        ¦   «         Šd„ Šg }g }| D ]§} ‰|‰d¦  «        }t          |¦  «        }|                     ¦   «         rK ‰|‰d¦  «        }|                     ||f¦  «         t          |¦  «        }	|                     ||	f¦  «         Œ}|                     |¦  «         |                     |¦  «         Œ¨|rd nˆˆfd„|D ¦   «         }
|
||fS )Nc                  óp   — t          j        d¦  «        5  t          | Ž cd d d ¦  «         S # 1 swxY w Y   d S )NÚcpu)r   r?   r�  )Úargss    r   Ú_vec_from_tensor_cpuz+_make_vectors.<locals>._vec_from_tensor_cpu6  s€   € õ Œ\˜%Ñ Ô ð 	+ð 	+Ý# TÐ*ð	+ð 	+ð 	+ð 	+ñ 	+ô 	+ð 	+ð 	+ð 	+ð 	+ð 	+ð 	+øøøð 	+ð 	+ð 	+ð 	+ð 	+ð 	+s   •	+«/²/Tc                 ó(   •— g | ]} ‰|‰¦  «        ‘ŒS r   r   )r5   r:   r¯  Úg_cpus     €€r   r8   z!_make_vectors.<locals>.<listcomp>L  s'   ø€ ÐBÐBÐB°3Ð"Ð" 3¨Ñ.Ô.ÐBÐBÐBr   )r   Ú	Generatorrª  r*   rå   )r  rÁ   r  r  Úall_u_denserÆ   r—  Úur_denser˜  Úui_denserD  r¯  r±  s              @@r   r  r  2  s'  øø€ åŒOÑÔ€Eð+ð +ð +ð €EØ€KØð 
)ð 
)ˆØ!Ð! # u¨dÑ3Ô3ˆÝ+¨BÑ/Ô/ˆØ�>Š>ÑÔð 	)Ø%Ð% c¨5°$Ñ7Ô7ˆBØ�LŠL˜"˜b˜Ñ"Ô"Ð"Ý/°Ñ3Ô3ˆHØ×Ò ¨(Ð3Ñ4Ô4Ð4Ð4à�LŠL˜ÑÔÐØ×Ò˜xÑ(Ô(Ð(Ð(ð ð	CˆˆàBÐBÐBÐBÐB¸'ÐBÑBÔBð 
ð
 �%˜Ð$Ð$r   rÝ  c                óÜ  — t          |¦  «        D ]Û\  }}t          |¦  «        D ]Æ\  }}|r| |         |         }n| |         |         }|                     |j        ¬¦  «        }t          |	||         |r||         nd ¦  «        }t	          ||                     |j        ¦  «        ||¦  «        s<t          |||||||	|
|¦	  «	        }t          t          |||||||¦  «        |z   ¦  «        ‚ŒÇŒÜd S )Nr‡   )r¢   r_   r?   rž  rw  r¨  r   r?  )Úall_analyticalÚall_numericalr7  r«  rÁ   r¸  rD  r  re  rP  rÃ   r8  rº   r„   Úall_numerical_for_input_ir]  r{  r¸   Úupdated_atolÚjacobians_strs                       r   Ú!_check_analytical_numerical_equalr¼  Q  s/  € õ  )2°-Ñ(@Ô(@ð ð Ñ$ˆÐ$ÝÐ7Ñ8Ô8ð 	ð 	‰DˆAˆqàð )Ø" 1Ô% aÔ(��à" 1Ô% aÔ(�Ø—’˜AœH�Ñ%Ô%ˆAÝ)¨$°°a´ÀeÐ:U¸%À¼(¸(ÐQUÑVÔVˆLÝ0°°A·D²D¸¼±N´NÀDÈ,ÑWÔWð 	Ý <Ø˜-¨°!°Q¸¸dÀCÈñ!ô !�õ %Ý(Ø˜1˜a  O°YÀñô ð $ñ$ñô ð ð		ð	ðð r   c	                ó¶  — t          |¦  «        \  }}t          |||	¬¦  «        \  }}}|r|||fnt          |||f¦  «        \  }}}t          | |||||||	¬¦  «        }|	r3|�t	          d¦  «        ‚t          | |t          |¦  «        ||¬¦  «        }n+|st          | ||||||¦  «         t          ||||||¦  «        }t          |||
||| |||||||	¬¦  «         dS )Nr  rÝ  zExpected all_v to be None.)r  r  T)
r  r  rW   rK  rr   r1  r»   rÀ  r†  r¼  )r¸  rC  rÀ   rÁ   rÃ   re  rP  r  ro  r  r7  r8  rý  Úinp_tensors_idxr  rD  r  r³  Úinputs_numericalÚall_u_numericalÚall_v_numericalÚnumerical_vJuÚanalytical_vJus                          r   Ú_fast_gradcheckrÄ  v  sv  € õ" $4°FÑ#;Ô#;Ñ €O�[õ !.Ø�W¨^ð!ñ !ô !Ñ€Eˆ5�+ð
 #)ÐNˆ�˜ÐÐ­h¸ÀÀuÐ7MÑ.NÔ.Nñ 7Ð�o õ 'ØØØØØØØØ$ð	ñ 	ô 	€Mð ð 
ØÐÝ Ð!=Ñ>Ô>Ð>Ý<ØØÝ�hÑÔØØ/ð
ñ 
ô 
ˆˆð ð 	Ý1Ø�h ¨¸ÀÀZñô ð õ ;Ø�G˜ZÐ):¸EÀ;ñ
ô 
ˆõ &ØØØØØØØØØØØØØ$ðñ ô ð ð  ˆ4r   g�íµ ÷Æ°>gñhãˆµøä>T)rÃ   rP  re  Úraise_exceptionro  rh  r  Úcheck_batched_gradÚcheck_batched_forward_gradrf  rg  rù   rý  r¸  rÀ   rÃ   rP  re  rÅ  ro  rh  r  rÆ  rÇ  rf  rg  rù   rý  c                ó6  — |s|st          d¦  «        ‚|	r|st          d¦  «        ‚|
r|st          d¦  «        ‚t          ¦   «                              ¦   «         }|                     d¦  «         |s	 t	          di |¤ŽS # t
          $ r Y dS w xY wt	          di |¤ŽS )a‹  Check gradients computed via small finite differences against analytical
    gradients wrt tensors in :attr:`inputs` that are of floating point or complex type
    and with ``requires_grad=True``.

    The check between numerical and analytical gradients uses :func:`~torch.allclose`.

    For most of the complex functions we consider for optimization purposes, no notion of
    Jacobian exists. Instead, gradcheck verifies if the numerical and analytical values of
    the Wirtinger and Conjugate Wirtinger derivatives are consistent. Because the gradient
    computation is done under the assumption that the overall function has a real-valued
    output, we treat functions with complex output in a special way. For these functions,
    gradcheck is applied to two real-valued functions corresponding to taking the real
    components of the complex outputs for the first, and taking the imaginary components
    of the complex outputs for the second. For more details, check out
    :ref:`complex_autograd-doc`.

    .. note::
        The default values are designed for :attr:`input` of double precision.
        This check will likely fail if :attr:`input` is of less precision, e.g.,
        ``FloatTensor``.

    .. note::
        Gradcheck may fail when evaluated on non-differentiable points
        because the numerically computed gradients via finite differencing may differ
        those computed analytically (not necessarily because either is incorrect).
        For more context, see :ref:`non-differentiable-func-grad`.

    .. warning::
       If any checked tensor in :attr:`input` has overlapping memory, i.e.,
       different indices pointing to the same memory address (e.g., from
       :func:`torch.Tensor.expand`), this check will likely fail because the numerical
       gradients computed by point perturbation at such indices will change
       values at all other indices that share the same memory address.

    Args:
        func (function): a Python function that takes Tensor inputs and returns
            a Tensor or a tuple of Tensors
        inputs (tuple of Tensor or Tensor): inputs to the function
        eps (float, optional): perturbation for finite differences
        atol (float, optional): absolute tolerance
        rtol (float, optional): relative tolerance
        raise_exception (bool, optional): indicating whether to raise an exception if
            the check fails. The exception gives more information about the
            exact nature of the failure. This is helpful when debugging gradchecks.
        nondet_tol (float, optional): tolerance for non-determinism. When running
            identical inputs through the differentiation, the results must either match
            exactly (default, 0.0) or be within this tolerance.
        check_undefined_grad (bool, optional): if ``True``, check if undefined output grads
            are supported and treated as zeros, for ``Tensor`` outputs.
        check_batched_grad (bool, optional): if ``True``, check if we can compute
            batched gradients using prototype vmap support. Defaults to False.
        check_batched_forward_grad (bool, optional): if ``True``, checks if we can compute
            batched forward gradients using forward ad and prototype vmap support. Defaults to ``False``.
        check_forward_ad (bool, optional): if ``True``, check that the gradients computed with forward
            mode AD match the numerical ones. Defaults to ``False``.
        check_backward_ad (bool, optional): if ``False``, do not perform any checks that rely on
            backward mode AD to be implemented. Defaults to ``True``.
        fast_mode (bool, optional): Fast mode for gradcheck and gradgradcheck is currently only
            implemented for R to R functions. If none of the inputs and outputs are complex
            a faster implementation of gradcheck that no longer computes the entire jacobian
            is run; otherwise, we fall back to the slow implementation.
        masked (bool, optional): if ``True``, the gradients of unspecified elements of
            sparse tensors are ignored. Defaults to ``False``.
    Returns:
        ``True`` if all differences satisfy allclose condition

    zIExpected at least one of check_forward_ad or check_backward_ad to be TruezESetting check_batched_grad=True requires check_backward_ad to be TruezLSetting check_batched_forward_grad=True requires check_forward_ad to be TruerÅ  FNr   )rr   ÚlocalsÚcopyÚpopÚ_gradcheck_helperr   )r¸  rÀ   rÃ   rP  re  rÅ  ro  rh  r  rÆ  rÇ  rf  rg  rù   rý  r®  s                   r   r   r   Ï  s÷   € ðj ð 
Ð 1ð 
ÝØWñ
ô 
ð 	
ð ð 
Ð"3ð 
ÝØSñ
ô 
ð 	
ð "ð 
Ð*:ð 
ÝØZñ
ô 
ð 	
õ ‰8Œ8�=Š=‰?Œ?€DØ‡H‚HÐÑÔÐØð )ð	Ý$Ð,Ð, tÐ,Ð,Ð,øÝð 	ð 	ð 	Ø�5�5ð	øøøõ !Ð(Ð( 4Ð(Ð(Ð(s   Á2A> Á>
BÂBc                 óÐ  — t          |¦  «        }t          |¦  «          | |Ž }t          |¦  «        }t          |¦  «         t	          j        |rt          nt          |¬¦  «        }t          || ||||||||
|||¬¦  «         |	rt          | |¦  «         |sdS t          |¦  «        D ]\  }}|rt          |||¦  «         Œt          |||¦  «         |r|rt          | ||¦  «         dS )N)rý  )rf  rg  ro  rh  T)r»   r®  r)  r³  r  r  rÄ  r|  rs  rã  r¢   rñ  r  r+  )r¸  rÀ   rÃ   rP  re  ro  rh  r  rÆ  rÇ  rf  rg  rù   rý  r«  rC  rÁ   rd  r„   rµ   s                       r   rÌ  rÌ  ;  sE  € õ  ˜fÑ%Ô%€MÝ�-Ñ Ô Ð àˆt�]Ð#€HÝ% hÑ/Ô/€GÝ�7ÑÔÐåÔ$Ø$Ð9�ˆ­/À&ðñ ô €Lõ ØØØØØØØØØØ)Ø+ØØ1ðñ ô ð ð  "ð ;Ý% d¨MÑ:Ô:Ð:ð ð Øˆtå˜'Ñ"Ô"ð 4ð 4‰ˆˆ1Øð 	4Ý˜}¨a°Ñ3Ô3Ð3øå% g¨}¸fÑEÔEÐEàð DÐ 1ð DÝ% d¨G°]ÑCÔCÐCØˆ4r   )rÃ   rP  re  Úgen_non_contig_grad_outputsrÅ  ro  rh  r  rÆ  Úcheck_fwd_over_revÚcheck_rev_over_revrù   rý  rð  rÎ  rÏ  rÐ  c                óì  ‡ ‡‡‡‡— |s|st          d¦  «        ‚|	r|st          d¦  «        ‚|r|st          d¦  «        ‚t          |¦  «        }|€.t           ‰ |Ž ¦  «        }t          ˆfd„|D ¦   «         ¦  «        }nt          |¦  «        }t	          |¦  «        Šd„ t          |¦  «        D ¦   «         Šd„ t          |¦  «        D ¦   «         Šˆˆˆ ˆfd„}t          |||z   ||||||	|
|||||¬	¦  «        S )
a5  Check gradients of gradients computed via small finite differences
    against analytical gradients wrt tensors in :attr:`inputs` and
    :attr:`grad_outputs` that are of floating point or complex type and with
    ``requires_grad=True``.

    This function checks that backpropagating through the gradients computed
    to the given :attr:`grad_outputs` are correct.

    The check between numerical and analytical gradients uses :func:`~torch.allclose`.

    .. note::
        The default values are designed for :attr:`input` and
        :attr:`grad_outputs` of double precision. This check will likely fail if
        they are of less precision, e.g., ``FloatTensor``.

    .. warning::
       If any checked tensor in :attr:`input` and :attr:`grad_outputs` has
       overlapping memory, i.e., different indices pointing to the same memory
       address (e.g., from :func:`torch.Tensor.expand`), this check will likely fail
       because the numerical gradients computed by point perturbation at such
       indices will change values at all other indices that share the same
       memory address.

    Args:
        func (function): a Python function that takes Tensor inputs and returns
            a Tensor or a tuple of Tensors
        inputs (tuple of Tensor or Tensor): inputs to the function
        grad_outputs (tuple of [Tensor or None] or Tensor, optional): The gradients with
            respect to the function's outputs.
        eps (float, optional): perturbation for finite differences
        atol (float, optional): absolute tolerance
        rtol (float, optional): relative tolerance
        gen_non_contig_grad_outputs (bool, optional): if :attr:`grad_outputs` is
            ``None`` and :attr:`gen_non_contig_grad_outputs` is ``True``, the
            randomly generated gradient outputs are made to be noncontiguous
        raise_exception (bool, optional): indicating whether to raise an exception if
            the check fails. The exception gives more information about the
            exact nature of the failure. This is helpful when debugging gradchecks.
        nondet_tol (float, optional): tolerance for non-determinism. When running
            identical inputs through the differentiation, the results must either match
            exactly (default, 0.0) or be within this tolerance. Note that a small amount
            of nondeterminism in the gradient will lead to larger inaccuracies in
            the second derivative.
        check_undefined_grad (bool, optional): if True, check if undefined output grads
            are supported and treated as zeros
        check_batched_grad (bool, optional): if True, check if we can compute
            batched gradients using prototype vmap support. Defaults to False.
        fast_mode (bool, optional): if True, run a faster implementation of gradgradcheck that
            no longer computes the entire jacobian.
        masked (bool, optional): if True, the gradients of unspecified elements of
            sparse tensors are ignored (default, False).
    Returns:
        True if all differences satisfy allclose condition
    zLExpected at least one of check_fwd_over_rev or check_rev_over_rev to be TruezHSetting check_undefined_grad=True requires check_rev_over_rev to be TruezFSetting check_batched_grad=True requires check_rev_over_rev to be TrueNc           
   3   óê   •K  — | ]m}t           j                             |j        |                     ¦   «         s|                     ¦   «         r|j        nt           j        |j        d dd‰¬¦  «        V — ŒndS )rS   rP   T)r>   r?   ÚlowÚhighr1   ÚnoncontiguousN)	r   ÚtestingÚmake_tensorr[   r)   r*   r>   Údoubler?   )r5   rD   rÎ  s     €r   r�   z gradgradcheck.<locals>.<genexpr>Õ  s•   øè è € ð $
ð $
ð õ ŒM×%Ò%Ø”à×&Ò&Ñ(Ô(ð"Ø,-¯LªL©N¬Nð"�a”g�gå”\Ø”xØØØ"Ø9ð &ñ 
ô 
ð$
ð $
ð $
ð $
ð $
ð $
r   c                 óD   — h | ]\  }}t          |¦  «        ¯|j        ¯|’ŒS r   r·   ©r5   r„   rD   s      r   ú	<setcomp>z gradgradcheck.<locals>.<setcomp>ê  sC   € ð ð ð Ùˆa�µ.ÀÑ2CÔ2CðØHIÌðØ	ðð ð r   c                 ó&   — h | ]\  }}|j         ¯|’ŒS r   r0  rÚ  s      r   rÛ  z gradgradcheck.<locals>.<setcomp>í  s3   € ð  ð  ð  Ùˆa�¸¼ð Ø	ð ð  ð  r   c                  óº  •— t          ˆfd„t          | d ‰	 …         ¦  «        D ¦   «         ¦  «        }t           ‰|Ž ¦  «        }t          ˆfd„t          | ‰	 d …         ¦  «        D ¦   «         ¦  «        }t          ˆfd„t          |¦  «        D ¦   «         ¦  «        }t          j                             |||dd¬¦  «        }t          d„ |D ¦   «         ¦  «        }|S )Nc              3   óR   •K  — | ]!\  }}|‰v r|                      ¦   «         n|V — Œ"d S r   ©rl   ©r5   r„   rD   Údiff_input_args_indicess      €r   r�   z2gradgradcheck.<locals>.new_func.<locals>.<genexpr>ó  sW   øè è € ð 
ð 
á��1ð #$Ð'>Ð">Ð">ˆA×ÒÑÔÐÀAð
ð 
ð 
ð 
ð 
ð 
r   c              3   óR   •K  — | ]!\  }}|‰v r|                      ¦   «         n|V — Œ"d S r   rß  )r5   r„   rD   Údiff_grad_output_indicess      €r   r�   z2gradgradcheck.<locals>.new_func.<locals>.<genexpr>ø  sW   øè è € ð 
ð 
á��1ð #$Ð'?Ð"?Ð"?ˆA×ÒÑÔÐÀQð
ð 
ð 
ð 
ð 
ð 
r   c              3   ó*   •K  — | ]\  }}|‰v ¯	|V — Œd S r   r   rà  s      €r   r�   z2gradgradcheck.<locals>.new_func.<locals>.<genexpr>ü  s;   øè è € ð  
ð  
Ù�!�Q°1Ð8OÐ3OÐ3OˆAÐ3OÐ3OÐ3OÐ3Oð 
ð  
r   T)Úcreate_graphre  c              3   ó   K  — | ]}|®|V — Œ	d S r   r   )r5   Úgs     r   r�   z2gradgradcheck.<locals>.new_func.<locals>.<genexpr>	  s"   è è € ÐDÐD !°a°m˜A°m°m°m°mÐDÐDr   )r9   r¢   r)  r   r  rg  )
r®  Ú
input_argsrÁ   rð  Údiff_input_argsr˜  rã  rá  r¸  Únum_outputss
         €€€€r   Únew_funczgradgradcheck.<locals>.new_funcñ  s6  ø€ åð 
ð 
ð 
ð 
å! $ }¨¨ }Ô"5Ñ6Ô6ð
ñ 
ô 
ñ 
ô 
ˆ
õ *¨$¨$°
Ð*;Ñ<Ô<ˆÝð 
ð 
ð 
ð 
å! $¨ | } }Ô"5Ñ6Ô6ð
ñ 
ô 
ñ 
ô 
ˆõ  ð  
ð  
ð  
ð  
Ý# JÑ/Ô/ð 
ñ  
ô  
ñ 
ô 
ˆõ ”n×)Ò)Ø�_ lÀÐTXð *ñ 
ô 
ˆõ ÐDÐD {ÐDÑDÔDÑDÔDˆØÐr   )rÃ   rP  re  rÅ  ro  rh  r  rÆ  rù   rf  rg  rý  )rr   r»   r)  r9   r   r¢   r   )r¸  rÀ   rð  rÃ   rP  re  rÎ  rÅ  ro  rh  r  rÆ  rÏ  rÐ  rù   rý  r«  rÁ   Útupled_grad_outputsrë  rã  rá  rê  s   `     `             @@@r   r   r   w  sÁ  øøøøø€ ðR ð 
Ð"4ð 
ÝØZñ
ô 
ð 	
ð ð 
Ð$6ð 
ÝØVñ
ô 
ð 	
ð ð 
Ð"4ð 
ÝØTñ
ô 
ð 	
õ ˜fÑ%Ô%€MàÐõ *¨$¨$°Ð*>Ñ?Ô?ˆÝ#ð $
ð $
ð $
ð $
ð ð$
ñ $
ô $
ñ 
ô 
ÐÐõ (¨Ñ5Ô5ÐåÐ)Ñ*Ô*€Kðð Ý Ñ.Ô.ðñ ô Ðð ð  ÝÐ 3Ñ4Ô4ð ñ  ô  Ððð ð ð ð ð ð ð õ( ØØÐ+Ñ+ØØØØ'ØØ1Ø+Ø-ØØ+Ø,Øðñ ô ð r   )NN)F)NNr°   F)Nr°   rÊ   r   )r-   N)NF)FN)r  rÊ   )FF)erH   r  r¦  Úcollections.abcr   r   Ú	itertoolsr   Útyping_extensionsr   r   Útorch.testingÚtorch._vmap_internalsr   r   Útorch.overridesr	   Útorch.typesr
   r   Ú__all__rª  r   r“  r$   r'   r+   r9   r;   rC   ÚboolrK   rW   r¯   rT   rÈ   ÚFutureWarningr   râ   rï   ÚdictÚintrõ   rú   r  r   r1  r3  r  r  r<  r>  r@  rK  rS  r_  rn  ry  r†  r   r�  rm  rl  r®  r³  r»  rÀ  rÅ  rÄ  rÃ  rÇ  rã  rñ  r  r  r+  r»   r)  rJ   r?  rB  rK  r[  rs  r|  rB  rw  r„  r�  r  rž  r¢  r¨  rª  r  r¼  rÄ  r‹  r   rÌ  r   r   r   r   ú<module>rù     sæ
  ðà Ð Ð Ð Ø Ð Ð Ð Ø €€€Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø Ð Ð Ð Ð Ð Ø (Ð (Ð (Ð (Ð (Ð (à €€€Ø Ð Ð Ð ð .Ð -Ð -Ð -Ð -Ð -Ð -Ð -Ø *Ð *Ð *Ð *Ð *Ð *Ø BÐ BÐ BÐ BÐ BÐ BÐ BÐ Bðð ð €ðHð Hð Hð Hð H�\ñ Hô Hð Hð e¤lð ð ð ð ðO˜uœ|ð Oð Oð Oð OðQð Qð QðØðà
ˆ5Œ<˜ÐÔðð ð ð ð" <@ðð Øðà
ˆ5Œ<˜ÐÔðð ð ð ð" KPð	@ð 	@Ø„|�h˜uœ|Ô,Ñ,ð	@ØCGð	@àˆeŒlÔð	@ð 	@ð 	@ð 	@ð;ð ;ð ;ð|d)ð d)ð d)ðP DIð2ð 2à	ˆ%�”˜cÐ!Ô
"Ô#ð2ð 2ð 2ð 2ðj €ðCð
 ðñ ô ð!Xð !Xð !Xñô ð!XðH#>ð #>ð #>ðN 49ðð à	ˆ%Œ,Ôðð ð ð ð:Ø˜˜d 5¤<Ô0Ð0Ô1ðà
ˆ5Œ<˜ÐÔðð ð ð ð PUðð ØŒ<ðØ05´¸tÑ0Cðà
„\ðð ð ð ð0
ð 
ð 
ð 
ð 
ð& DIðPð Pà
ˆ5Œ<˜ÐÔðPð Pð Pð Pð8 /4¸4ðið ið ià
ˆ5�”˜sÐ"Ô# SÐ(Ô)ðið ið ið iðXð ð ð ð ð ð ðð ð ðð ð ðð ð ð 27ðð à	ˆ%Œ,Ôðð ð ð ðð ð ðBð ð ðAà
ˆ5�”˜sÐ"Ô# T¨4Ð/Ô0ðAð Að Að Að<Ð ð$ GKð)ð )à
ˆ5Œ<˜ÐÔð)ð )ð )ð )ðXð ð ð0 €ðCð
 ðñ ô ð!5ð !5ð !5ñô ð!5ðH ð  ð  ðà	ˆ$ˆuŒ|˜dÑ"Ô
#Ô$ðð ð ð ð6à	ˆ$ˆuŒ|˜dÑ"Ô
#Ô$ðð ð ð ð' Dð 'ð 'ð 'ð 'ðT
ð 
ð 
ð 
ðà	ðð ð ð ð"ð ð ð÷, 
‚E�G„Gð- ð0"Ð ð$ 49ðð ð ð ð 9°4ð 9ð 9ð 9ð 9ðx4°Tð 4ð 4ð 4ð 4ðn3À$ð 3ð 3ð 3ð 3ðl@ð @ð @ðF@T¸Dð @Tð @Tð @Tð @TðFð ð ð=ð =ð =ð Øðð ð 	ðð ð ð ð8)ð )ð )ð

Lð 
Lð 
Lðð ð ð*DKð DKð DKðd ØØØð>ð >ð >ð >ð >ðBð ð ð,ð ,ð ,ðð ð ð+ð +ð +ð +ð\Lð Lð Lðð ð ðD÷ 
‚E�G„Gð ð"%ð %ð %ðPð ð ð%ð %ð %ðZ ð"ð "ð "ð "ð "ð` ØØØðPð Pð Pð Pð Pðz ØØØ ØØ!%Ø#Ø$Ø',Ø"Ø"ØØð!i)ð i)ð i)Ø
�3Ð(Ð(Ô
)ði)àði)ð 
ð	i)ð
 ði)ð ði)ð ði)ð ði)ð ði)ð ði)ð ði)ð !%ði)ð ði)ð ði)ð ði)ð  �4‰Kð!i)ð" 
ð#i)ð i)ð i)ð i)ðX9ð 9ð 9ð~ 59ð]ð
 ØØØ(-Ø ØØ!%Ø#Ø$Ø$Ø#ØØð#]ð ]ð ]Ø
�3Ð(Ð(Ô
)ð]àð]ð +¨TÑ1ð]ð
 
ð]ð ð]ð ð]ð "&ð]ð ð]ð ð]ð ð]ð ð]ð ð]ð ð]ð ð]ð  ð!]ð" ð#]ð$ 
ð%]ð ]ð ]ð ]ð ]ð ]r   