§
    ŠŠtjr*  ã                   óô  — d Z ddlZddlZddlmZ ddlmZmZ ddlm	Z	 ddl
Z
ddlZ
ddlmZ ddlmZ ddlmZ dd	lmZmZmZmZmZmZ dd
lmZmZmZmZmZ ddlm Z  ddl!m"Z"  ej#        e$¦  «        Z%de
j&        j'        de(e)         fd„Z*de
j&        j+        de,e
j-        e
j&        j.        f         fd„Z/de
j&        j+        de)de0dz  fd„Z1de
j&        j+        de)de0dz  fd„Z2de
j&        j+        de)fd„Z3de
j&        j+        de4e0dz           fd„Z5de
j&        j+        dee	         de	fd„Z6 G d„ d¦  «        Z7 e"d e7¦   «         ¬¦  «         	 	 d'd!ed"e	f         d#ee	         d$e8d%e8ded"ee	         f         f
d&„Z9dS )(a�  
This module implements CUDA graphs support for TorchDynamo backends.

CUDA graphs allow for capturing and replaying GPU operations, which can significantly
reduce CPU overhead in GPU-accelerated PyTorch models. This module provides:

- CUDA graph creation and management for both forward and backward passes
- Input mutation detection and handling
- Device compatibility checking
- Stack trace management for debugging
- Integration with TorchInductor's cudagraph trees

The backend supports two main modes:
1. cudagraphs: Full CUDA graph support with both forward and backward pass optimization
2. cudagraphs_inner: Lower-level CUDA graph implementation used for benchmarking

Key components:
- CudagraphsBackend: Main backend class for CUDA graph integration
- Mutation detection utilities to ensure graph safety
- Device mapping and compatibility checks
- Stack trace collection for debugging
é    N)Údefaultdict)ÚCallableÚSequence)ÚAny)Úconfig)Úaot_autograd)Ú	boxed_nop)ÚBoxedDeviceIndexÚ'check_multiple_devices_or_any_cpu_nodesÚformat_default_skip_messageÚget_mutation_stack_traceÚget_placeholder_infoÚ#log_cudagraph_skip_and_bump_counter)Ú	BoxedBoolÚcount_tangentsÚ%get_first_incompatible_cudagraph_nodeÚnum_fw_fixed_argumentsÚoutput_node)ÚStorageWeakRefé   )Úregister_backendÚgÚreturnc           	      ón  — dt           t          t          f         dt          fd„}t          t          ¦  «        }d}t	          ¦   «         }| j        D �]c}|j        dk    rvt           ||j        ¦  «        t          j
        ¦  «        rH|t           ||j        ¦  «                             ¦   «         ¦  «                                      |¦  «         |dz  }Œ„|j        dk    rÔt          |j        d¦  «        sŒ¥|j        j        }t#          |j        ¦  «        D ]�\  }}|t'          |j        ¦  «        k     r|j        |         }	n!|j        |j        vrŒ:|j        |j                 }	d	}
|j        r|j        j        rd
}
|
r8||t           ||	j        ¦  «                             ¦   «         ¦  «                 z  }Œž�Œe|S )NÚmetar   c                 ó*   — d| v r| d         n| d         S )NÚvalÚfake_result© )r   s    ú_/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/torch/_dynamo/backends/cudagraphs.pyÚmeta_fkz%find_input_mutations.<locals>.meta_fk;   s   € Ø# t˜m˜mˆt�EŒ{ˆ{°°mÔ1DÐDó    r   Úplaceholderr   Úcall_functionÚ_schemaFT)ÚdictÚstrr   r   ÚsetÚnodesÚopÚ
isinstancer   ÚtorchÚTensorr   Ú_typed_storageÚaddÚhasattrÚtargetr%   Ú	enumerateÚ	argumentsÚlenÚargsÚnameÚkwargsÚ
alias_infoÚis_write)r   r!   ÚinputsÚ	input_idxÚmutated_inputsÚnÚschemaÚiÚargÚargumentÚmut_args              r    Úfind_input_mutationsrC   :   sÁ  € ðE•d�3¥˜8”nð E­ð Eð Eð Eð Eõ �ÑÔ€FØ€IÝ‘U”U€NØŒWð ñ ˆØŒ4�=Ò Ð Ý˜'˜' !¤&™/œ/­5¬<Ñ8Ô8ð XØ•~ g g¨a¬f¡o¤o×&DÒ&DÑ&FÔ&FÑGÔGÔH×LÒLÈYÑWÔWÐWØ˜‰NˆIˆIØŒT�_Ò$Ð$Ý˜1œ8 YÑ/Ô/ð Øà”XÔ%ˆFÝ# FÔ$4Ñ5Ô5ð ð ‘��3Ø•s˜1œ6‘{”{’?�?Ø œv aœy�H�Hà”x q¤xÐ/Ð/Ø Ø œx¨¬Ô1�HØ�Ø”>ð 'Ø”~Ô.ð 'Ø"&˜Øð ð # fÝ& w w¨x¬}Ñ'=Ô'=×'LÒ'LÑ'NÔ'NÑOÔOô'ñ �Nøùð
 Ðr"   Úgmc                 ó¸   — i }| j         j        D ]J}|j                             dd ¦  «        }t	          |t
          j        ¦  «        r|j        |vr
|||j        <   ŒK|S )Nr   )Úgraphr)   r   Úgetr+   r,   r-   Údevice)rD   Údevice_node_mappingr=   Úts       r    Úget_device_node_mappingrK   a   sh   € ð >@ÐØŒXŒ^ð .ð .ˆØŒF�JŠJ�u˜dÑ#Ô#ˆÝ�a�œÑ&Ô&ð 	.¨1¬8Ð;NÐ+NÐ+NØ,-Ð ¤Ñ)øØÐr"   Ú	aot_modelÚ	num_fixedc                 ó´   — t          | j        ¦  «        t          t          |¦  «        ¦  «        z
  }|sd S t	          | j        ¦  «        }t          ||¦  «        S ©N)rC   rF   r(   Úranger   r   )rL   rM   Úmutation_indicesÚplaceholderss       r    Ú3check_for_mutation_ignore_cuda_graph_managed_tensorrS   l   sV   € õ ,¨I¬OÑ<Ô<½sÅ5ÈÑCSÔCSÑ?TÔ?TÑTÐØð Øˆtå'¨	¬Ñ8Ô8€LÝ# LÐ2BÑCÔCÐCr"   c                 óØ   — t           j        st          | |¦  «        x}r|S t          t	          | ¦  «        ¦  «        x}r|S t          | ¦  «        x}rt          d|j        › d�¦  «        S d S )Nzincompatible op (ú))r   Ú(cudagraph_backend_support_input_mutationrS   r   rK   r   r   r6   )rL   rM   Úmut_skipÚskipÚnodes        r    Úcheck_for_skiprZ   w   s”   € ÝÔ:ð ÝJØ�yñ
ô 
ð 
ˆ8ð 	ð ˆOå6Ý 	Ñ*Ô*ñô ð €tð ð ˆå4°YÑ?Ô?Ð?€tð MÝ*Ð+K¸t¼yÐ+KÐ+KÐ+KÑLÔLÐLàˆ4r"   c                 ó¦   — t          t          t          | ¦  «        ¦  «        ¦  «        }|j        dk    rt	          d|j        › �¦  «        ‚|j        S )NÚcudazExpected CUDA device, got )ÚnextÚiterrK   ÚtypeÚAssertionErrorÚindex)rD   rH   s     r    Úget_device_indexrb   ‰   sN   € Ý•$Õ.¨rÑ2Ô2Ñ3Ô3Ñ4Ô4€FØ„{�fÒÐÝÐG¸&¼+ÐGÐGÑHÔHÐHØŒ<Ðr"   c                 óî   — t          | ¦  «        }t          |j        ¦  «        dk    r$t          dt          |j        ¦  «        › �¦  «        ‚|j        d         }t	          |d¦  «        sg S d„ |D ¦   «         S )Nr   z(Expected output node to have 1 arg, got r   Ú__iter__c                 óh   — g | ]/}t          |t          j        j        j        ¦  «        r|j        nd ‘Œ0S rO   )r+   r,   ÚfxrY   ÚNodeÚstack_trace)Ú.0r@   s     r    ú
<listcomp>z$get_stack_traces.<locals>.<listcomp>™   sD   € ð ð ð àõ ' s­E¬H¬MÔ,>Ñ?Ô?Ð	IˆŒˆÀTðð ð r"   )r   r4   r5   r`   r0   )rD   Úoutputr5   s      r    Úget_stack_tracesrl   �   sŽ   € Ý˜‰_Œ_€FÝ
ˆ6Œ;ÑÔ˜1ÒÐÝØIµs¸6¼;Ñ7GÔ7GÐIÐIñ
ô 
ð 	
ð Œ;�qŒ>€DÝ�4˜Ñ$Ô$ð Øˆ	ðð àðñ ô ð r"   Údynamo_modelÚdynamo_inputsc           	      ó¼  ‡‡‡‡— ddl mŠ t          d¦  «        Št          d ¦  «        Š	 ddt          j        j        dt          t                   dt          dt          fˆˆˆˆfd	„}dt          j        j        dt          t                   dt          fˆˆˆfd
„}t          ||t          j        |d¬¦  «        t          j        j        j        ¬¦  «        } || ‰¦  «        S )Nr   )Úcudagraphify_implTFrL   Ú
aot_inputsÚis_inferencer   c                 óð  •— t          | |¦  «        }t          t          ‰
¦  «        t          |¦  «        ¦  «        }t          | |¦  «        x}r(t	          j        ‰	¦  «         t          d|› �¦  «         |S ‰                     t          | ¦  «        ¦  «          ‰||t          |¦  «        ‰j
        d|t          | ¦  «        t          | j        ¦  «        t          | j        ¦  «        ¬¦	  «	        }d|_        |S )Núskipping cudagraphs due to F©Údevice_indexÚis_backwardrr   Ústack_tracesrR   Úmutated_input_idxsT)r	   r   r4   rZ   r   Údisabler   r(   rb   rP   Úvaluerl   r   rF   rC   Ú_boxed_call)rL   rq   rr   ÚinterpÚfixedÚskip_msgÚoutÚboxed_device_indexrp   Údo_cudagraphsrn   s          €€€€r    Úforward_cudagraphsz&cudagraphs.<locals>.forward_cudagraphs¥   sÿ   ø€ õ
 ˜9 jÑ1Ô1ˆÝ&¥s¨=Ñ'9Ô'9½3¸z¹?¼?ÑKÔKˆÝ% i°Ñ7Ô7Ð7ˆ8ð 	ÝÔ˜mÑ,Ô,Ð,Ý/Ø8¨hÐ8Ð8ñô ð ð ˆMà×ÒÕ/°	Ñ:Ô:Ñ;Ô;Ð;ØÐØØÝ�%‰LŒLØ+Ô1ØØ%Ý)¨)Ñ4Ô4Ý-¨i¬oÑ>Ô>Ý3°I´OÑDÔDð

ñ 

ô 

ˆð ˆŒØˆ
r"   c                 ó<  •‡ ‡— t          ‰ |¦  «        }‰s‰ S t          ‰ ¦  «        }t          ‰ |¦  «        x}r}t          d|› �¦  «         ‰	j        }|€d}t
          j        j                             |d¬¦  «        Š‰€t          d¦  «        ‚dt          t                   dt          fˆ ˆfd„}d	|_        |S  ‰
||t          |¦  «        t          ‰ ¦  «        d	dt          ‰ ¦  «        t!          ‰ j        ¦  «        t%          ‰ j        ¦  «        ¬
¦	  «	        }d	|_        |S )Nrt   r   F)Úcreate_if_none_existsz)cudagraph manager must exist for backwardr:   r   c                 óB   •— ‰                      ¦   «           ‰| ¦  «        S rO   )Úset_to_running_backward)r:   rL   Úmanagers    €€r    Úfnz3cudagraphs.<locals>.backward_cudagraphs.<locals>.fnÙ   s%   ø€ à×/Ò/Ñ1Ô1Ð1Ø �y Ñ(Ô(Ð(r"   Tru   )r	   r   rZ   r   r{   r,   Ú	_inductorÚcudagraph_treesÚget_managerr`   Úlistr   r|   rP   rb   rl   r   rF   rC   )rL   rq   r}   r~   r   Ú
device_idxr‰   r€   rˆ   r�   rp   r‚   s   `       @€€€r    Úbackward_cudagraphsz'cudagraphs.<locals>.backward_cudagraphsÂ   sb  øøø€ õ ˜9 jÑ1Ô1ˆØð 	ØÐå˜yÑ)Ô)ˆÝ% i°Ñ7Ô7Ð7ˆ8ð 	Ý/Ø8¨hÐ8Ð8ñô ð ð
 ,Ô1ˆJØÐ!Ø�
Ý”oÔ5×AÒAØ°%ð Bñ ô ˆGð ˆÝ$Ð%PÑQÔQÐQð)�4¥œ9ð )­ð )ð )ð )ð )ð )ð )ð )ð
 "ˆBŒNØˆIàÐØØÝ�%‰LŒLÝ)¨)Ñ4Ô4ØØÝ)¨)Ñ4Ô4Ý-¨i¬oÑ>Ô>Ý3°I´OÑDÔDð

ñ 

ô 

ˆð ˆŒØˆ
r"   )rr   )Úfw_compilerÚbw_compilerÚinference_compilerÚkeep_inference_input_mutations)F)Útorch._inductor.cudagraph_treesrp   r   r
   r,   rf   ÚGraphModuler�   r   Úboolr   Ú	functoolsÚpartialÚ_dynamor   Ú%cudagraph_backend_keep_input_mutation)rm   rn   rƒ   r�   Úaot_cudagraphsr�   rp   r‚   s    `   @@@r    Ú
cudagraphsrœ   Ÿ   s.  øøøø€ ØAÐAÐAÐAÐAÐAå˜d‘O”O€MÝ)¨$Ñ/Ô/Ðð
 #ðð Ý”8Ô'ðå�”Iðõ ðõ 
ð	ð ð ð ð ð ð ð ð ð:+Ý”8Ô'ð+Ý59½#´Yð+å	ð+ð +ð +ð +ð +ð +ð +ð +õZ "Ø&Ø'Ý$Ô,Ð-?ÈdÐSÑSÔSÝ',¤}Ô';Ô'að	ñ ô €Nð ˆ>˜,¨Ñ6Ô6Ð6r"   c            	       ór   — e Zd ZdZed	d„¦   «         Zedej        j        de	e
         de
de
fd„¦   «         ZdS )
ÚCudagraphsBackendrœ   r   Nc                  ó&   — ddl m}   | ¦   «          d S )Nr   ©Úreset_cudagraph_trees)r”   r¡   r    s    r    ÚresetzCudagraphsBackend.resetû   s)   € àIÐIÐIÐIÐIÐIàÐÑÔÐÐÐr"   Úmodelr:   r7   c                 ó\   — |rt                                d|¦  «         t          | |¦  «        S )Nz+cudagraphs backend ignoring extra kwargs %s)ÚlogÚwarningrœ   )r£   r:   r7   s      r    Ú__call__zCudagraphsBackend.__call__  s2   € ð ð 	OÝ�KŠKÐEÀvÑNÔNÐNÝ˜% Ñ(Ô(Ð(r"   )r   N)Ú__name__Ú
__module__Ú__qualname__Úcompiler_nameÚstaticmethodr¢   r,   rf   r•   r   r   r§   r   r"   r    rž   rž   ø   s€   € € € € € Ø €Màð ð  ð  ñ „\ð ð
 ð)ØŒxÔ#ð)Ø-5°c¬]ð)ØFIð)à	ð)ð )ð )ñ „\ð)ð )ð )r"   rž   rœ   )r6   Úcompiler_fnTr£   .r:   Úcopy_outputsÚcopy_inputsc                 ó  ‡‡‡‡‡— t          |t          t          f¦  «        st          dt	          |¦  «        › �¦  «        ‚‰rd„ |D ¦   «         Šnt          |¦  «        Št
          j                             ¦   «          t
          j                             ¦   «         }| 	                    t
          j         
                    ¦   «         ¦  «         t
          j                             |¦  «        5   | |Ž  ddd¦  «         n# 1 swxY w Y   |                     ¦   «          t
          j         
                    ¦   «          	                    |¦  «         t
          j                             ¦   «          t
          j                             ¦   «         Št
          j                             ‰|¬¦  «        5   | ‰Ž Šddd¦  «         n# 1 swxY w Y   t          ‰t          t          f¦  «        s‰fŠdt          dt          t                   fˆˆˆˆˆfd„}|S )zBThis isn't registered as a backend, but is used in some benchmarksz+Expected inputs to be a list or tuple, got c                 ó6   — g | ]}t          j        |¦  «        ‘ŒS r   )r,   Ú
zeros_like©ri   Úxs     r    rj   z$cudagraphs_inner.<locals>.<listcomp>  s#   € Ð=Ð=Ð=°�Ô)¨!Ñ,Ô,Ð=Ð=Ð=r"   N)ÚstreamÚ
new_inputsr   c                  óD  •— t          ‰¦  «        t          | ¦  «        k    r/t          dt          ‰¦  «        › dt          | ¦  «        › �¦  «        ‚‰r+t          ‰| ¦  «        D ]\  }}|                     |¦  «         Œ‰                     ¦   «          ‰rd„ ‰D ¦   «         S ‰S )Nz	Expected z inputs, got c                 ó6   — g | ]}|                      ¦   «         ‘ŒS r   )Úcloner³   s     r    rj   z1cudagraphs_inner.<locals>.run.<locals>.<listcomp>:  s    € Ð6Ð6Ð6 !�A—G’G‘I”IÐ6Ð6Ð6r"   )r4   r`   ÚzipÚcopy_Úreplay)r¶   ÚdstÚsrcr¯   r®   rF   Ústatic_inputsÚstatic_outputss      €€€€€r    Úrunzcudagraphs_inner.<locals>.run0  s¶   ø€ Ýˆ}ÑÔ¥ Z¡¤Ò0Ð0Ý ØN�C Ñ.Ô.ÐNÐN½SÀ¹_¼_ÐNÐNñô ð ð ð 	Ý ¨zÑ:Ô:ð ð ‘��SØ—	’	˜#‘”��Ø�Š‰ŒˆØð 	"Ø6Ð6 ~Ð6Ñ6Ô6Ð6à!Ð!r"   )r+   r�   Útupler`   r_   r,   r\   ÚsynchronizeÚStreamÚwait_streamÚcurrent_streamrµ   Ú	CUDAGraphrF   r   r   )	r£   r:   r®   r¯   rµ   rÁ   rF   r¿   rÀ   s	     ``  @@@r    Úcudagraphs_innerrÈ     so  øøøøø€ õ �f�t¥U˜mÑ,Ô,ð 
ÝØH½$¸v¹,¼,ÐHÐHñ
ô 
ð 	
ð ð %Ø=Ð=°fÐ=Ñ=Ô=ˆˆå˜V™œˆõ 
„J×ÒÑÔÐÝŒZ×ÒÑ Ô €FØ
×Ò•u”z×0Ò0Ñ2Ô2Ñ3Ô3Ð3Ý	Œ×	Ò	˜6Ñ	"Ô	"ð ð Øˆˆvˆˆðð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð à
×ÒÑÔÐÝ	„J×ÒÑÔ×+Ò+¨FÑ3Ô3Ð3Ý	„J×ÒÑÔÐõ ŒJ× Ò Ñ"Ô"€EÝ	Œ×	Ò	˜%¨Ð	Ñ	/Ô	/ð /ð /Ø˜ Ð.ˆð/ð /ð /ñ /ô /ð /ð /ð /ð /ð /ð /øøøð /ð /ð /ð /å�n¥t­U mÑ4Ô4ð +Ø(Ð*ˆð"�ð "¥­#¤ð "ð "ð "ð "ð "ð "ð "ð "ð "ð "ð €Js$   Ã+C=Ã=DÄDÆ*F<Æ<G ÇG )TT):Ú__doc__r—   ÚloggingÚcollectionsr   Úcollections.abcr   r   Útypingr   r,   Útorch.fxÚtorch._dynamor   Útorch._dynamo.backends.commonr   Ú torch._dynamo.backends.debuggingr	   Útorch._inductor.cudagraph_utilsr
   r   r   r   r   r   Útorch._inductor.utilsr   r   r   r   r   Ú torch.multiprocessing.reductionsr   Úregistryr   Ú	getLoggerr¨   r¥   rf   ÚGraphr(   ÚintrC   r•   r&   rH   rg   rK   r'   rS   rZ   rb   r�   rl   rœ   rž   r–   rÈ   r   r"   r    ú<module>rÙ      sœ  ððð ð. Ð Ð Ð Ø €€€Ø #Ð #Ð #Ð #Ð #Ð #Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø Ð Ð Ð Ð Ð à €€€Ø €€€Ø  Ð  Ð  Ð  Ð  Ð  Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ðð ð ð ð ð ð ð ð ð ð ð ð ð ð <Ð ;Ð ;Ð ;Ð ;Ð ;à &Ð &Ð &Ð &Ð &Ð &ð €gÔ˜Ñ!Ô!€ð$˜EœHœNð $¨s°3¬xð $ð $ð $ð $ðNØŒÔðà	ˆ%Œ,˜œœÐ
%Ô&ðð ð ð ðDØŒxÔ#ðDØ03ðDàˆ4�ZðDð Dð Dð Dð˜eœhÔ2ð ¸sð ÀsÈTÁzð ð ð ð ð$˜œÔ-ð °#ð ð ð ð ð˜œÔ-ð °$°s¸T±zÔ2Bð ð ð ð ðV7˜UœXÔ1ð V7À(È3Ä-ð V7ÐTWð V7ð V7ð V7ð V7ðr)ð )ð )ð )ð )ñ )ô )ð )ð( Ð �lÐ0AÐ0AÑ0CÔ0CÐ DÑ DÔ DÐ Dð Øð	/ð /Ø�C˜�HÔð/à�SŒMð/ð ð/ð ð	/ð
 ˆc�8˜C”=Ð Ô!ð/ð /ð /ð /ð /ð /r"   