§
    ŠŠtj¸Ç  ã                   ón  — d dl Z d dlZd dlZd dlmZ d dlmZ d dlmZ d dl	m
Z
mZmZ d dlmZ d dlmZ d dlZd dlmZ d dlmZ d d	lmZ d d
lmZmZmZmZ d dlmZ d dlm Z  ddl!m"Z"m#Z# ddl$m%Z% ddl&m'Z' ddl(m)Z)  ej*        e+¦  «        Z,dZ-de.fd„Z/d„ Z0dej1        j2        dej3        fd„Z4dej3        dej5        dej5        fd„Z6dej3        fd„Z7 G d„ dej1        j8        ¦  «        Z9 G d „ d!ej1        j2        ¦  «        Z: G d"„ d#e:¦  «        Z;ej<         =                    d$d%¦  «         ej<         >                    d$d&¦  «        d'„ ¦   «         Z?ej<         @                    d$¦  «        d(„ ¦   «         Z?ejA        jB        j?        jC        ZDejE        jF         G                    eD¦  «         d)„ ZH G d*„ d+e¦  «        ZIeIeJeKeIf         z  ZL G d,„ d-ejM        ¦  «        ZN G d.„ d/¦  «        ZO G d0„ d1¦  «        ZPd2„ ZQd3„ ZRd4ej        jS        d5ejT        fd6„ZU G d7„ d8ej1        j2        ¦  «        ZV G d9„ d:e¦  «        ZW G d;„ d<¦  «        ZXd=„ ZYd>„ ZZdej1        j2        d?eJeKeWf         fd@„Z[	 	 	 dIdAej1        j2        dBe\edCf         dDeJeKef         dz  dEeJeKeWf         dz  dFeejS        gejS        f         dz  dGeVfdH„Z]dS )Jé    N)Údefaultdict)ÚCallable)ÚEnum)Ú	ParameterÚ	SignatureÚ	signature)Ú
MethodType)ÚAny)ÚProcessGroup)ÚExportedProgram)Ú_assign_attrÚ	_AttrKindÚ_sink_paramsÚInterpreterModule)Úmap_aggregate)Úsplit_moduleé   )Ú_null_coalesce_accumulateÚstage_backward)Ú_outline_submodules)ÚPipeInfo)Ú_PipelineStageÚ	submod_ppÚ	stage_idxc                 óT   — d                      t          t          | ¦  «        g¦  «        S )zpReturns the name of the submod for a given stage index.
    For example, "submod_pp_0", "submod_pp_1", etc.
    Ú_)ÚjoinÚPP_SUBMOD_PREFIXÚstr)r   s    ú^/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/torch/distributed/pipelining/_IR.pyÚget_submod_namer!   *   s"   € ð �8Š8Õ%¥s¨9¡~¤~Ð6Ñ7Ô7Ð7ó    c                 óÔ  — |du rd S |du r.t          | t          j        ¦  «        st          d| › �¦  «        ‚| S t          |t          t
          f¦  «        r¤t          | t          t
          f¦  «        st          d| › d|› �¦  «        ‚t          | ¦  «        t          |¦  «        k    rt          d| › d|› �¦  «        ‚t          | |¦  «        D ]\  }}t          ||¦  «        }|�|c S Œt          d|› �¦  «        ‚t          |t          ¦  «        r¼t          | t          ¦  «        st          d| › d|› �¦  «        ‚t          |                      ¦   «         ¦  «        t          |                     ¦   «         ¦  «        k    rt          d| › d|› �¦  «        ‚|D ]$}t          | |         ||         ¦  «        }|�|c S Œ%t          d|› �¦  «        ‚t          d	t          |¦  «        › d
�¦  «        ‚)NFTz/Loss spec must specify a dynamic value but got zOutput value z' must match type of loss specification z) must match length of loss specification z)Did not find loss value in specification z' must match keys of loss specification zUnsupported type z in loss specification)Ú
isinstanceÚfxÚNodeÚRuntimeErrorÚtupleÚlistÚlenÚzipÚ_find_loss_from_output_and_specÚdictÚsetÚkeysÚtype)Ú
output_valÚspec_valÚoutÚspecÚloss_valÚks         r    r,   r,   1   s|  € Ø�5ÐÐØˆtØ�4ÐÐÝ˜*¥b¤gÑ.Ô.ð 	ÝØNÀ*ÐNÐNñô ð ð Ðå�(�U¥D˜MÑ*Ô*ð SÝ˜*¥u­d mÑ4Ô4ð 	Ýð 
ð ð Øðð ñô ð õ ˆz‰?Œ?�c (™mœmÒ+Ð+Ýð 
ð ð Øðð ñô ð õ ˜Z¨Ñ2Ô2ð 	 ð 	 ‰IˆC�Ý6°s¸DÑAÔAˆHØÐ#Ø���ð $åÐQÀxÐQÐQÑRÔRÐRå�(�DÑ!Ô!ð SÝ˜*¥dÑ+Ô+ð 	Ýð 
ð ð Øðð ñô ð õ ˆz�ŠÑ Ô Ñ!Ô!¥S¨¯ª©¬Ñ%9Ô%9Ò9Ð9Ýð 
ð ð Øðð ñô ð ð ð 	 ð 	 ˆAÝ6°zÀ!´}ÀhÈqÄkÑRÔRˆHØÐ#Ø���ð $åÐQÀxÐQÐQÑRÔRÐRå
ÐQ­4°©>¬>ÐQÐQÐQÑ
RÔ
RÐRr"   ÚmodÚgc                 ó  — d„ |j         D ¦   «         }t          |¦  «        dk    st          dt          |¦  «        › �¦  «        ‚|d         }|j        d         }d }t	          | t
          ¦  «        rKt          |j        ¦  «        dk    s$t          dt          |j        ¦  «        › �¦  «        ‚|}t
          j        }nG|€3t	          |t          ¦  «        rd|v r|d         }d„ |D ¦   «         }nd }d }nt          ||¦  «        }|}|||fS )Nc                 ó(   — g | ]}|j         d k    ¯|‘ŒS )Úoutput©Úop)Ú.0Úns     r    ú
<listcomp>z%_find_loss_output.<locals>.<listcomp>a   s$   € Ð;Ð;Ð;˜!¨!¬$°(Ò*:Ð*:�AÐ*:Ð*:Ð*:r"   r   zExpected 1 output node, got r   zExpected 1 output arg, got Úlossc                 ó   — i | ]	}||d k    “Œ
S )rA   © )r>   r6   s     r    ú
<dictcomp>z%_find_loss_output.<locals>.<dictcomp>s   s   € ÐAÐAÐA°˜a  f¢ÐAÐAÐAr"   )	Únodesr*   ÚAssertionErrorÚargsr$   ÚTrivialLossWrapperÚ	loss_specr-   r,   )r7   r8   Úoutput_loss_value_specÚoutput_nodesÚoutput_noder1   Úgenerated_specÚ	loss_nodes           r    Ú_find_loss_outputrO   `   s/  € Ø;Ð;˜qœwÐ;Ñ;Ô;€LÝˆ|ÑÔ Ò!Ð!ÝÐO½CÀÑ<MÔ<MÐOÐOÑPÔPÐPØ˜q”/€KØÔ! !Ô$€JØ€Nå�#Õ)Ñ*Ô*ð 0õ �;Ô#Ñ$Ô$¨Ò)Ð)Ý Ð!V½sÀ;ÔCSÑ?TÔ?TÐ!VÐ!VÑWÔWÐWØˆ	Ý+Ô5ˆˆØ	Ð	'å�j¥$Ñ'Ô'ð 	"¨F°jÐ,@Ð,@Ø" 6Ô*ˆIØAÐA°jÐAÑAÔAˆNˆNàˆIØ!ˆNˆNå3°JÐ@VÑWÔWˆ	Ø/ˆà�k >Ð1Ð1r"   rN   rL   c           
      óî  ‡ ‡‡— i }t          ‰ j        ¦  «        D ]ñ}|j        dk    rä|j        t          j        k    st          d¦  «        ‚t          |j        ¦  «        dk    st          d¦  «        ‚t          |j        ¦  «        \  }}||v rt          ||         ¦  «        nd}t          |dz   |¦  «        }d„ t          |¦  «        D ¦   «         }	||v r t          ||         ¦  «        D ]
\  }
}||	|
<   Œ||	|<   t          |	¦  «        ||<   Œò|d iŠ|d iŠˆ ˆfd„}‰                      |¦  «        5  t          ‰ j        ¦  «        D �]—}|‰vrŒˆfd	„}t          j                             |j        |¦  «         t          j                             |j        |¦  «         |j        d
k    �r4||v rK||         }t          ˆfd„||         D ¦   «         ¦  «        }ˆfd„t          ||         ¦  «        D ¦   «         }n|f}‰|         }dg}t%          |t          ¦  «        s|fn|}‰                      t(          ||t+          |j        ¦  «        |dœ¬¦  «        }t/          |j        ¦  «        }||_        t          j        |¦  «        }|j        }t+          |j        ¦  «        }t          j        |¦  «        }t          |¦  «        D ]\  }
} ||||
         j        ¦  «         Œ�Œ™	 d d d ¦  «         n# 1 swxY w Y   ‰ S )NÚcall_functionzDFound non-getitem call in forward pass. Please report a bug to PiPPyé   z:Found malformed getitem call. Please report a bug to PiPPyéÿÿÿÿr   c                 ó   — g | ]}d ‘ŒS ©NrC   )r>   r   s     r    r@   z3_insert_stage_symbolic_backward.<locals>.<listcomp>œ   s   € Ð!EÐ!EÐ!E¨1 $Ð!EÐ!EÐ!Er"   c                 óv   •— | ‰v r.| j         dk    r#‰                     t          ‰|          |f¦  «        }|‰| <   d S )NÚplaceholder)r=   rQ   r   )Úforward_nodeÚ
grad_valuer8   Úval_to_grads     €€r    Úassign_or_accumulate_gradzB_insert_stage_symbolic_backward.<locals>.assign_or_accumulate_grad®   sO   ø€ Ø˜;Ð&Ð&¨<¬?¸mÒ+KÐ+KØŸšÝ)Ø˜\Ô*¨JÐ7ñô ˆJð %/ˆ�LÑ!Ð!Ð!r"   c                 ó4   •— ‰                      | d ¦  «         d S rU   )Ú
setdefault)r?   Ú
live_nodess    €r    Úadd_to_live_nodesz:_insert_stage_symbolic_backward.<locals>.add_to_live_nodes»   s   ø€ Ø×%Ò% a¨Ñ.Ô.Ð.Ð.Ð.r"   Úcall_modulec              3   óB   •K  — | ]}‰                      |¦  «        V — Œd S rU   )Úget)r>   r?   rZ   s     €r    ú	<genexpr>z2_insert_stage_symbolic_backward.<locals>.<genexpr>Ä   s/   øè è € Ð(RÐ(RÀ¨¯ª¸Ñ);Ô);Ð(RÐ(RÐ(RÐ(RÐ(RÐ(Rr"   c                 ó"   •— g | ]\  }}|‰v ¯	|‘ŒS rC   rC   )r>   Úir?   r^   s      €r    r@   z3_insert_stage_symbolic_backward.<locals>.<listcomp>Å   s)   ø€ ð /ð /ð /Ù"˜a ÀÀjÀÀ˜ÀÀÀr"   r   )Ústage_outputÚoutput_gradsÚinput_valuesÚoutputs_with_grads_idxs)Úkwargs)ÚreversedrE   r=   ÚtargetÚoperatorÚgetitemrF   r*   rG   r(   ÚmaxÚrangeÚ	enumerateÚinserting_beforer%   ÚnodeÚmap_argrj   r$   rQ   r   r)   Úall_input_nodesr-   ÚProxy)r8   rN   rL   Útuplesrs   Úindexed_valueÚnode_idxÚexisting_list_sizeÚnew_list_sizeÚreconstructed_listre   Úvalr[   r_   rf   rg   ri   Ú	grad_callÚkwargs_copyÚgrad_call_proxyÚgradsÚinput_nodesÚgrads_proxyÚ
input_noder^   rZ   s   `                       @@r    Ú_insert_stage_symbolic_backwardr…   ~   s  øøø€ ð $&€FÝ˜œÑ!Ô!ð !>ð !>ˆØŒ7�oÒ%Ð%ð ”;¥(Ô"2Ò2Ð2Ý$ØZñô ð õ �t”y‘>”> QÒ&Ð&Ý$ØPñô ð õ ',¨D¬IÑ&6Ô&6Ñ#ˆM˜8ð /<¸vÐ.EÐ.E•�F˜=Ô)Ñ*Ô*Ð*È2ð õ   ¨1¡Ð.@ÑAÔAˆMà!EÐ!Eµ°mÑ0DÔ0DÐ!EÑ!EÔ!EÐð  Ð&Ð&Ý'¨¨}Ô(=Ñ>Ô>ð 0ð 0‘F�A�sØ,/Ð& qÑ)Ð)ð ,0Ð˜xÑ(å$)Ð*<Ñ$=Ô$=ˆF�=Ñ!øð
 ˜TÐ"€JØ2;¸TÐ1B€Kð/ð /ð /ð /ð /ð /ð 
×	Ò	˜KÑ	(Ô	(ð 0Oð 0OÝ˜QœWÑ%Ô%ð /	Oñ /	OˆDØ˜:Ð%Ð%Øð/ð /ð /ð /ð /õ ŒG�OŠO˜DœIÐ'8Ñ9Ô9Ð9ÝŒG�OŠO˜DœKÐ):Ñ;Ô;Ð;ØŒw˜-Ò'Ñ'à˜6�>�>Ø#)¨$¤<�LÝ#(Ð(RÐ(RÐ(RÐ(RÀVÈDÄ\Ð(RÑ(RÔ(RÑ#RÔ#R�Lð/ð /ð /ð /Ý&/°°t´Ñ&=Ô&=ð/ñ /ô /Ð+Ð+ð %) 7�LØ#.¨tÔ#4�LØ/0¨cÐ+õ & lµEÑ:Ô:ð&�\�O�Oà%ð ð ŸOšOÝ"à(4Ø(4Ý(,¨TÔ-AÑ(BÔ(BØ3Jð	ð ð ,ñ ô �	õ # 9Ô#3Ñ4Ô4�Ø#.�	Ô å"$¤(¨9Ñ"5Ô"5�Ø'Ô,�å" 4Ô#7Ñ8Ô8�Ý œh u™oœo�Ý%.¨{Ñ%;Ô%;ð Oð O‘M�A�zØ-Ð-¨j¸+Àa¼.Ô:MÑNÔNÐNÐNùð_/	Oð0Oð 0Oð 0Oñ 0Oô 0Oð 0Oð 0Oð 0Oð 0Oð 0Oð 0Oøøøð 0Oð 0Oð 0Oð 0Oðd €Hs   Ä/F.K*Ë*K.Ë1K.Úgraphc                 óà   — d}| j         D ]}|j        dk    r|}Œ|€dS g }| j         D ](}||u r n!|j        dk    r|                     |¦  «         Œ)|}|D ]}|                     |¦  «         |}ŒdS )z…Ensure all placeholder nodes form a contiguous prefix of the graph.
    split_module can interleave get_attr nodes with placeholders.NrW   )rE   r=   Úappend)r†   Úlast_phrs   Únodes_to_moveÚinsert_points        r    Ú_move_placeholders_to_frontrŒ   ë   s¶   € ð €GØ”ð ð ˆØŒ7�mÒ#Ð#ØˆGøØ€ØˆØ€MØ”ð 'ð 'ˆØ�7ˆ?ˆ?ØˆEØŒ7�mÒ#Ð#Ø× Ò  Ñ&Ô&Ð&øØ€LØð ð ˆØ×Ò˜DÑ!Ô!Ð!Øˆˆðð r"   c                   óD   — e Zd Zedej        j        fd„¦   «         Zd„ ZdS )ÚPipeSequentialÚsequential_instancec                 ó(   — t          d„ | D ¦   «         Ž S )Nc                 ó6   — g | ]}t          j         |¦  «        ‘ŒS rC   )Úcopy)r>   Úms     r    r@   z2PipeSequential.from_sequential.<locals>.<listcomp>  s    € ÐJÐJÐJ°¥¤	¨!¡¤ÐJÐJÐJr"   )rŽ   )r�   s    r    Úfrom_sequentialzPipeSequential.from_sequential  s   € åÐJÐJÐ6IÐJÑJÔJÐKÐKr"   c                 óŽ   — t          | ¦  «        D ]4\  }} ||¦  «        }|t          | ¦  «        dz
  k    rt          ¦   «          Œ5|S )Nr   )rq   r*   Ú
pipe_split)ÚselfÚinputre   Úmodules       r    ÚforwardzPipeSequential.forward  sL   € Ý" 4™œð 	ð 	‰IˆAˆvØ�F˜5‘M”MˆEØ•C˜‘I”I ‘MÒ!Ð!Ý‘”�øØˆr"   N)	Ú__name__Ú
__module__Ú__qualname__ÚstaticmethodÚtorchÚnnÚ
Sequentialr”   rš   rC   r"   r    rŽ   rŽ      sR   € € € € € ØðL¨U¬XÔ-@ð Lð Lð Lñ „\ðLðð ð ð ð r"   rŽ   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚLossWrapperaJ  
    LossWrapper is a convenient abstract class that allows you to wrap up both
    your model as well as its loss function and specify the connectivity between
    the inputs, model, loss function, and output value. Example::

        class MyModelWrapper(LossWrapper):
            def forward(self, x, targets):
                model_out = self.module(x)
                loss_value = self.loss_fn(model_out, targets)
                return loss_value

    The above example defines a connectivity where we expect the forward/loss/backward
    training procedure to take two arguments (x and targets), pass x into the module
    to get the output of the feedforward computation, pass the model output and the
    targets value into the loss function, and get and return the loss value, which will
    be backpropagated by PiPPy. The above class would then be instantiated like::

        model = ...  # instantiate the model
        loss_fn = torch.nn.MSELoss()  # for the sake of demonstration

        wrapper = MyModelWrapper(model, loss_fn)
        pipe = Pipe.from_tracing(wrapper, ...)

    c                 ód   •— t          ¦   «                              ¦   «          || _        || _        d S rU   )ÚsuperÚ__init__r™   Úloss_fn)r—   r™   r§   Ú	__class__s      €r    r¦   zLossWrapper.__init__'  s+   ø€ Ý‰Œ×ÒÑÔÐØˆŒØˆŒˆˆr"   c                 ó    — t          d¦  «        ‚)Nz¶This instance of LossWrapper does not have an overriddenforward(). Please implement forward() to specify the arguments, connection between the module and loss, and loss output value.)ÚNotImplementedError©r—   rG   rj   s      r    rš   zLossWrapper.forward,  s   € Ý!ðñ
ô 
ð 	
r"   )r›   rœ   r�   Ú__doc__r¦   rš   Ú__classcell__©r¨   s   @r    r£   r£     sQ   ø€ € € € € ðð ð2ð ð ð ð ð

ð 
ð 
ð 
ð 
ð 
ð 
r"   r£   c                   ó   — e Zd Zd„ ZdZdS )rH   c                 óX   — |                       |¦  «        }|                      ||¦  «        S rU   )r™   r§   )r—   ÚxÚtargetsÚ	model_outs       r    rš   zTrivialLossWrapper.forward7  s%   € Ø—K’K ‘N”Nˆ	Ø�|Š|˜I wÑ/Ô/Ð/r"   TN)r›   rœ   r�   rš   rI   rC   r"   r    rH   rH   5  s$   € € € € € ð0ð 0ð 0ð €I€I€Ir"   rH   zpippy::_pipe_splitz() -> ()ÚBackendSelectc                  ó   — d S rU   rC   rC   r"   r    Ú_pipe_splitr¶   X  ó   € àˆ4r"   c                  ó   — d S rU   rC   rC   r"   r    r¶   r¶   ]  r·   r"   c                  óH   — t           j        j                             ¦   «         S )aþ  
    pipe_split is a special operator that is used to mark the boundary between
    stages in a module. It is used to split the module into stages. It is a
    no-op if your annotated module is run eagerly.

    Example:
        >>> # xdoctest: +SKIP
        >>> def forward(self, x):
        >>>     x = torch.mm(x, self.mm_param)
        >>>     x = torch.relu(x)
        >>>     pipe_split()
        >>>     x = self.lin(x)
        >>>     return x

    The above example will be split into two stages.
    )rŸ   ÚopsÚpippyr¶   rC   r"   r    r–   r–   k  s   € õ" Œ9Œ?×&Ò&Ñ(Ô(Ð(r"   c                   ó   — e Zd ZdZdZdS )ÚMultiUseParameterConfigr   rR   N)r›   rœ   r�   ÚTRANSMITÚ	REPLICATErC   r"   r    r½   r½     s   € € € € € Ø€HØ€I€I€Ir"   r½   c                   óH   ‡ — e Zd ZdZd	ˆ fd„	Zddœˆ fd„
Zˆ fd„Zˆ fd„Zˆ xZS )
ÚDetachExecutorzÏ
    Special interpreter to run the split_gm in testing that detaches all inputs to
    a module invocation. This is needed so that the values at the boundary are
    leaf modules in autograd execution.
    Tc                 ó^   •— d}t          ¦   «                              ||¦  «         i | _        d S )NF)r¥   r¦   Úvalue_remap)r—   r™   Úgarbage_collect_valuesr¨   s      €r    r¦   zDetachExecutor.__init__Ž  s2   ø€ Ø!&ÐÝ‰Œ×Ò˜Ð!7Ñ8Ô8Ð8ØˆÔÐÐr"   N)Úinitial_envc                óD   •— i | _          t          ¦   «         j        |d|iŽS )NrÅ   )rÃ   r¥   Úrun)r—   rÅ   rG   r¨   s      €r    rÇ   zDetachExecutor.run“  s&   ø€ ØˆÔØ�u‰wŒwŒ{˜DÐ:¨kÐ:Ð:Ð:r"   c                 ó˜   •‡ — ˆ fd„}	 t          ||¦  «        }t          ||¦  «        }t          ¦   «                              |||¦  «        S )Nc                 óØ   •— t          | t          j        ¦  «        rN| j        rG| ‰j        vr1|                      ¦   «                              d¦  «        }|‰j        | <   ‰j        |          S | S )NT)r$   rŸ   ÚTensorÚrequires_gradrÃ   ÚdetachÚrequires_grad_)ÚaÚnew_valr—   s     €r    Údetach_tensorsz2DetachExecutor.call_module.<locals>.detach_tensors˜  si   ø€ Ý˜!�Uœ\Ñ*Ô*ð ¨q¬ð Ø˜DÔ,Ð,Ð,ØŸhšh™jœj×7Ò7¸Ñ=Ô=�GØ*1�DÔ$ QÑ'ØÔ'¨Ô*Ð*à�r"   )r   r¥   r`   )r—   rl   rG   rj   rÐ   r¨   s   `    €r    r`   zDetachExecutor.call_module—  sn   øø€ ð	ð 	ð 	ð 	ð 	ð	õ
 ØØñ
ô 
ˆõ ØØñ
ô 
ˆõ
 ‰wŒw×"Ò" 6¨4°Ñ8Ô8Ð8r"   c                 óª   •‡ — |t           u r&t          |¦  «        }ˆ fd„|d         D ¦   «         |d<   t          ¦   «                              |||¦  «        S )Nc                 óF   •— g | ]}‰j                              ||¦  «        ‘ŒS rC   )rÃ   rb   )r>   Úvr—   s     €r    r@   z0DetachExecutor.call_function.<locals>.<listcomp>µ  s9   ø€ ð &ð &ð &Ø/0�Ô ×$Ò$ Q¨Ñ*Ô*ð&ð &ð &r"   rh   )r   r-   r¥   rQ   )r—   rl   rG   rj   r¨   s   `   €r    rQ   zDetachExecutor.call_function±  sj   øø€ à•^Ð#Ð#Ý˜&‘\”\ˆFð&ð &ð &ð &Ø4:¸>Ô4Jð&ñ &ô &ˆF�>Ñ"õ ‰wŒw×$Ò$ V¨T°6Ñ:Ô:Ð:r"   )T)	r›   rœ   r�   r¬   r¦   rÇ   r`   rQ   r­   r®   s   @r    rÁ   rÁ   ‡  s¡   ø€ € € € € ðð ðð ð ð ð ð ð
 &*ð ;ð ;ð ;ð ;ð ;ð ;ð ;ð9ð 9ð 9ð 9ð 9ð4;ð ;ð ;ð ;ð ;ð ;ð ;ð ;ð ;r"   rÁ   c                   ó    — e Zd ZU d„ Zeed<   dS )Ú_NodeReferencec                 ó   — || _         d S rU   )Úname)r—   r×   s     r    r¦   z_NodeReference.__init__¼  s   € ØˆŒ	ˆ	ˆ	r"   r×   N)r›   rœ   r�   r¦   r   Ú__annotations__rC   r"   r    rÕ   rÕ   »  s*   € € € € € € ðð ð ð €I€I�I€I€Ir"   rÕ   c                   ó   — e Zd Zd„ Zd„ ZdS )Ú_LinearNodeListc           
      ó„  — g | _         |D ]µ}t          j                             |j        d„ ¦  «        }t          j                             |j        d„ ¦  «        }t          j        d |j        |j        |j	        |||j
        ¬¦  «        }t          j        |j        ¦  «        |_        | j                              |¦  «         Œ¶d S )Nc                 ó*   — t          | j        ¦  «        S rU   ©rÕ   r×   ©r?   s    r    ú<lambda>z*_LinearNodeList.__init__.<locals>.<lambda>Æ  s   € ½^ÈAÌFÑ=SÔ=S€ r"   c                 ó*   — t          | j        ¦  «        S rU   rÝ   rÞ   s    r    rß   z*_LinearNodeList.__init__.<locals>.<lambda>Ç  s   € ÅÐPQÔPVÑAWÔAW€ r"   )r†   r×   r=   rl   rG   rj   Úreturn_type)Úserialize_node_listr%   rs   rt   rG   rj   r&   r×   r=   rl   r0   r’   Úmetarˆ   )r—   Ú	node_listrs   Ú	node_argsÚnode_kwargsÚserialize_nodes         r    r¦   z_LinearNodeList.__init__Ã  s¼   € Ø#%ˆÔ Øð 	<ð 	<ˆDÝœŸš¨¬	Ð3SÐ3SÑTÔTˆIÝœ'Ÿ/š/¨$¬+Ð7WÐ7WÑXÔXˆKÝœWØØ”YØ”7Ø”{ØØ"Ø œIðñ ô ˆNõ #'¤)¨D¬IÑ"6Ô"6ˆNÔØÔ$×+Ò+¨NÑ;Ô;Ð;Ð;ð	<ð 	<r"   c           	      ó  ‡— t          j        ¦   «         }i Šˆfd„}| j        D ]e}t          |j        |¦  «        }t          |j        |¦  «        }|                     |j        |j        |||j	        |j
        ¬¦  «        }|‰|j	        <   Œf|S )Nc                 óL   •— t          | t          ¦  «        r‰| j                 S | S rU   )r$   rÕ   r×   )ÚargÚref_str_to_nodes    €r    Úref_to_nodez-_LinearNodeList.to_graph.<locals>.ref_to_nodeÙ  s'   ø€ Ý˜#�~Ñ.Ô.ð Ø& s¤xÔ0Ð0à�
r"   )r=   rl   rG   rj   r×   Ú	type_expr)r%   ÚGraphrâ   r   rG   rj   Úcreate_noder=   rl   r×   r0   )r—   r†   rì   rs   rå   ræ   Ú
deser_noderë   s          @r    Úto_graphz_LinearNodeList.to_graphÔ  s¬   ø€ Ý”‘
”
ˆà.0ˆð	ð 	ð 	ð 	ð 	ð Ô,ð 	4ð 	4ˆDÝ% d¤i°Ñ=Ô=ˆIÝ'¨¬°[ÑAÔAˆKØ×*Ò*Ø”7Ø”{ØØ"Ø”YØœ)ð +ñ ô ˆJð *4ˆO˜DœIÑ&Ð&àˆr"   N)r›   rœ   r�   r¦   rñ   rC   r"   r    rÚ   rÚ   Â  s2   € € € € € ð<ð <ð <ð"ð ð ð ð r"   rÚ   c                 ó    —  G d„ dt           j        j        ¦  «        } || ¦  «        }t          j        ||                     ¦   «         ¦  «        S )aË  
    Custom `__reduce__` method for serialization.
    DO AS I SAY -- NOT AS I DO. This violates the principle that
    GraphModules serialize via code export & re-tracing. We allow
    for this here because **PIPE STAGES SHOULD NOT BE PERSISTED
    TO DISK -- THIS IS ONLY FOR TRANSMISSION VIA RPC**. Persisting
    these instances to disk will expose internal implementation
    details of `fx.Graph` and related data structures and is
    NOT advised.
    c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ú6_direct_serialization_deserialize.<locals>.DummyModulec                 ó|   •— t          ¦   «                              ¦   «          | j                             |¦  «         d S rU   )r¥   r¦   Ú__dict__Úupdate)r—   Úbodyr¨   s     €r    r¦   z?_direct_serialization_deserialize.<locals>.DummyModule.__init__ü  s5   ø€ Ý‰GŒG×ÒÑÔÐØŒM× Ò  Ñ&Ô&Ð&Ð&Ð&r"   )r›   rœ   r�   r¦   r­   r®   s   @r    ÚDummyModulerô   û  s8   ø€ € € € € ð	'ð 	'ð 	'ð 	'ð 	'ð 	'ð 	'ð 	'ð 	'r"   rù   )rŸ   r    ÚModuler%   ÚGraphModulerñ   )rø   rE   rù   Údummys       r    Ú!_direct_serialization_deserializerý   ï  s]   € ð'ð 'ð 'ð 'ð '•e”h”oñ 'ô 'ð 'ð
 ˆK˜ÑÔ€EåŒ>˜% §¢Ñ!1Ô!1Ñ2Ô2Ð2r"   c                 ó˜   — t          | j        ¦  «        }|                     d¦  «         t          |t	          | j        j        ¦  «        ffS )NÚ_graph)r-   rö   Úpoprý   rÚ   r†   rE   )r—   Úserialization_dicts     r    Ú_direct_serialization_reducer    sH   € Ý˜dœmÑ,Ô,ÐØ×Ò˜8Ñ$Ô$Ð$å)Ø	�_¨T¬ZÔ-=Ñ>Ô>Ð?ðð r"   ÚgmÚ
new_devicec           	      ó¢  — d}| j         j        D �]&}|j        dk    rfd|j        v r\|j        d         |k    rKt                               d|j        › d|j        d         › d|› �¦  «         |                     d|¦  «         d}Œt|j        dk    r§|                      |j	        ¦  «        }t          |t          j        j        ¦  «        rt          ||¦  «         ŒÉt          |t          ¦  «        rt          |j        |¦  «         Œôt                               d	|j	        › d
t%          |¦  «        › �¦  «         �Œ(|r|                      ¦   «          dS dS )zÅ
    Modify the device argument of all "call_function" nodes in the graph.  This
    is useful for moving the graph to a different device. In particular for
    generator ops, like torch.ones.
    FrQ   ÚdevicezChanging device of Node z from ú to Tr`   z+Skipping device modification for submodule z because it is a N)r†   rE   r=   rj   ÚloggerÚdebugr×   Úupdate_kwargÚget_submodulerl   r$   rŸ   r%   rû   Ú_modify_graph_op_devicer   Úgraph_moduleÚwarningr0   Ú	recompile)r  r  Úmodifiedrs   Úsubmods        r    r  r    sp  € ð €HØ””ð ñ ˆØŒ7�oÒ%Ð%Ø˜4œ;Ð&Ð&¨4¬;°xÔ+@ÀJÒ+NÐ+NÝ—’Øg¨t¬yÐgÐgÀÄÈHÔ@UÐgÐgÐ[eÐgÐgñô ð ð ×!Ò! (¨JÑ7Ô7Ð7Ø�øØŒW˜Ò%Ð%à×%Ò% d¤kÑ2Ô2ˆFÝ˜&¥%¤(Ô"6Ñ7Ô7ð Ý'¨°
Ñ;Ô;Ð;Ð;Ý˜FÕ$5Ñ6Ô6ð å'¨Ô(;¸ZÑHÔHÐHÐHå—’ØnÀ$Ä+ÐnÐnÕ`dÐekÑ`lÔ`lÐnÐnñô ð ùð ð Ø
�Š‰Œˆˆˆðð r"   c                   óJ  — e Zd Zdej        dedefd„Zd„ Zdede	j
        j        fd„Zed	ej        fd
„¦   «         Ze	 	 	 dde	j
        j        dededz  dee	j        j        ge	j        j        f         dz  fd„¦   «         Zd„ Ze	 dde	j
        j        deedf         deeef         dz  defd„¦   «         Ze	 	 d de	j
        j        deedf         deeef         dz  deej        gej        f         dz  fd„¦   «         Zd„ Zd„ Zdefd„Z	 ddede	j        dedz  de fd„Z!dS )!ÚPipeÚsplit_gmÚ
num_stagesÚhas_loss_and_backwardc                 óÚ  — t           j        j                             | ¦  «         || _        t          | j        ¦  «        | _        || _        || _        || _	        |j
        j        D ]}|j        dv st|j        |j        fdt          j        fk    sV|j        |j        fdk    sD|j        |j        fdt           fk    s+|j        |j        fdt"          fk    st%          d|› �¦  «        ‚Œ€i }| j                             ¦   «         D ]@\  }}|                     ¦   «         D ]&\  }	}
|                     |
i ¦  «         |	||
         |<   Œ'ŒAd„ |                     ¦   «         D ¦   «         | _        | j        D ] }|                     ¦   «         D ]‰\  }}t3          | j        |¦  «        }|                     d¦  «        }|d d…         D ]}t3          ||¦  «        }Œt7          ||d         t9          j        t3          ||d         ¦  «        ¦  «        ¦  «         ŒŠŒ¡d„ }|| j        _        d	}	 	 t?          |¦  «        }t3          | j        |¦  «        }t@          |j!        _"        |dz  }n# tF          $ r Y d S w xY wŒN)N>   r;   r`   rW   rQ   )Úcall_methodÚbackwardzUnexpected node: c                 ó8   — g | ]}t          |¦  «        d k    ¯|‘ŒS )r   )r*   )r>   Úuse_mappings     r    r@   z!Pipe.__init__.<locals>.<listcomp>\  s5   € ð 8
ð 8
ð 8
àÝ�;ÑÔ !Ò#Ð#ð à#Ð#Ð#r"   ú.rS   c                 ó    — t          d¦  «        ‚)NzHTo run pipeline locally, invoke the Pipe object directly, not `split_gm`)r'   r«   s      r    ÚthrowzPipe.__init__.<locals>.throwo  s   € ÝØZñô ð r"   r   Tr   )$rŸ   r    rú   r¦   r  rÁ   Úexecutorr  r  rI   r†   rE   r=   rl   rm   rn   r   r   rF   Únamed_childrenÚnamed_parametersr]   ÚvaluesÚreplicated_paramsÚitemsÚgetattrÚsplitÚsetattrr’   Údeepcopyrš   r!   r  r¨   Ú
__reduce__ÚAttributeError)r—   r  r  r  rI   rs   Úparams_to_usersÚ
m_qualnamer7   Ú
p_qualnameÚparamÚparam_mappingÚsubmod_nameÚparam_qualnamer  ÚatomsÚatomr  re   r×   s                       r    r¦   zPipe.__init__2  sÞ  € õ 	ŒŒ× Ò  Ñ&Ô&Ð&Ø(0ˆŒÝ(6°t´}Ñ(EÔ(EˆŒØ)ˆŒØ%:ˆÔ"Ø"ˆŒà”NÔ(ð 		Að 		AˆDà”ÐCÐCÐCØ”G˜Tœ[Ð)¨o½xÔ?OÐ-PÒPÐPØ”G˜Tœ[Ð)Ð-HÒHÐHØ”G˜Tœ[Ð)¨o½~Ð-NÒNÐNØ”G˜Tœ[Ð)Ø#Õ%>Ð?ò@ð @õ %Ð%?¸Ð%?Ð%?Ñ@Ô@Ð@øð EGˆà#œ}×;Ò;Ñ=Ô=ð 	@ð 	@‰OˆJ˜Ø%(×%9Ò%9Ñ%;Ô%;ð @ð @Ñ!�
˜EØ×*Ò*¨5°"Ñ5Ô5Ð5Ø5?� Ô& zÑ2Ð2ð@ð8
ð 8
à.×5Ò5Ñ7Ô7ð8
ñ 8
ô 8
ˆÔð "Ô3ð 	Vð 	VˆMØ/<×/BÒ/BÑ/DÔ/Dð Vð VÑ+�˜^Ý  ¤°Ñ<Ô<�Ø&×,Ò,¨SÑ1Ô1�Ø! # 2 #œJð 3ð 3�DÝ$ V¨TÑ2Ô2�F�FÝ˜  b¤	­4¬=½ÀÈÈrÌÑ9SÔ9SÑ+TÔ+TÑUÔUÐUÐUðVð	ð 	ð 	ð
 !&ˆŒÔð ˆð	ðÝ& qÑ)Ô)�Ý  ¤°Ñ5Ô5�Ý.J�Ô Ô+Ø�Q‘��øÝ!ð ð ð Ø��ðøøøð	s   È :I É
I)É(I)c           	      ó  — |}t          |¦  «        dk    �rbg }| j        j        j        D �]}|j        dk    rö|j        rXt          |j        ¦  «        dk    r@|                     t          |j        t          j	        |j        d         ¬¦  «        ¦  «         Œmt          j	        }|j        }|j         
                    d¦  «        rt          j        }|dd …         }n0|j         
                    d¦  «        rt          j        }|dd …         }|                     t          ||¦  «        ¦  «         �Œt          |¦  «        } |j        |i |¤Ž}	|	                     ¦   «          |	j                             ¦   «         } | j        j        |Ž }
|
S )Nr   rW   )Údefaultz**rR   Ú*r   )r*   r  r†   rE   r=   rG   rˆ   r   rl   ÚPOSITIONAL_OR_KEYWORDÚ
startswithÚVAR_KEYWORDÚVAR_POSITIONALr   ÚbindÚapply_defaultsÚ	argumentsr"  r  rÇ   )r—   rG   rj   Úexecutor_argsÚ
parametersrs   Úparameter_kindÚ
param_namer   ÚbaÚress              r    rš   zPipe.forward�  s‹  € ØˆÝˆv‰;Œ;˜Š?‰?ØˆJØœÔ+Ô1ð Qñ Q�Ø”7˜mÒ+Ð+Ø”yð Q¥S¨¬¡^¤^°aÒ%7Ð%7Ø"×)Ò)Ý%Ø $¤Ý )Ô ?Ø(,¬	°!¬ðñ ô ñô ð ð õ *3Ô)H˜Ø%)¤[˜
Øœ;×1Ò1°$Ñ7Ô7ð 8Ý-6Ô-B˜NØ)3°A°B°B¬˜J˜JØ!œ[×3Ò3°CÑ8Ô8ð 8Ý-6Ô-E˜NØ)3°A°B°B¬˜JØ"×)Ò)­)°JÀÑ*OÔ*OÑPÔPÐPùÝ! *Ñ-Ô-ˆIØ�” Ð0¨Ð0Ð0ˆBØ×ÒÑÔÐØœL×/Ò/Ñ1Ô1ˆMàˆdŒmÔ Ð/ˆàˆ
r"   r   Úreturnc                 ó’   — |dk     s|| j         k    rt          d|› d�¦  «        ‚t          |¦  «        }t          | j        |¦  «        S )zS
        Return a stage module corresponding to `stage_idx` of the `pipe`.
        r   zInvalid stage index ú!)r  Ú
ValueErrorr!   r%  r  )r—   r   r0  s      r    Úget_stage_modulezPipe.get_stage_module¢  sR   € ð �qŠ=ˆ=˜I¨¬Ò8Ð8ÝÐ@°IÐ@Ð@Ð@ÑAÔAÐAå% iÑ0Ô0ˆÝ�t”} kÑ2Ô2Ð2r"   r  c                 ó<  — d}i }| j         j        D ]Š}|j        dk    r}|j                             t
          ¦  «        r^t          |j        t          t
          ¦  «        dz   d …         ¦  «        |j        d<   | 	                    |j        d         ¦  «         |dz  }Œ‹|S )Nr   r`   r   r   )
r†   rE   r=   rl   r8  r   Úintr*   rã   r]   )r  r  Ú
found_idxsrs   s       r    Ú _number_and_count_forward_stagesz%Pipe._number_and_count_forward_stages¬  sŸ   € àˆ
Ø&(ˆ
Ø”H”Nð 	 ð 	 ˆDØŒw˜-Ò'Ð'¨D¬K×,BÒ,BÕCSÑ,TÔ,TÐ'Ý),¨T¬[½Õ=MÑ9NÔ9NÐQRÑ9RÐ9TÐ9TÔ-UÑ)VÔ)V�”	˜+Ñ&Ø×%Ò% d¤i°Ô&<Ñ=Ô=Ð=Ø˜a‘�
øð Ðr"   Nr7   Úexported_programÚmulti_use_param_specÚsplit_policyc                 óª  ‡.‡/‡0‡1— |                      d¬¦  «        }|�%t                               d¦  «          ||¦  «        }t                               |                     d¬¦  «        ¦  «         i }|j        j        D ]s}|j        dk    rf|                     |j	        |¦  «         ||j	                 |k    r:| 
                    ||j	                 ¦  «         |j                             |¦  «         Œtd}t          ¦   «         }	t          |j        j        ¦  «        D ]>\  }
}|j        |j	        fdt          fk    r ||
d	z
  k    r|	                     |¦  «         |
}Œ?|	D ]}|j                             |¦  «         Œ|                     ¦   «          d
Š/dt"          j        fˆ/fd„}t'          || |d¬¦  «        Š0‰0j                             ¦   «          ‰0                     ¦   «         D ]r}t-          |t"          j        ¦  «        rV|j        j        D ]5}|j        |j	        fdt0          fk    r|j                             |¦  «         Œ6|                     ¦   «          Œs‰0                     ¦   «         D ]]\  }}t-          |t"          j        ¦  «        r>t5          |j        ¦  «         t7          |j        ¦  «        }‰0                     ||¦  «         Œ^d„ }g Š1d„ Š.ˆ.ˆ0ˆ1fd„}t;          t=          d„ ‰0j        j        ¦  «        ¦  «        }|D ]ˆ}t?          |j         ¦  «        d	k    r+t                               d|j	        › d|j         › d�¦  «         |j         D ];}|j        dk    stC          d|j        › �¦  «        ‚ |‰0|j	        |j	        ¦  «         Œ<Œ‰tE          t          ¦  «        }|  #                    d¬¦  «         $                    ¦   «         D ]-\  }}|tK          |¦  «                                      |¦  «         Œ.|  &                    ¦   «         D ]-\  }}|tK          |¦  «                                      |¦  «         Œ.i }|D ]c} ‰.| |j	        ¦  «        \  }}t;          |tK          |¦  «                 ¦  «        }|r|||j'        <   ŒE|j	        |j(        v r|j	        g||j'        <   ŒdtE          t:          ¦  «        }|  #                    d¬¦  «         $                    ¦   «         D ]‘\  }}‰0                     ¦   «         D ]w\  }}t-          |t"          j        ¦  «        rX ‰.||¦  «        \  }}|rG|€E||          )                    |¦  «         tU          || +                    d¦  «        d         |¦  «         ŒxŒ’‰1D ]&\  }}	 tY          ||¦  «         Œ# tZ          $ r Y Œ#w xY w‰0 .                    ¦   «         D ]Z}t-          |t"          j        ¦  «        r>t_          ||g ¦  «         |j         0                    ¦   «          |                     ¦   «          Œ[| $                    ¦   «         D �]5\  }} tc          ‰0|¦  «        }t          | ¦  «        }!d|fg}"|"rº|" 2                    ¦   «         \  }#}$t-          |$t"          j        tf          f¦  «        rE|$j        j        D ]8}|j        dk    r+|#r|#dz   |j	        z   n|j	        }|! 4                    |¦  «         Œ9|$                     ¦   «         D ]&\  }%}&|" )                    |#r|#dz   |%z   n|%|&f¦  «         Œ'|"°º|!D ]L}|| +                    d¦  «        }'}|'dd…         D ]}(tc          ||(¦  «        }ŒtY          ||'d         ¦  «         ŒM�Œ7|D ]f}tk          j5        |j         ¦  «        D ]0}|j        dk    stC          d|j        › �¦  «        ‚ |||¦  «         Œ1‰0j                             |¦  «         Œg‰0 6                    ¦   «          ‰0j         0                    ¦   «          ‰0                     ¦   «          tn           8                    ‰0¦  «        })d}*|}+|�uts          | ‰0j        |¦  «        \  },}-}+|,�Gtu          ‰0j        |,|-¦  «         ‰0                     ¦   «          d}*t                               d¦  «         n,tw          d|›�¦  «        ‚t                               d¦  «         t                               d‰0› �¦  «         to          ‰0|)|*|+¦  «        S )a  
        Additionally, the ``output_loss_value_spec`` value can be specified to disambiguate
        which value in the output of `forward` is the loss value on which PiPPy should apply
        backpropagation. For example, if your ``forward`` returns a tuple ``(loss, model_out)``,
        you can specify ``output_loss_value_spec=(True, False)``. Or, if your ``forward`` returns
        a dict ``{'loss': loss_value, 'model_out': model_out}``, you can specify
        ``output_loss_value_spec={'loss': True, 'model_out': False}``
        F)Úcheck_guardsNzAuto-splitting model)Úprint_outputÚget_attrrS   rQ   r   r   r?   c                 ó~   •— | j         | j        fdt          fk    r"t                               d‰› �¦  «         ‰dz  Š‰S )NrQ   zFound pipe_split r   )r=   rl   Úaten_pipe_split_aliasr  r	  )r?   Úpart_idxs    €r    Úsplit_callbackz)Pipe._from_traced.<locals>.split_callback÷  sO   ø€ à”�a”hÐØÝ%ð$ò ð õ —’Ð;°Ð;Ð;Ñ<Ô<Ð<Ø˜A‘�ØˆOr"   Úpp)Úpartition_affixc           	      óü  ‡ — t          |j        ¦  «        dk    s$t          dt          |j        ¦  «        › �¦  «        ‚ˆ fd„t          |j        ¦  «        D ¦   «         }t          |¦  «        dk    st          dt          |¦  «        › �¦  «        ‚t          |j        ¦  «        }|                     |d         ¦  «         t          |¦  «        |_        t           	                    d‰ › d|› d|d         › �¦  «         d	S )
zÌ
            Delete reference of `node` from `user`'s arg list.
            Args:
                - node: a `get_attr` node at root.
                - user: a submodule node that uses `node`.
            r   z&Expected user.kwargs to be empty, got c                 ó&   •— g | ]\  }}|‰k    ¯|‘ŒS rC   rC   )r>   re   rê   rs   s      €r    r@   zDPipe._from_traced.<locals>.delete_user_reference.<locals>.<listcomp>%  s"   ø€ ÐLÐLÐL™f˜a ÀÀtÂÀ˜ÀÀÀr"   r   zExpected 1 use index, got zDeleted z from user z, arg index = N)
r*   rj   rF   rq   rG   r)   r   r(   r  r	  )rs   ÚuserÚuse_idxsÚ	args_copys   `   r    Údelete_user_referencez0Pipe._from_traced.<locals>.delete_user_reference  s  ø€ õ �t”{Ñ#Ô# qÒ(Ð(Ý$ØO½SÀÄÑ=MÔ=MÐOÐOñô ð ð MÐLÐLÐL­	°$´)Ñ(<Ô(<ÐLÑLÔLˆHÝ�x‘=”= AÒ%Ð%Ý$Ð%QÅ#ÀhÁ-Ä-Ð%QÐ%QÑRÔRÐRÝ˜TœY™œˆIØ�MŠM˜( 1œ+Ñ&Ô&Ð&Ý˜iÑ(Ô(ˆDŒIÝ�LŠLØM˜4ÐMÐM¨DÐMÐMÀÈÄÐMÐMñô ð ð ð r"   c                 óô   — |                      d¦  «        }|d d…         D ]%}t          | |¦  «        s dS t          | |¦  «        } Œ&t          | |d         ¦  «        s| d fS t          | |d         ¦  «        }| |fS )Nr  rS   ©NN)r&  Úhasattrr%  )r7   Úfqnr2  r3  Úattrs        r    Ú_recursive_getattr_with_parentz9Pipe._from_traced.<locals>._recursive_getattr_with_parent3  s�   € à—I’I˜c‘N”NˆEØ˜c˜r˜cœ
ð )ð )�Ý˜s DÑ)Ô)ð &Ø%˜:˜:Ý˜c 4Ñ(Ô(��Ý˜3  b¤	Ñ*Ô*ð !Ø˜D�yÐ Ý˜3  b¤	Ñ*Ô*ˆDØ˜�9Ðr"   c           
      óÂ  •— |                      d¦  «        } ‰‰	|¦  «        \  }}|d         |j        v }t          |t          j        ¦  «        s[t          d|› dt          j        › dt          |¦  «        › d�t          |t          j        j        ¦  «        r	d|› d|› d�nd	z   ¦  «        ‚|  	                    |¦  «        }t          ||¦  «        rt          d
|› d|› �¦  «        ‚|rt          |||t          j        d¬¦  «         nt          |||t          j        ¬¦  «         t                               d|› d|› �¦  «         ‰
                     ||d         f¦  «         dS )a'  
            Move a parameter from the root module to a submodule.
            Args:
                root: The root module.
                callee_name: The name of the submodule to move the parameter to.
                param_fqn: The fully qualified name of the parameter to move.
            r  rS   z
Expected 'z' to be z	 but got z It might happen if module 'zu' was passed to some 'leaf function'(see https://pytorch.org/docs/stable/fx.html#fx.wrap). Please inspect usages of 'z' in the traced graph.Ú zModule z already has a parameter named T)Ú	attr_kindÚ
persistent)rh  zMoved parameter r  N)r&  Ú_buffersr$   rŸ   rÊ   rF   r0   r    rú   r  rb  r   r   ÚBUFFERÚ	PARAMETERr  r	  rˆ   )ÚrootÚcallee_nameÚ	param_fqnr2  Úmod_itrÚ	param_valÚ	is_bufferÚcalleere  r&  Ú	to_deletes           €€€r    Úmove_param_to_calleez/Pipe._from_traced.<locals>.move_param_to_callee?  sÔ  ø€ ð —O’O CÑ(Ô(ˆEØ!?Ð!?ÀÀyÑ!QÔ!QÑˆG�Yà˜bœ	 WÔ%5Ð5ˆIõ ˜i­¬Ñ6Ô6ð 
Ý$Ø] Ð]Ð]µE´LÐ]Ð]Í4ÐPYÉ?Ì?Ð]Ð]Ð]õ
 & iµ´´ÑAÔAð ð H°yð Hð Hà&/ðHð Hð Hð Hð  ññ	ô 	ð 	ð ×'Ò'¨Ñ4Ô4ˆFÝ�v˜yÑ)Ô)ð Ý$ØU˜kÐUÐUÈ)ÐUÐUñô ð ð
 ð ÝØØØÝ'Ô.Ø#ðñ ô ð ð õ ØØØÝ'Ô1ð	ñ ô ð õ �LŠLÐH¨IÐHÐH¸;ÐHÐHÑIÔIÐIð ×Ò˜g u¨R¤yÐ1Ñ2Ô2Ð2Ð2Ð2r"   c                 ó   — | j         dk    S )NrS  r<   rÞ   s    r    rß   z#Pipe._from_traced.<locals>.<lambda>‚  s   € ¨1¬4°:Ò+=€ r"   z
Parameter z used in multiple stages: r  r`   z*Expected user.op to be 'call_module', got T)Ú	keep_varsrg  z5Pipeline is in training mode, backward pass generatedz@Did not find any loss value according to output_loss_value_spec=z:Pipeline is in inference mode, backward pass not generatedzFull pipe model:
)<r™   r  Úinfor	  Úprint_readabler†   rE   r=   r]   rl   Úreplace_all_uses_withÚ
erase_noder.   rq   r–   Úaddr  r%   r&   r   Úeliminate_dead_codeÚmodulesr$   rû   rU  r   rŒ   r   Úregister_moduler)   Úfilterr*   ÚusersrF   r   Ú
state_dictr$  ÚidÚnamed_buffersr×   Ú	constantsrˆ   r'  r&  Údelattrr*  Úchildrenr   Úlintr%  r   r   Údiscardr’   Údelete_all_unused_submodulesr  rL  rO   r…   r'   )2r7   rM  rN  rJ   rO  ÚtracedÚget_attr_nodesrs   Úprev_pipe_split_idxÚpipe_split_nodes_to_erasere   rW  Ú	submoduler×   Ú
new_submodr_  ru  Ú
attr_nodesr\  Ú
id_to_fqnsrc  ÚtensorÚinputs_to_staterd  r   ÚfqnsÚadded_attributesr  ÚparentÚchildrp  Ú	last_atomÚ
attributesÚunused_attributesÚstackÚscopeÚ_modÚ_nameÚ_submodr2  r3  r  r  Úgenerated_loss_specrN   rL   re  rV  r&  rt  s2                                                 @@@@r    Ú_from_tracedzPipe._from_tracedÁ  s
  øøøø€ ð$ "×(Ò(°eÐ(Ñ<Ô<ˆàÐ#Ý�KŠKÐ.Ñ/Ô/Ð/Ø!�\ &Ñ)Ô)ˆFå�Š�V×*Ò*¸Ð*Ñ>Ô>Ñ?Ô?Ð?ð
 .0ˆØ”LÔ&ð 	2ð 	2ˆDØŒw˜*Ò$Ð$Ø×)Ò)¨$¬+°tÑ<Ô<Ð<à! $¤+Ô.°$Ò6Ð6Ø×.Ò.¨~¸d¼kÔ/JÑKÔKÐKØ”L×+Ò+¨DÑ1Ô1Ð1øð !ÐÝ$'¡E¤EÐ!Ý  ¤Ô!3Ñ4Ô4ð 	(ð 	(‰GˆAˆtØ”˜œÐ%¨/½:Ð)FÒFÐFØ&¨!¨a©%Ò/Ð/Ø-×1Ò1°$Ñ7Ô7Ð7Ø&'Ð#øà-ð 	*ð 	*ˆDØŒL×#Ò# DÑ)Ô)Ð)Ð)à×ÒÑÔÐàˆð	�bœgð 	ð 	ð 	ð 	ð 	ð 	õ ˜V S¨.È$ÐOÑOÔOˆàŒ×'Ò'Ñ)Ô)Ð)ð Ÿš™œð 	&ð 	&ˆIÝ˜)¥R¤^Ñ4Ô4ð &Ø%œOÔ1ð 9ð 9�DØœ ¤Ð-Ø'Ý-ð2ò ð ð "œ×2Ò2°4Ñ8Ô8Ð8øØ×#Ò#Ñ%Ô%Ð%øà$×3Ò3Ñ5Ô5ð 	8ð 	8‰OˆD�)Ý˜)¥R¤^Ñ4Ô4ð 8Ý+¨I¬OÑ<Ô<Ð<Ý0°´ÑAÔA�
à×%Ò% d¨JÑ7Ô7Ð7øð	ð 	ð 	ð. ˆ	ð
	ð 
	ð 
	ð@	3ð @	3ð @	3ð @	3ð @	3ð @	3ð @	3õF �&Ð!=Ð!=¸u¼{Ô?PÑQÔQÑRÔRˆ
Øð 	ð 	ˆDå�4”:‰Œ Ò"Ð"Ý—’ØU ¤ÐUÐUÈÌ
ÐUÐUÐUñô ð ð œ
ð 
ð 
�Ø”w -Ò/Ð/Ý(ØNÀTÄWÐNÐNñô ð ð %Ð$ØØ”KØ”Kñô ð ð ð
õ +6µcÑ*:Ô*:ˆ
ØŸ>š>°D˜>Ñ9Ô9×?Ò?ÑAÔAð 	,ð 	,‰KˆC�Ø•r˜&‘z”zÔ"×&Ò& sÑ+Ô+Ð+Ð+Ø×,Ò,Ñ.Ô.ð 	,ð 	,‰KˆC�Ø•r˜&‘z”zÔ"×&Ò& sÑ+Ô+Ð+Ð+ð 13ˆØð 	;ð 	;ˆDØ6Ð6°s¸D¼KÑHÔH‰IˆAˆvÝ˜
¥2 f¡:¤:Ô.Ñ/Ô/ˆDØð ;Ø-1� ¤	Ñ*Ð*Ø”Ð 0Ô :Ð:Ð:Ø.2¬k¨]� ¤	Ñ*øõ
 2=½TÑ1BÔ1BÐØŸ>š>°D˜>Ñ9Ô9×?Ò?ÑAÔAð 	Dð 	D‰KˆC�Ø %× 4Ò 4Ñ 6Ô 6ð Dð D‘��fÝ˜f¥b¤nÑ5Ô5ð DØ$BÐ$BÀ6È3Ñ$OÔ$O‘M�F˜EàðDØ#( =à(¨Ô.×5Ò5°cÑ:Ô:Ð:Ý ¨¯	ª	°#©¬°rÔ(:¸FÑCÔCÐCøðDð #,ð 	ð 	ÑˆG�YðÝ˜ Ñ+Ô+Ð+Ð+øÝ!ð ð ð à�ðøøøð
 —n’nÑ&Ô&ð 	#ð 	#ˆFÝ˜&¥"¤.Ñ1Ô1ð #Ý˜V _°bÑ9Ô9Ð9Ø”×!Ò!Ñ#Ô#Ð#Ø× Ò Ñ"Ô"Ð"øð
 !1× 6Ò 6Ñ 8Ô 8ð 	,ñ 	,ÑˆD�*Ý˜U DÑ)Ô)ˆFÝ # J¡¤Ðà˜&�\�NˆEØð 	UØ#Ÿiši™kœk‘��tÝ˜d¥R¤^Õ5FÐ$GÑHÔHð ;Ø $¤
Ô 0ð ;ð ;˜Øœ7 jÒ0Ð0à?DÐ"U %¨#¡+°´Ñ";Ð";È$Ì+˜CØ-×5Ò5°cÑ:Ô:Ð:øØ&*×&9Ò&9Ñ&;Ô&;ð Uð U‘N�E˜7Ø—L’L¸Ð"I %¨#¡+°Ñ"5Ð"5ÀEÈ7Ð!SÑTÔTÐTÐTð ð 	Uð *ð ,ð ,�Ø!'¨¯ª°C©¬˜�Ø! # 2 #œJð 5ð 5�DÝ% g¨tÑ4Ô4�G�GÝ˜  r¤Ñ+Ô+Ð+Ð+ñ	,ð ð 		)ð 		)ˆDåœ	 $¤*Ñ-Ô-ð 2ð 2�Ø”w -Ò/Ð/Ý(ØNÀTÄWÐNÐNñô ð ð &Ð% d¨DÑ1Ô1Ð1Ð1àŒK×"Ò" 4Ñ(Ô(Ð(Ð(à×*Ò*Ñ,Ô,Ð,ØŒ×ÒÑÔÐØ�ŠÑÔÐå×:Ò:¸5ÑAÔAˆ
à %ÐØ4Ðà!Ð-Ý:KØ�U”[Ð"8ñ;ô ;Ñ7ˆI�{Ð$7ð Ð$Ý/Ø”KØØñô ð ð
 —’Ñ!Ô!Ð!Ø(,Ð%Ý—’ÐTÑUÔUÐUÐUå"ØYÐ@VÐYÐYñô ð õ �LŠLÐUÑVÔVÐVå�ŠÐ1¨%Ð1Ð1Ñ2Ô2Ð2åØØØ!Øñ	
ô 
ð 	
s   Õ2VÖ
VÖVc                 ó8   — | j                              ¦   «          dS )z~
        Print the pipe in a human-readable format.
        This will print both the root pipe and each stage module.
        N)r  ry  ©r—   s    r    ry  zPipe.print_readable  s   € ð
 	Œ×$Ò$Ñ&Ô&Ð&Ð&Ð&r"   Úexample_args.Úexample_kwargsc                 óÄ   — t                                d¦  «         	 t          j                             | ||¦  «        }n"# t          $ r}t          d¦  «        |‚d }~ww xY w|S )NzTracing model ...a.  It seems that we cannot capture your model as a full graph. Typical reasons include graph breaks, data/shape-dependent control flow, or missing meta kernels for custom operators. You can use our manual pipeline interfaces, or try to fix the graph breaks, see https://pytorch.org/docs/stable/export.html)r  rx  rŸ   ÚexportÚ	Exceptionr'   )r7   r¥  r¦  ÚepÚes        r    Ú_trace_with_exportzPipe._trace_with_export  sz   € õ 	�ŠÐ'Ñ(Ô(Ð(ð		Ý”×$Ò$ S¨,¸ÑGÔGˆBˆBøÝð 	ð 	ð 	ÝðPñô ð ðøøøøð	øøøð ˆ	s   œ!> ¾
AÁAÁAc                 ó<  — t           j        }d }	 t                               | ||¦  «        }t                               | ||||¬¦  «        }|j        }|                     ¦   «         }	t          t          | 	                    ¦   «         ¦  «        ¦  «        }
t          |
j        ¦  «        }t          |	j        ¦  «        }t          |j        ¦  «        t          |j        ¦  «        k    rFt                               dt          |j        ¦  «        › dt          |j        ¦  «        › d�¦  «         npt!          j        |	j        j        ¦  «        |
j        _        |
j        j        j                             d ¬¦  «        |
j        j        _        |
                     ¦   «          |S )N)rJ   rO  zOriginal model takes z) args but the first pipeline stage takes z4. Please provide args to respective pipeline stages.)Úout_spec)r½   r¿   r  r¬  r¢  r  r™   ÚnextÚiterr‡  r   rš   r*   r?  r  rx  r’   r(  r†   Ú_codegenÚpytree_infoÚ_replacer  )r7   r¥  r¦  rO  rN  rJ   rM  Úpiper&  r‹  Úsubmod0Úsubmod0_signÚ
model_signs                r    Úfrom_tracingzPipe.from_tracing-  s™  € õ  7Ô@Ðð '+Ðð	õ  ×2Ò2ØØØñ
ô 
Ðõ × Ò ØØØ Ø#9Ø%ð !ñ 
ô 
ˆð ”ˆØ!×(Ò(Ñ*Ô*ˆÝ•t˜EŸNšNÑ,Ô,Ñ-Ô-Ñ.Ô.ˆÝ  ¤Ñ1Ô1ˆÝ˜vœ~Ñ.Ô.ˆ
ÝˆzÔ$Ñ%Ô%­¨\Ô-DÑ)EÔ)EÒEÐEõ �KŠKðE­¨JÔ,AÑ(BÔ(Bð Eð EÝ.1°,Ô2IÑ.JÔ.JðEð Eð Eñô ð ð õ &*¤]°6´<Ô3HÑ%IÔ%IˆGŒMÔ"ð
 ”Ô&Ô2×;Ò;ÀTÐ;ÑJÔJð ŒMÔ"Ô.ð ×ÒÑÔÐàˆr"   c                 ó4   — | j                              ¦   «         S rU   )r  Ú__str__r¤  s    r    rº  zPipe.__str__p  s   € ØŒ}×$Ò$Ñ&Ô&Ð&r"   c                 ó4   — | j                              ¦   «         S rU   )r  Ú__repr__r¤  s    r    r¼  zPipe.__repr__s  s   € ØŒ}×%Ò%Ñ'Ô'Ð'r"   c                 óN   — t          | j        j        | j        | j        ¬¦  «        S )z¢
        Get information about the pipe.

        Returns
        -------
        PipeInfo
            A dataclass containing information about the pipe.
        )r†   r  r  )r   r  r†   r  r  r¤  s    r    rx  z	Pipe.infov  s.   € õ Ø”-Ô%Ø”Ø"&Ô"<ð
ñ 
ô 
ð 	
r"   Ústage_indexr  Úgroupc                 ó.  — |                       |¦  «        }t          |t          j        j        ¦  «        rt          ||¦  «         n*t                               dt          |¦  «        › �¦  «         |  	                    ¦   «         }t          |||||¦  «        S )z“
        Create a `PipelineStage` given a stage index and distributed group.
        The `PipelineStage` can run with `PipelineSchedule`s.
        z*Expected a `torch.fx.GraphModule` but got )rH  r$   rŸ   r%   rû   r  r  r  r0   rx  r   )r—   r¾  r  r¿  Ústage_moduleÚ	pipe_infos         r    Úbuild_stagezPipe.build_stage…  s‘   € ð ×,Ò,¨[Ñ9Ô9ˆõ �l¥E¤HÔ$8Ñ9Ô9ð 	Ý# L°&Ñ9Ô9Ð9Ð9å�NŠNØQ½TÀ,Ñ=OÔ=OÐQÐQñô ð ð —I’I‘K”Kˆ	Ý˜l¨K¸ÀFÈEÑRÔRÐRr"   ©NNNrU   ra  )"r›   rœ   r�   r%   rû   rJ  Úboolr¦   rš   rŸ   r    rú   rH  rž   rL  r   ÚMultiUseParamSpecr   r¢  ry  r(   r
   r-   r   r¬  r¸  rº  r¼  r   rx  r  r   r   rÃ  rC   r"   r    r  r  1  sÂ  € € € € € ðMà”.ðMð ðMð  $ð	Mð Mð Mð Mð^ð ð ðB3¨#ð 3°%´(´/ð 3ð 3ð 3ð 3ð ð¨R¬^ð ð ð ñ „\ðð( ð :>Ø#àðN
ð N
ØŒXŒ_ðN
à)ðN
ð 0°$Ñ6ðN
ð
  ¤Ô 4Ð5°u´xÔ7KÐKÔLØ
ñðN
ð N
ð N
ñ „\ðN
ð`
'ð 'ð 'ð ð 15ðð ØŒXŒ_ðà˜C ˜H”oðð ˜S #˜Xœ¨Ñ-ðð 
ð	ð ð ñ „\ðð& ð 15ØJNð	@ð @ØŒXŒ_ð@à˜C ˜H”oð@ð ˜S #˜Xœ¨Ñ-ð@ð  ¤Ð/°´Ð?Ô@À4ÑGð	@ð @ð @ñ „\ð@ðD'ð 'ð 'ð(ð (ð (ð
�hð 
ð 
ð 
ð 
ð& &*ð	 Sð  Sàð Sð ”ð Sð ˜dÑ"ð	 Sð
 
ð Sð  Sð  Sð  Sð  Sð  Sr"   r  c                   ó   — e Zd ZdZdZdZdS )Ú
SplitPointac  
    Enum representing the points at which a split can occur in the execution of a submodule.
    Attributes:
        BEGINNING: Represents adding a split point *before* the execution of a certain submodule in the `forward` function.
        END: Represents adding a split point *after* the execution of a certain submodule in the `forward` function.
    r   rR   N)r›   rœ   r�   r¬   Ú	BEGINNINGÚENDrC   r"   r    rÈ  rÈ  ¨  s$   € € € € € ðð ð €IØ
€C€C€Cr"   rÈ  c                   ó   — e Zd ZeZdS )ÚPipeSplitWrapperN)r›   rœ   r�   rÈ  rC   r"   r    rÌ  rÌ  ¶  s   € € € € € à€J€J€Jr"   rÌ  c                 ó8   — t          ¦   «           | j        |i |¤ŽS rU   )r–   Ú_orig_forwardr«   s      r    Ú_split_before_forwardrÏ  »  s#   € Ý�L„L€LØˆ4Ô˜tÐ. vÐ.Ð.Ð.r"   c                 ó`   — 	  | j         |i |¤Žt          ¦   «          S # t          ¦   «          w xY wrU   )rÎ  r–   r«   s      r    Ú_split_after_forwardrÑ  À  s5   € ðØ!ˆtÔ! 4Ð2¨6Ð2Ð2å‰Œˆˆø�
‰Œˆˆøøøs   ‚ �-r4   c                 ó`  — |                      ¦   «         D �]\  }}|                     d¦  «        }| }t          |d d…         ¦  «        D ]\\  }}	 t          ||¦  «        }Œ# t          $ r9}t	          d|› dd                     |d |dz   …         ¦  «        › �¦  «        |‚d }~ww xY wt          ||d         ¦  «        }	|	j        |	_        |t          j	        k    rt          t          |	¦  «        |	_        ŒÞ|t          j        k    rt          t          |	¦  «        |	_        �Œ
t          d¦  «        ‚d S )Nr  rS   zSpecified target z referenced nonexistent module r   zUnknown split point type.)r$  r&  rq   r%  r*  r   rš   rÎ  rÈ  rÉ  r	   rÏ  rÊ  rÑ  rG  )
r7   r4   ÚqualnameÚ
split_typer2  Úpredecessor_modulere   r3  r«  Úmod_to_wraps
             r    Úannotate_split_pointsr×  Ç  sc  € à $§
¢
¡¤ð :ñ :Ñˆ�*Ø—’˜sÑ#Ô#ˆØ ÐÝ   s¨ s¤Ñ,Ô,ð 	ð 	‰GˆAˆtðÝ%,Ð-?ÀÑ%FÔ%FÐ"Ð"øÝ!ð ð ð Ý$ðE¨ð Eð EØ*-¯(ª(°5¸¸1¸q¹5¸´>Ñ*BÔ*BðEð Eñô ð ðøøøøðøøøõ Ð0°%¸´)Ñ<Ô<ˆØ$/Ô$7ˆÔ!Ø�Ô-Ò-Ð-Ý",Õ-BÀKÑ"PÔ"PˆKÔÐØ�:œ>Ò)Ð)Ý",Õ-AÀ;Ñ"OÔ"OˆKÔÑåÐ8Ñ9Ô9Ð9ð':ð :s   ÁA Á 
B#Á*4BÂB#r™   Úmb_args.Ú	mb_kwargsÚ
split_specrO  rD  c                 óÂ   — |�|�t          d¦  «        ‚|�-t          | |¦  «         t                               | ||¬¦  «        S t                               | |||¬¦  «        S )a6  
    Split a module based on a specification.

    See `Pipe` for more details.

    Arguments
    ---------
    module:
        The module to be split.
    mb_args:
        Example positional inputs, in micro-batch form.
    mb_kwargs:
        Example keyword inputs, in micro-batch form. (default: `None`)
    split_spec:
        A dictionary using submodule names as split marker. (default: `None`)
    split_policy:
        The policy to use for splitting the module. (default: `None`)

    Returns
    -------
    A pipeline representation of class `Pipe`.
    NzQCannot specify both `split_spec` and `split_policy`. Please use only one of them.)r7   r¥  r¦  )r7   r¥  r¦  rO  )rG  r×  r  r¸  )r™   rØ  rÙ  rÚ  rO  s        r    ÚpipelinerÜ  ß  s�   € ð: Ð ,Ð":ÝØ_ñ
ô 
ð 	
ð Ðå˜f jÑ1Ô1Ð1Ý× Ò ØØ Ø$ð !ñ 
ô 
ð 	
õ × Ò ØØ Ø$Ø%ð	 !ñ 
ô 
ð 	
r"   rÄ  )^r’   Úloggingrm   Úcollectionsr   Úcollections.abcr   Úenumr   Úinspectr   r   r   Útypesr	   Útypingr
   rŸ   Útorch.fxr%   Útorch.distributedr   Útorch.exportr   Útorch.export.unflattenr   r   r   r   Útorch.fx.noder   Útorch.fx.passes.split_moduler   Ú	_backwardr   r   Ú
_unflattenr   Ú_utilsr   Ústager   Ú	getLoggerr›   r  r   rJ  r!   r,   r    rú   rî   rO   r&   r…   rŒ   r¡   rŽ   r£   rH   ÚlibraryÚdefineÚimplr¶   Úregister_fakerº   r»   r5  rU  rs   Ú_side_effectful_functionsr|  r–   r½   r-   r   rÆ  ÚInterpreterrÁ   rÕ   rÚ   rý   r  rû   r  r  r  rÈ  rÌ  rÏ  rÑ  r×  r(   rÜ  rC   r"   r    ú<module>rõ     s  ðð €€€Ø €€€Ø €€€Ø #Ð #Ð #Ð #Ð #Ð #Ø $Ð $Ð $Ð $Ð $Ð $Ø Ð Ð Ð Ð Ð Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð Ø *Ð *Ð *Ð *Ð *Ð *Ø (Ð (Ð (Ð (Ð (Ð (ðð ð ð ð ð ð ð ð ð ð ð ð (Ð 'Ð 'Ð 'Ð 'Ð 'Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5à @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ø +Ð +Ð +Ð +Ð +Ð +Ø Ð Ð Ð Ð Ð Ø !Ð !Ð !Ð !Ð !Ð !ð 
ˆÔ	˜8Ñ	$Ô	$€ð Ð ð8˜sð 8ð 8ð 8ð 8ð,Sð ,Sð ,Sð^2˜5œ8œ?ð 2¨r¬xð 2ð 2ð 2ð 2ð<jØ	„xðjàŒwðjð ”ðjð jð jð jðZ r¤xð ð ð ð ð*
ð 
ð 
ð 
ð 
�U”XÔ(ñ 
ô 
ð 
ð%
ð %
ð %
ð %
ð %
�%”(”/ñ %
ô %
ð %
ðPð ð ð ð ˜ñ ô ð ð@ „× Ò Ð)¨:Ñ 6Ô 6Ð 6ð „×ÒÐ(¨/Ñ:Ô:ðð ñ ;Ô:ðð „×ÒÐ1Ñ2Ô2ðð ñ 3Ô2ðð
 œ	œÔ3Ô;Ð ð „Ô !× %Ò %Ð&;Ñ <Ô <Ð <ð)ð )ð )ð(ð ð ð ð ˜dñ ô ð ð
 ,¨d°3Ð8OÐ3OÔ.PÑPÐ ð1;ð 1;ð 1;ð 1;ð 1;�R”^ñ 1;ô 1;ð 1;ðhð ð ð ð ñ ô ð ð*ð *ð *ð *ð *ñ *ô *ð *ðZ3ð 3ð 3ð,ð ð ð ØŒÔð à”ð ð  ð  ð  ðFt	Sð t	Sð t	Sð t	Sð t	Sˆ5Œ8Œ?ñ t	Sô t	Sð t	Sðn	ð 	ð 	ð 	ð 	�ñ 	ô 	ð 	ðð ð ð ð ñ ô ð ð
/ð /ð /ð
ð ð ð:˜uœxœð :°d¸3À
¸?Ô6Kð :ð :ð :ð :ð6 (,Ø/3ØFJð1
ð 1
ØŒHŒOð1
à�3˜�8Œ_ð1
ð �C˜�HŒ~ Ñ$ð1
ð �S˜*�_Ô%¨Ñ,ð	1
ð
 ˜BœNÐ+¨R¬^Ð;Ô<¸tÑCð1
ð 
ð1
ð 1
ð 1
ð 1
ð 1
ð 1
r"   