§
    ‚Štj¾M  ã                   ó^  — d dl mZ d dlmZ d dlZd dlmc mZ d dlmZ ddl	m
Z ddlmZ ddlmZ dd	lmZmZ dd
lmZmZ ddlmZ ddlmZmZmZ ddlmZ ddlmZ ddl m!Z! e G d„ de¦  «        ¦   «         Z" G d„ dej#        ¦  «        Z$ G d„ dej#        ¦  «        Z% G d„ dej#        ¦  «        Z& G d„ dej#        ¦  «        Z'	 	 d7dej#        dej(        dej(        dej(        d ej(        dz  d!e)dz  d"e)d#ee         fd$„Z* G d%„ d&ej#        ¦  «        Z+ G d'„ d(ej#        ¦  «        Z, G d)„ d*ej#        ¦  «        Z- G d+„ d,ej#        ¦  «        Z. G d-„ d.ej#        ¦  «        Z/ G d/„ d0e¦  «        Z0e G d1„ d2e¦  «        ¦   «         Z1 G d3„ d4e1¦  «        Z2e G d5„ d6e1¦  «        ¦   «         Z3d6d2gZ4dS )8é    )ÚCallable)Ú	dataclassN)Únné   )Úinitialization)ÚACT2FN)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚModelOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Úmerge_with_config_defaults)Úcapture_outputsé   )ÚRadioConfigc                   óÂ   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	ej        dz  ed<   dZ
eej                 dz  ed<   dZeej                 dz  ed<   dS )ÚRadioModelOutputa  Output of [`RadioModel`].

    Args:
        summary (`torch.FloatTensor` of shape `(batch_size, num_summary_idxs * hidden_size)`):
            Flattened summary embedding, gathered from the cls tokens selected by `config.summary_idxs`.
        features (`torch.FloatTensor` of shape `(batch_size, num_patches, hidden_size)`):
            Dense spatial patch features.
        last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
            Full token sequence (prefix tokens + patches) from the final encoder layer.
        hidden_states (`tuple[torch.FloatTensor]`, *optional*, returned when `output_hidden_states=True`):
            Tuple of `(batch_size, sequence_length, hidden_size)` tensors, one for the embedding output plus one for
            each encoder layer.
        attentions (`tuple[torch.FloatTensor]`, *optional*, returned when `output_attentions=True`):
            Tuple of `(batch_size, num_heads, sequence_length, sequence_length)` attention weights, one per layer.
    NÚsummaryÚfeaturesÚlast_hidden_stateÚhidden_statesÚ
attentions)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚtorchÚFloatTensorÚ__annotations__r   r   r   Útupler   © ó    úf/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/radio/modeling_radio.pyr   r   (   s¡   € € € € € € ðð ð  )-€GˆUÔ Ñ%Ð,Ð,Ñ,Ø)-€HˆeÔ $Ñ&Ð-Ð-Ñ-Ø26Ð�uÔ(¨4Ñ/Ð6Ð6Ñ6Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ð6Ð6r&   r   c                   óL   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚRadioInputConditionerzFNormalizes pixel values; arithmetic is done in float32 then cast back.Úconfigc                 óX  •— t          ¦   «                              ¦   «          |                      dt          j        |j        ¦  «                             ddd¦  «        d¬¦  «         |                      dt          j        |j        ¦  «                             ddd¦  «        d¬¦  «         d S )NÚ	norm_meanéÿÿÿÿr   T©Ú
persistentÚnorm_std)ÚsuperÚ__init__Úregister_bufferr!   Útensorr,   Úviewr0   ©Úselfr*   Ú	__class__s     €r'   r2   zRadioInputConditioner.__init__D   s˜   ø€ Ý‰Œ×ÒÑÔÐØ×Ò˜[­%¬,°vÔ7GÑ*HÔ*H×*MÒ*MÈbÐRSÐUVÑ*WÔ*WÐdhÐÑiÔiÐiØ×Ò˜Z­¬°f´oÑ)FÔ)F×)KÒ)KÈBÐPQÐSTÑ)UÔ)UÐbfÐÑgÔgÐgÐgÐgr&   Úpixel_valuesÚreturnc                 óÆ   — |                      ¦   «         | j                              ¦   «         z
  | j                              ¦   «         z  }|                     |j        ¦  «        S ©N)Úfloatr,   r0   ÚtoÚdtype)r7   r9   Ú
normalizeds      r'   ÚforwardzRadioInputConditioner.forwardI   sN   € Ø"×(Ò(Ñ*Ô*¨T¬^×-AÒ-AÑ-CÔ-CÑCÀtÄ}×GZÒGZÑG\ÔG\Ñ\ˆ
Ø�}Š}˜\Ô/Ñ0Ô0Ð0r&   ©
r   r   r   r    r   r2   r!   ÚTensorrA   Ú__classcell__©r8   s   @r'   r)   r)   A   sw   ø€ € € € € ØPÐPðh˜{ð hð hð hð hð hð hð
1 E¤Lð 1°U´\ð 1ð 1ð 1ð 1ð 1ð 1ð 1ð 1r&   r)   c                   ó¨   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        fd„Zde	e
e
f         dej        dej        fd	„Zdej        dej        fd
„Zˆ xZS )ÚRadioPatchEmbeddingszÕCropped Position Embedding (CPE) patch generator.

    Splits the image into patches, projects them, adds a resolution-interpolated
    absolute position embedding, and prepends learned cls + register tokens.
    r*   c                 óX  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        |j        | _        |j        |j        z  | _        |j        |j        z  | _	        | j        | j	        z  }t          j        |j        |j        dz  z  |j        d¬¦  «        | _        t          j        t          j        d||j        ¦  «        ¦  «        | _        t          j        t          j        |j        |j        z   |j        ¦  «        ¦  «        | _        d S )Né   F©Úbiasr   )r1   r2   Ú
patch_sizeÚhidden_sizeÚ	embed_dimÚnum_cls_tokensÚnum_registersÚmax_img_sizeÚmax_rowsÚmax_colsr   ÚLinearÚnum_channelsÚpatch_projectionÚ	Parameterr!   ÚzerosÚposition_embeddingÚcls_register_token)r7   r*   Únum_positionsr8   s      €r'   r2   zRadioPatchEmbeddings.__init__U   sþ   ø€ Ý‰Œ×ÒÑÔÐØ Ô+ˆŒØÔ+ˆŒØ$Ô3ˆÔØ#Ô1ˆÔàÔ+¨vÔ/@Ñ@ˆŒØÔ+¨vÔ/@Ñ@ˆŒØœ¨¬Ñ5ˆå "¤	¨&Ô*=ÀÔ@QÐSTÑ@TÑ*TÐV\ÔVhÐotÐ uÑ uÔ uˆÔÝ"$¤,­u¬{¸1¸mÈVÔM_Ñ/`Ô/`Ñ"aÔ"aˆÔÝ"$¤,ÝŒK˜Ô-°Ô0DÑDÀfÔFXÑYÔYñ#
ô #
ˆÔÐÐr&   r9   r:   c                 óä   — | j         }|j        \  }}}}||z  ||z  }}|                     ||||||¦  «        }	|	                     dddddd¦  «                             |||z  ||z  |z  ¦  «        }	|	S )Nr   rI   é   r   r   é   )rL   ÚshapeÚreshapeÚpermute)
r7   r9   ÚpsÚbatchÚchannelsÚheightÚwidthÚrowsÚcolsÚpatchess
             r'   Ú_image_to_patchesz&RadioPatchEmbeddings._image_to_patchesf   s‹   € ØŒ_ˆØ)5Ô);Ñ&ˆˆx˜ Ø˜r‘\ 5¨B¡;ˆdˆØ×&Ò& u¨h¸¸bÀ$ÈÑKÔKˆØ—/’/ ! Q¨¨1¨a°Ñ3Ô3×;Ò;¸EÀ4È$Á;ÐPXÐ[]ÑP]Ð`bÑPbÑcÔcˆØˆr&   Ú
input_dimsr?   c                 óð  — | j                              d| j        | j        d¦  «                             dddd¦  «        }t          |¦  «        }t          j        |                     ¦   «         ||fdd¬¦  «         	                    |¦  «        }|d         |j
        d	         k     r|d
d |d         …d d …f         }|d         |j
        d         k     r|d
d d …d |d         …f         }|j
        d	d …         t          |¦  «        k    rJt          j        |                     ¦   «         t          |¦  «        dd¬¦  «         	                    |¦  «        }|                     d¦  «                             ddd¦  «        S )Nr   r-   r   r   rI   ÚbilinearF)ÚsizeÚmodeÚalign_cornerséþÿÿÿ.)rY   r`   rR   rS   ra   ÚmaxÚFÚinterpolater=   r>   r_   r$   Úflatten)r7   rk   r?   ÚposÚmax_dims        r'   Ú_interpolate_position_embeddingz4RadioPatchEmbeddings._interpolate_position_embeddingn   sU  € ØÔ%×-Ò-¨a°´ÀÄÈrÑRÔR×ZÒZÐ[\Ð^_ÐabÐdeÑfÔfˆÝ�j‘/”/ˆÝŒm˜CŸIšI™KœK¨w¸Ð.@ÀzÐafÐgÑgÔg×jÒjÐkpÑqÔqˆØ�aŒ=˜3œ9 Rœ=Ò(Ð(Ø�c˜?˜Z¨œ]˜?¨A¨A¨AÐ-Ô.ˆCØ�aŒ=˜3œ9 Rœ=Ò(Ð(Ø�c˜1˜1˜1˜o 
¨1¤˜oÐ-Ô.ˆCØŒ9�R�S�SŒ>�U :Ñ.Ô.Ò.Ð.Ý”- §	¢	¡¤µ%¸
Ñ2CÔ2CÈ*ÐdiÐjÑjÔj×mÒmÐnsÑtÔtˆCØ�{Š{˜1‰~Œ~×%Ò% a¨¨AÑ.Ô.Ð.r&   c                 ó†  — |                       |                      |¦  «        ¦  «        }|j        d         | j        z  |j        d         | j        z  f}||                      ||j        ¦  «        z   }| j                             d¦  «                             |j        d         dd¦  «        }t          j
        ||gd¬¦  «        S )Nrq   r-   r   r   ©Údim)rV   rj   r_   rL   rx   r?   rZ   Ú	unsqueezeÚexpandr!   Úcat)r7   r9   ri   rk   Úprefixs        r'   rA   zRadioPatchEmbeddings.forwardz   s´   € Ø×'Ò'¨×(>Ò(>¸|Ñ(LÔ(LÑMÔMˆØ"Ô(¨Ô,°´Ñ?ÀÔASÐTVÔAWÐ[_Ô[jÑAjÐkˆ
Ø˜D×@Ò@ÀÈWÌ]Ñ[Ô[Ñ[ˆØÔ(×2Ò2°1Ñ5Ô5×<Ò<¸W¼]È1Ô=MÈrÐSUÑVÔVˆÝŒy˜& 'Ð*°Ð2Ñ2Ô2Ð2r&   )r   r   r   r    r   r2   r!   rC   rj   r$   Úintr?   rx   rA   rD   rE   s   @r'   rG   rG   N   sÓ   ø€ € € € € ðð ð
˜{ð 
ð 
ð 
ð 
ð 
ð 
ð"¨e¬lð ¸u¼|ð ð ð ð ð
/¸%ÀÀSÀ¼/ð 
/ÐRWÔR]ð 
/ÐbgÔbnð 
/ð 
/ð 
/ð 
/ð3 E¤Lð 3°U´\ð 3ð 3ð 3ð 3ð 3ð 3ð 3ð 3r&   rG   c                   óD   ‡ — e Zd Zdˆ fd„Zdej        dej        fd„Zˆ xZS )ÚRadioMLPr:   Nc                 ó~  •— t          ¦   «                              ¦   «          |j        x}}t          |j        |j        z  ¦  «        }t          j        ||d¬¦  «        | _        t          |j	        t          ¦  «        rt          |j	                 | _        n|j	        | _        t          j        ||d¬¦  «        | _        d S )NTrJ   )r1   r2   rM   r€   Ú	mlp_ratior   rT   Úfc1Ú
isinstanceÚ
hidden_actÚstrr   Ú
activationÚfc2©r7   r*   Úin_featuresÚout_featuresÚhidden_featuresr8   s        €r'   r2   zRadioMLP.__init__ƒ   s¢   ø€ Ý‰Œ×ÒÑÔÐØ%+Ô%7Ð7ˆ�lÝ˜fÔ0°6Ô3CÑCÑDÔDˆÝ”9˜[¨/ÀÐEÑEÔEˆŒÝ�fÔ'­Ñ-Ô-ð 	0Ý$ VÔ%6Ô7ˆDŒOˆOà$Ô/ˆDŒOÝ”9˜_¨lÀÐFÑFÔFˆŒˆˆr&   Úhidden_statec                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r<   )r…   r‰   rŠ   ©r7   r�   s     r'   rA   zRadioMLP.forwardŽ   s;   € Ø—x’x Ñ-Ô-ˆØ—’ |Ñ4Ô4ˆØ—x’x Ñ-Ô-ˆØÐr&   ©r:   N©r   r   r   r2   r!   rC   rA   rD   rE   s   @r'   r‚   r‚   ‚   si   ø€ € € € € ð	Gð 	Gð 	Gð 	Gð 	Gð 	Gð E¤Lð °U´\ð ð ð ð ð ð ð ð r&   r‚   c                   óD   ‡ — e Zd Zdˆ fd„Zdej        dej        fd„Zˆ xZS )ÚRadioLayerScaler:   Nc                 ó¸   •— t          ¦   «                              ¦   «          t          j        |j        t          j        |j        ¦  «        z  ¦  «        | _        d S r<   )	r1   r2   r   rW   Úlayerscale_valuer!   ÚonesrM   Úlambda1r6   s     €r'   r2   zRadioLayerScale.__init__–   sC   ø€ Ý‰Œ×ÒÑÔÐÝ”| FÔ$;½e¼jÈÔI[Ñ>\Ô>\Ñ$\Ñ]Ô]ˆŒˆˆr&   r�   c                 ó   — || j         z  S r<   )r™   r‘   s     r'   rA   zRadioLayerScale.forwardš   s   € Ø˜dœlÑ*Ð*r&   r’   r“   rE   s   @r'   r•   r•   •   si   ø€ € € € € ð^ð ^ð ^ð ^ð ^ð ^ð+ E¤Lð +°U´\ð +ð +ð +ð +ð +ð +ð +ð +r&   r•   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 ó®  — |€|                      d¦  «        dz  }t          j        ||                     dd¦  «        ¦  «        |z  }|�||z   }t          j                             |d¬¦  «        }t          j                             ||| j        ¬¦  «        }t          j        ||¦  «        }	|	                     dd¦  «         	                    ¦   «         }	|	|fS )Nr-   ç      à¿rI   r   rz   )ÚpÚtrainingr   )
rn   r!   ÚmatmulÚ	transposer   Ú
functionalÚsoftmaxr¢   r§   Ú
contiguous)
rœ   r�   rž   rŸ   r    r¡   r¢   r£   Úattn_weightsÚattn_outputs
             r'   Úeager_attention_forwardr¯   ž   sÈ   € ð €Ø—*’*˜R‘.”. DÑ(ˆõ ”<  s§}¢}°Q¸Ñ':Ô':Ñ;Ô;¸gÑE€LàÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2Ð(Ñ>Ô>€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r&   c                   ór   ‡ — e Zd Zdefˆ fd„Zdej        dee         de	ej        ej        f         fd„Z
ˆ xZS )ÚRadioSelfAttentionr*   c                 ó¤  •— t          ¦   «                              ¦   «          |j        |j        z  dk    r0t	          |d¦  «        s t          d|j        › d|j        › d�¦  «        ‚|| _        |j        | _        t          |j        |j        z  ¦  «        | _        | j        | j        z  | _	        |j
        | _        | j        dz  | _        d| _        t          j        |j        | j	        |j        ¬¦  «        | _        t          j        |j        | j	        |j        ¬¦  «        | _        t          j        |j        | j	        |j        ¬¦  «        | _        d S )	Nr   Úembedding_sizezThe hidden size z4 is not a multiple of the number of attention heads ú.r¥   FrJ   )r1   r2   rM   Únum_attention_headsÚhasattrÚ
ValueErrorr*   r€   Úattention_head_sizeÚall_head_sizeÚattention_probs_dropout_probÚdropout_probr¡   Ú	is_causalr   rT   Úqkv_biasr�   rž   rŸ   r6   s     €r'   r2   zRadioSelfAttention.__init__¼   sB  ø€ Ý‰Œ×ÒÑÔÐØÔ Ô :Ñ:¸aÒ?Ð?ÍÐPVÐXhÑHiÔHiÐ?Ýð7 6Ô#5ð 7ð 7ØÔ3ð7ð 7ð 7ñô ð ð
 ˆŒØ#)Ô#=ˆÔ Ý#& vÔ'9¸FÔ<VÑ'VÑ#WÔ#WˆÔ Ø!Ô5¸Ô8PÑPˆÔØ"Ô?ˆÔØÔ/°Ñ5ˆŒØˆŒå”Y˜vÔ1°4Ô3EÈFÌOÐ\Ñ\Ô\ˆŒ
Ý”9˜VÔ/°Ô1CÈ&Ì/ÐZÑZÔZˆŒÝ”Y˜vÔ1°4Ô3EÈFÌOÐ\Ñ\Ô\ˆŒ
ˆ
ˆ
r&   r   r£   r:   c                 ó~  — |j         d         }|d| j        | j        f} |                      |¦  «        j        |Ž                      dd¦  «        } |                      |¦  «        j        |Ž                      dd¦  «        } |                      |¦  «        j        |Ž                      dd¦  «        }t          j	        | j
        j        t          ¦  «        } || |||d f| j        | j        | j        sdn| j        dœ|¤Ž\  }	}
|	                     ¦   «         d d…         | j        fz   }|	                     |¦  «        }	|	|
fS )Nr   r-   r   rI   r›   )r¼   r¡   r¢   rq   )r_   rµ   r¸   rž   r5   r©   rŸ   r�   r   Úget_interfacer*   Ú_attn_implementationr¯   r¼   r¡   r§   r»   rn   r¹   r`   )r7   r   r£   Ú
batch_sizeÚ	new_shapeÚ	key_layerÚvalue_layerÚquery_layerÚattention_interfaceÚcontext_layerÚattention_probsÚnew_context_layer_shapes               r'   rA   zRadioSelfAttention.forwardÐ   sd  € ð
 #Ô(¨Ô+ˆ
Ø  DÔ$<¸dÔ>VÐVˆ	à0�D—H’H˜]Ñ+Ô+Ô0°)Ð<×FÒFÀqÈ!ÑLÔLˆ	Ø4�d—j’j Ñ/Ô/Ô4°iÐ@×JÒJÈ1ÈaÑPÔPˆØ4�d—j’j Ñ/Ô/Ô4°iÐ@×JÒJÈ1ÈaÑPÔPˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð *=Ð)<ØØØØØð
*
ð ”nØ”LØ#œ}ÐC�C�C°$Ô2Cð
*
ð 
*
ð ð
*
ð 
*
Ñ&ˆ�ð #0×"4Ò"4Ñ"6Ô"6°s¸°sÔ";¸tÔ?QÐ>SÑ"SÐØ%×-Ò-Ð.EÑFÔFˆà˜oÐ-Ð-r&   )r   r   r   r   r2   r!   rC   r   r   r$   rA   rD   rE   s   @r'   r±   r±   »   s‘   ø€ € € € € ð]˜{ð ]ð ]ð ]ð ]ð ]ð ]ð(.à”|ð.ð Ð+Ô,ð.ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð	.ð .ð .ð .ð .ð .ð .ð .r&   r±   c                   óZ   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        dej        fd„Zˆ xZ	S )ÚRadioSelfOutputz¢
    The residual connection is defined in RadioLayer instead of here (as is the case with other models), due to the
    layernorm applied before each block.
    r*   c                 óÌ   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        ¦  «        | _        d S r<   )	r1   r2   r   rT   rM   ÚdenseÚDropoutÚhidden_dropout_probr¢   r6   s     €r'   r2   zRadioSelfOutput.__init__ù   sJ   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ý”z &Ô"<Ñ=Ô=ˆŒˆˆr&   r   Úinput_tensorr:   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S r<   )rÍ   r¢   )r7   r   rÐ   s      r'   rA   zRadioSelfOutput.forwardþ   s*   € ØŸ
š
 =Ñ1Ô1ˆØŸš ]Ñ3Ô3ˆØÐr&   rB   rE   s   @r'   rË   rË   ó   s   ø€ € € € € ðð ð
>˜{ð >ð >ð >ð >ð >ð >ð
 U¤\ð ÀÄð ÐRWÔR^ð ð ð ð ð ð ð ð r&   rË   c                   óX   ‡ — e Zd Zdefˆ fd„Zdej        dee         dej        fd„Z	ˆ xZ
S )ÚRadioAttentionr*   c                 ó˜   •— t          ¦   «                              ¦   «          t          |¦  «        | _        t	          |¦  «        | _        d S r<   )r1   r2   r±   Ú	attentionrË   Úoutputr6   s     €r'   r2   zRadioAttention.__init__  s;   ø€ Ý‰Œ×ÒÑÔÐÝ+¨FÑ3Ô3ˆŒÝ% fÑ-Ô-ˆŒˆˆr&   r   r£   r:   c                 óT   —  | j         |fi |¤Ž\  }}|                      ||¦  «        }|S r<   )rÕ   rÖ   )r7   r   r£   Úself_attn_outputÚ_rÖ   s         r'   rA   zRadioAttention.forward  s<   € ð
 -˜dœn¨]ÐEÐE¸fÐEÐEÑÐ˜!Ø—’Ð-¨}Ñ=Ô=ˆØˆr&   )r   r   r   r   r2   r!   rC   r   r   rA   rD   rE   s   @r'   rÓ   rÓ     s~   ø€ € € € € ð.˜{ð .ð .ð .ð .ð .ð .ð
à”|ðð Ð+Ô,ðð 
Œð	ð ð ð ð ð ð ð r&   rÓ   c                   óD   ‡ — e Zd Zdˆ fd„Zdej        dej        fd„Zˆ xZS )ÚRadioSwiGLUFFNr:   Nc                 óD  •— t          ¦   «                              ¦   «          |j        x}}t          |j        |j        z  ¦  «        }t          |dz  dz  ¦  «        dz   dz  dz  }t          j        |d|z  d¬¦  «        | _        t          j        ||d¬¦  «        | _        d S )NrI   r   é   é   TrJ   )	r1   r2   rM   r€   r„   r   rT   Ú
weights_inÚweights_outr‹   s        €r'   r2   zRadioSwiGLUFFN.__init__  s    ø€ Ý‰Œ×ÒÑÔÐØ%+Ô%7Ð7ˆ�lÝ˜fÔ0°6Ô3CÑCÑDÔDˆÝ˜°Ñ2°QÑ6Ñ7Ô7¸!Ñ;ÀÑAÀAÑEˆåœ) K°°_Ñ1DÈ4ÐPÑPÔPˆŒÝœ9 _°lÈÐNÑNÔNˆÔÐÐr&   r�   c                 óÎ   — |                       |¦  «        }|                     dd¬¦  «        \  }}t          j                             |¦  «        |z  }|                      |¦  «        S )NrI   r-   rz   )rß   Úchunkr   rª   Úsilurà   )r7   r�   Úx1Úx2Úhiddens        r'   rA   zRadioSwiGLUFFN.forward  s]   € Ø—’ |Ñ4Ô4ˆØ×#Ò# A¨2Ð#Ñ.Ô.‰ˆˆBÝ”×#Ò# BÑ'Ô'¨"Ñ,ˆØ×Ò Ñ'Ô'Ð'r&   r’   r“   rE   s   @r'   rÛ   rÛ     si   ø€ € € € € ðOð Oð Oð Oð Oð Oð( E¤Lð (°U´\ð (ð (ð (ð (ð (ð (ð (ð (r&   rÛ   c                   ó^   ‡ — e Zd ZdZd
deddfˆ fd„Zdej        dej        fd„Zde	fd	„Z
ˆ xZS )ÚRadioDropPathzÏStochastic depth (DropPath) per sample, for residual blocks.

    Identity when ``drop_prob`` is 0 or outside training. See `Deep Networks with Stochastic Depth
    <https://arxiv.org/abs/1603.09382>`_.
    r›   Ú	drop_probr:   Nc                 óV   •— t          ¦   «                              ¦   «          || _        d S r<   )r1   r2   ré   )r7   ré   r8   s     €r'   r2   zRadioDropPath.__init__-  s$   ø€ Ý‰Œ×ÒÑÔÐØ"ˆŒˆˆr&   r   c                 ó  — | j         dk    s| j        s|S d| j         z
  }|j        d         fd|j        dz
  z  z   }t	          j        ||j        |j        ¬¦  «        }t	          j        ||z   ¦  «        }| 	                    |¦  «        |z  S )Nr›   r   r   )r   )r?   Údevice)
ré   r§   r_   Úndimr!   Úrandr?   rì   ÚfloorÚdiv)r7   r   Ú	keep_probr_   Úrandom_tensors        r'   rA   zRadioDropPath.forward1  s“   € ØŒ>˜SÒ Ð ¨¬Ð Ø Ð Ø˜œÑ&ˆ	ØÔ$ QÔ'Ð)¨D°MÔ4FÈÑ4JÑ,KÑKˆÝœ
 5°Ô0CÈMÔL`ÐaÑaÔaˆÝœ M°IÑ$=Ñ>Ô>ˆØ× Ò  Ñ+Ô+¨mÑ;Ð;r&   c                 ó   — d| j         › �S )Nzp=)ré   ©r7   s    r'   Ú
extra_reprzRadioDropPath.extra_repr:  s   € Ø$�D”NÐ$Ð$Ð$r&   )r›   )r   r   r   r    r=   r2   r!   rC   rA   rˆ   rõ   rD   rE   s   @r'   rè   rè   &  s›   ø€ € € € € ðð ð#ð # %ð #°$ð #ð #ð #ð #ð #ð #ð< U¤\ð <°e´lð <ð <ð <ð <ð%˜Cð %ð %ð %ð %ð %ð %ð %ð %r&   rè   c                   óP   ‡ — e Zd ZdZdeddfˆ fd„Zdej        dej        fd„Zˆ xZ	S )Ú
RadioLayerzCThis corresponds to the Block class in the original implementation.r*   r:   Nc                 ó"  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¬¦  «        | _        t          |¦  «        | _        t          |¦  «        | _
        |j        dk    rt          |j        ¦  «        nt          j        ¦   «         | _        t          j        |j        |j        ¬¦  «        | _        |j        rt#          |¦  «        | _        nt'          |¦  «        | _        t          |¦  «        | _        d S )N)Úepsr›   )r1   r2   r   Ú	LayerNormrM   Úlayer_norm_epsÚnorm1rÓ   rÕ   r•   Úlayer_scale1Údrop_path_raterè   ÚIdentityÚ	drop_pathÚnorm2Úuse_swiglu_ffnrÛ   Úmlpr‚   Úlayer_scale2r6   s     €r'   r2   zRadioLayer.__init__A  sà   ø€ Ý‰Œ×ÒÑÔÐå”\ &Ô"4¸&Ô:OÐPÑPÔPˆŒ
Ý'¨Ñ/Ô/ˆŒÝ+¨FÑ3Ô3ˆÔØAGÔAVÐY\ÒA\ÐA\� vÔ'<Ñ=Ô=Ð=ÕbdÔbmÑboÔboˆŒå”\ &Ô"4¸&Ô:OÐPÑPÔPˆŒ
àÔ ð 	(Ý% fÑ-Ô-ˆDŒHˆHå Ñ'Ô'ˆDŒHÝ+¨FÑ3Ô3ˆÔÐÐr&   r   c                 ób  — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        |z   }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        |z   }|S r<   )rü   rÕ   rý   r   r  r  r  )r7   r   Úhidden_states_normÚself_attention_outputÚlayer_outputs        r'   rA   zRadioLayer.forwardQ  s¬   € ð "ŸZšZ¨Ñ6Ô6ÐØ $§¢Ð/AÑ BÔ BÐØ $× 1Ò 1Ð2GÑ HÔ HÐð ŸšÐ'<Ñ=Ô=ÀÑMˆð —z’z -Ñ0Ô0ˆØ—x’x Ñ-Ô-ˆØ×(Ò(¨Ñ6Ô6ˆð —~’~ lÑ3Ô3°mÑCˆàÐr&   rB   rE   s   @r'   r÷   r÷   >  s{   ø€ € € € € ØMÐMð4˜{ð 4¨tð 4ð 4ð 4ð 4ð 4ð 4ð à”|ðð 
Œðð ð ð ð ð ð ð r&   r÷   c                   ól   — e Zd ZeZdZdZdZdgZdgZ	dZ
dZeedœZ ej        ¦   «         d„ ¦   «         ZdS )	ÚRadioPreTrainedModelÚmodelr9   Tr÷   zlayer_scale\d+\.lambda1)r   r   c                 óŽ  — | j         j        }t          |t          j        ¦  «        r@t          j        |j        d|¬¦  «         |j        �t          j	        |j        ¦  «         d S d S t          |t          j
        ¦  «        r4t          j	        |j        ¦  «         t          j        |j        ¦  «         d S t          |t          ¦  «        r:t          j        |j        d|¬¦  «         t          j        |j        d|¬¦  «         d S t          |t          ¦  «        r&t          j        |j        | j         j        ¦  «         d S t          |t&          ¦  «        r˜t          j        |j        t-          j        | j         j        ¦  «                             ddd¦  «        ¦  «         t          j        |j        t-          j        | j         j        ¦  «                             ddd¦  «        ¦  «         d S t          |t4          ¦  «        rDt          j        |j        t-          j        | j         j        t,          j        ¬¦  «        ¦  «         d S d S )Nr›   )ÚmeanÚstdr-   r   ©r?   )r*   Úinitializer_ranger†   r   rT   ÚinitÚtrunc_normal_ÚweightrK   Úzeros_rú   Úones_rG   rY   rZ   r•   Ú	constant_r™   r—   r)   Úcopy_r,   r!   r4   r5   r0   Ú
RadioModelÚsummary_idxsÚlong)r7   rœ   r  s      r'   Ú_init_weightsz"RadioPreTrainedModel._init_weightsv  s  € ð ŒkÔ+ˆÝ�f�bœiÑ(Ô(ð 	fÝÔ˜vœ}°3¸CÐ@Ñ@Ô@Ð@ØŒ{Ð&Ý”˜FœKÑ(Ô(Ð(Ð(Ð(ð 'Ð&å˜¥¤Ñ-Ô-ð 	fÝŒK˜œÑ$Ô$Ð$ÝŒJ�v”}Ñ%Ô%Ð%Ð%Ð%Ý˜Õ 4Ñ5Ô5ð 		fÝÔ˜vÔ8¸sÈÐLÑLÔLÐLÝÔ˜vÔ8¸sÈÐLÑLÔLÐLÐLÐLÝ˜¥Ñ0Ô0ð 	fÝŒN˜6œ>¨4¬;Ô+GÑHÔHÐHÐHÐHÝ˜Õ 5Ñ6Ô6ð 	fÝŒJ�vÔ'­¬°d´kÔ6KÑ)LÔ)L×)QÒ)QÐRTÐVWÐYZÑ)[Ô)[Ñ\Ô\Ð\ÝŒJ�v”­¬°T´[Ô5IÑ(JÔ(J×(OÒ(OÐPRÐTUÐWXÑ(YÔ(YÑZÔZÐZÐZÐZÝ˜¥
Ñ+Ô+ð 	fÝŒJ�vÔ*­E¬L¸¼Ô9QÕY^ÔYcÐ,dÑ,dÔ,dÑeÔeÐeÐeÐeð	fð 	fr&   N)r   r   r   r   Úconfig_classÚbase_model_prefixÚmain_input_nameÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_keys_to_ignore_on_load_missingÚ_supports_sdpaÚ_supports_flash_attnr÷   r±   Ú_can_record_outputsr!   Úno_gradr  r%   r&   r'   r
  r
  g  s�   € € € € € à€LØÐØ$€OØ&*Ð#Ø%˜ÐØ'AÐ&BÐ#Ø€NØÐà#Ø(ðð Ðð
 €U„]�_„_ðfð fñ „_ðfð fð fr&   r
  c                   ó‚   ‡ — e Zd Zdefˆ fd„Ze ed¬¦  «        dej        de	e
         defd„¦   «         ¦   «         Zˆ xZS )	ÚRadioEncoderr*   c                 óâ   •‡— t          ¦   «                              ‰¦  «         t          j        ˆfd„t	          ‰j        ¦  «        D ¦   «         ¦  «        | _        |                      ¦   «          d S )Nc                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r%   )r÷   )Ú.0rÙ   r*   s     €r'   ú
<listcomp>z)RadioEncoder.__init__.<locals>.<listcomp>‘  s!   ø€ Ð#`Ð#`Ð#`¸1¥J¨vÑ$6Ô$6Ð#`Ð#`Ð#`r&   )r1   r2   r   Ú
ModuleListÚrangeÚnum_hidden_layersÚlayerÚ	post_initr6   s    `€r'   r2   zRadioEncoder.__init__�  sc   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý”]Ð#`Ð#`Ð#`Ð#`ÅÀfÔF^Ñ@_Ô@_Ð#`Ñ#`Ô#`ÑaÔaˆŒ
Ø�ŠÑÔÐÐÐr&   F)Útie_last_hidden_statesr   r£   r:   c                 óL   — | j         D ]} ||¦  «        }Œt          |¬¦  «        S )N)r   )r/  r
   )r7   r   r£   r/  s       r'   rA   zRadioEncoder.forward”  s7   € ð ”Zð 	1ð 	1ˆEØ!˜E -Ñ0Ô0ˆMˆMÝ°Ð?Ñ?Ô?Ð?r&   )r   r   r   r   r2   r   r   r!   rC   r   r   r
   rA   rD   rE   s   @r'   r'  r'  Ž  s¦   ø€ € € € € ð˜{ð ð ð ð ð ð ð
  Ø€_¨EÐ2Ñ2Ô2ð@ U¤\ð @¸VÐDVÔ=Wð @Ð\kð @ð @ð @ñ 3Ô2ñ  Ôð@ð @ð @ð @ð @r&   r'  c                   ó�   ‡ — e Zd Zdefˆ fd„Zedefd„¦   «         Zd„ Ze	e
dej        dee         defd„¦   «         ¦   «         Zˆ xZS )	r  r*   c                 ón  •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        t          |¦  «        | _        t          |¦  «        | _        |  	                    dt          j        |j        t          j        ¬¦  «        d¬¦  «         |                      ¦   «          d S )Nr  r  Tr.   )r1   r2   r*   r)   Úinput_conditionerrG   Ú
embeddingsr'  Úencoderr3   r!   r4   r  r  r0  r6   s     €r'   r2   zRadioModel.__init__ž  s˜   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ!6°vÑ!>Ô!>ˆÔÝ.¨vÑ6Ô6ˆŒÝ# FÑ+Ô+ˆŒØ×Ò˜^­U¬\¸&Ô:MÕUZÔU_Ð-`Ñ-`Ô-`ÐmqÐÑrÔrÐrØ�ŠÑÔÐÐÐr&   r:   c                 ó   — | j         j        S r<   )r*   rL   rô   s    r'   rL   zRadioModel.patch_size§  s   € àŒ{Ô%Ð%r&   c                 óD   — | j         }t          j        ¦   «         | _         |S )zCDetach the input conditioner (caller applies normalization itself).)r5  r   rÿ   )r7   Úconditioners     r'   Úmake_preprocessor_externalz%RadioModel.make_preprocessor_external«  s   € àÔ,ˆÝ!#¤¡¤ˆÔØÐr&   r9   r£   c                 óh  — |                       |¦  «        }|                      |¦  «        } | j        |fi |¤Ž}|j        }| j        j        }|d d …d | j        j        …f         }|d d …| j        f                              d¦  «        }|d d …|d …f         }	t          ||	||j
        |j        ¬¦  «        S )Nr   )r   r   r   r   r   )r5  r6  r7  r   r*   Únum_summary_tokensrO   r  ru   r   r   r   )
r7   r9   r£   r   Úencoder_outputsr   Únum_skipÚall_summaryr   r   s
             r'   rA   zRadioModel.forward±  sØ   € ð ×-Ò-¨lÑ;Ô;ˆØŸš¨Ñ5Ô5ˆØ+7¨4¬<¸Ð+PÐ+PÈÐ+PÐ+PˆØ+Ô=Ðà”;Ô1ˆØ'¨¨¨Ð+G¨T¬[Ô-GÐ+GÐ(GÔHˆØ˜a˜a˜a Ô!2Ð2Ô3×;Ò;¸AÑ>Ô>ˆØ$ Q Q Q¨¨	¨	 \Ô2ˆåØØØ/Ø)Ô7Ø&Ô1ð
ñ 
ô 
ð 	
r&   )r   r   r   r   r2   Úpropertyr€   rL   r;  r   r   r!   rC   r   r   r   rA   rD   rE   s   @r'   r  r  œ  sÃ   ø€ € € € € ð˜{ð ð ð ð ð ð ð ð&˜Cð &ð &ð &ñ „Xð&ðð ð ð Øð
 E¤Lð 
¸FÐCUÔ<Vð 
Ð[kð 
ð 
ð 
ñ „^ñ Ôð
ð 
ð 
ð 
ð 
r&   r  )Nr›   )5Úcollections.abcr   Údataclassesr   r!   Útorch.nn.functionalr   rª   rs   Ú r   r  Úactivationsr   Úmodeling_layersr	   Úmodeling_outputsr
   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   Úutils.output_capturingr   Úconfiguration_radior   r   ÚModuler)   rG   r‚   r•   rC   r=   r¯   r±   rË   rÓ   rÛ   rè   r÷   r
  r'  r  Ú__all__r%   r&   r'   ú<module>rQ     s^  ðð* %Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø <Ð <Ð <Ð <Ð <Ð <Ð <Ð <Ø FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,ð ð7ð 7ð 7ð 7ð 7�{ñ 7ô 7ñ „ð7ð0
1ð 
1ð 
1ð 
1ð 
1˜BœIñ 
1ô 
1ð 
1ð13ð 13ð 13ð 13ð 13˜2œ9ñ 13ô 13ð 13ðhð ð ð ð ˆrŒyñ ô ð ð&+ð +ð +ð +ð +�b”iñ +ô +ð +ð !Øð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð �T‰\ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð:4.ð 4.ð 4.ð 4.ð 4.˜œñ 4.ô 4.ð 4.ðpð ð ð ð �b”iñ ô ð ð$ð ð ð ð �R”Yñ ô ð ð (ð (ð (ð (ð (�R”Yñ (ô (ð (ð"%ð %ð %ð %ð %�B”Iñ %ô %ð %ð0&ð &ð &ð &ð &Ð+ñ &ô &ð &ðR ð#fð #fð #fð #fð #f˜?ñ #fô #fñ „ð#fðL@ð @ð @ð @ð @Ð'ñ @ô @ð @ð ð'
ð '
ð '
ð '
ð '
Ð%ñ '
ô '
ñ „ð'
ðT Ð/Ð
0€€€r&   