§
    ‚ŠtjÚE  ã                   ó´  — d Z ddlZddlZddlmZ ddlmZ ddlmZ ddl	m
Z
 ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZmZmZmZmZmZmZmZmZmZm Z  ddl!m"Z" dZ# G d„ de¦  «        Z$ G d„ de¦  «        Z% G d„ de¦  «        Z& G d„ dej'        ¦  «        Z( G d„ de¦  «        Z) G d„ dej'        ¦  «        Z* G d„ de¦  «        Z+ G d„ d e¦  «        Z, G d!„ d"e¦  «        Z- G d#„ d$e¦  «        Z. G d%„ d&ej'        ¦  «        Z/e G d'„ d(e¦  «        ¦   «         Z0e G d)„ d*e0¦  «        ¦   «         Z1 G d+„ d,e¦  «        Z2 G d-„ d.e¦  «        Z3g d/¢Z4dS )0zPyTorch SEW model.é    N)Únné   )Úinitialization)ÚACT2FN)Úis_deepspeed_zero3_enabled)Úis_fsdp_managed_module)ÚBaseModelOutput)ÚPreTrainedModel)Úauto_docstring)Úis_flash_attention_requestedé   )ÚWav2Vec2AttentionÚWav2Vec2EncoderLayerÚWav2Vec2FeatureEncoderÚWav2Vec2FeedForwardÚWav2Vec2ForCTCÚ!Wav2Vec2ForSequenceClassificationÚWav2Vec2GroupNormConvLayerÚWav2Vec2LayerNormConvLayerÚWav2Vec2NoLayerNormConvLayerÚWav2Vec2SamePadLayerÚ_compute_mask_indicesé   )Ú	SEWConfigc                   ó   — e Zd ZdS )ÚSEWNoLayerNormConvLayerN©Ú__name__Ú
__module__Ú__qualname__© ó    úa/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/sew/modular_sew.pyr   r   0   ó   € € € € € Ø€Dr"   r   c                   ó   — e Zd ZdS )ÚSEWLayerNormConvLayerNr   r!   r"   r#   r&   r&   4   r$   r"   r&   c                   ó   — e Zd ZdS )ÚSEWGroupNormConvLayerNr   r!   r"   r#   r(   r(   8   r$   r"   r(   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚSEWPositionalConvEmbeddingc                 óÖ  •— t          ¦   «                              ¦   «          t          j        |j        |j        |j        |j        dz  |j        |j        ¬¦  «        | _        t          j	        j
        }t          t          j	        j        d¦  «        rt          j	        j        j
        }t          ¦   «         rëdd l}|j                             | j        j        d¬¦  «        5   || j        dd¬¦  «        | _        d d d ¦  «         n# 1 swxY w Y   t          | j        d¦  «        r-| j        j        j        j        }| j        j        j        j        }n| j        j        }| j        j        }|j                             | |¦  «         |j                             | |¦  «         n || j        dd¬¦  «        | _        t/          |j        ¦  «        | _        t2          |j                 | _        d S )	Nr   )Úkernel_sizeÚpaddingÚgroupsÚstrideÚweight_normr   ©Úmodifier_rankÚweight)ÚnameÚdimÚparametrizations)ÚsuperÚ__init__r   ÚConv1dÚhidden_sizeÚnum_conv_pos_embeddingsÚnum_conv_pos_embedding_groupsÚsqueeze_factorÚconvÚutilsr0   Úhasattrr6   r   Ú	deepspeedÚzeroÚGatheredParametersr3   Ú	original0Ú	original1Úweight_gÚweight_vÚregister_external_parameterÚSEWSamePadLayerr-   r   Úfeat_extract_activationÚ
activation)ÚselfÚconfigr0   rA   rF   rG   Ú	__class__s         €r#   r8   z#SEWPositionalConvEmbedding.__init__=   s  ø€ Ý‰Œ×ÒÑÔÐÝ”IØÔØÔØÔ6ØÔ2°aÑ7ØÔ7ØÔ(ð
ñ 
ô 
ˆŒ	õ ”hÔ*ˆÝ•2”8Ô,¨mÑ<Ô<ð 	@Ýœ(Ô3Ô?ˆKå%Ñ'Ô'ð 	EØÐÐÐà”×2Ò2°4´9Ô3CÐSTÐ2ÑUÔUð Ið IØ'˜K¨¬	¸ÀaÐHÑHÔH�”	ðIð Ið Iñ Iô Ið Ið Ið Ið Ið Ið Iøøøð Ið Ið Ið Iå�t”yÐ"4Ñ5Ô5ð .Øœ9Ô5Ô<ÔF�Øœ9Ô5Ô<ÔF��àœ9Ô-�Øœ9Ô-�ØŒN×6Ò6°t¸XÑFÔFÐFØŒN×6Ò6°t¸XÑFÔFÐFÐFà#˜ D¤I°HÀ!ÐDÑDÔDˆDŒIå& vÔ'EÑFÔFˆŒÝ  Ô!?Ô@ˆŒˆˆs   Ã DÄD	ÄD	c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S ©N)r>   r-   rK   )rL   Úhidden_statess     r#   Úforwardz"SEWPositionalConvEmbedding.forward_   s;   € ØŸ	š	 -Ñ0Ô0ˆØŸš ]Ñ3Ô3ˆØŸš¨Ñ6Ô6ˆàÐr"   ©r   r   r    r8   rR   Ú__classcell__©rN   s   @r#   r*   r*   <   sM   ø€ € € € € ð Að  Að  Að  Að  AðDð ð ð ð ð ð r"   r*   c                   ó   — e Zd ZdS )rI   Nr   r!   r"   r#   rI   rI   g   r$   r"   rI   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚSEWUpsamplingc                 óæ   •— t          ¦   «                              ¦   «          t          j        |j        |j        |j        z  ¦  «        | _        t          |j                 | _	        |j        | _        d S rP   )
r7   r8   r   ÚLinearr:   r=   Ú
projectionr   rJ   rK   ©rL   rM   rN   s     €r#   r8   zSEWUpsampling.__init__l   sZ   ø€ Ý‰Œ×ÒÑÔÐÝœ) FÔ$6¸Ô8JÈVÔMbÑ8bÑcÔcˆŒÝ  Ô!?Ô@ˆŒØ$Ô3ˆÔÐÐr"   c                 ó0  — |                       |¦  «        }|                      |¦  «        }| j        dk    r`|                     ¦   «         \  }}}|| j        z  }|| j        z  }|                     ||| j        |¦  «        }|                     |||¦  «        }|S )Nr   )r[   rK   r=   ÚsizeÚreshape)rL   rQ   ÚbszÚsrc_lenÚsrc_embed_dimÚtgt_lenÚtgt_embed_dims          r#   rR   zSEWUpsampling.forwardr   sž   € ØŸš¨Ñ6Ô6ˆØŸš¨Ñ6Ô6ˆàÔ Ò"Ð"à*7×*<Ò*<Ñ*>Ô*>Ñ'ˆC�˜-Ø Ô 3Ñ3ˆGØ)¨TÔ-@Ñ@ˆMØ)×1Ò1°#°wÀÔ@SÐUbÑcÔcˆMØ)×1Ò1°#°wÀÑNÔNˆMàÐr"   rS   rU   s   @r#   rX   rX   k   sG   ø€ € € € € ð4ð 4ð 4ð 4ð 4ðð ð ð ð ð ð r"   rX   c                   ó   — e Zd ZdS )ÚSEWFeatureEncoderNr   r!   r"   r#   rf   rf   �   r$   r"   rf   c                   ó   — e Zd ZdS )ÚSEWAttentionNr   r!   r"   r#   rh   rh   …   r$   r"   rh   c                   ó   — e Zd ZdS )ÚSEWFeedForwardNr   r!   r"   r#   rj   rj   ‰   r$   r"   rj   c                   ó   — e Zd ZdS )ÚSEWEncoderLayerNr   r!   r"   r#   rl   rl   �   r$   r"   rl   c                   ó.   ‡ — e Zd Zˆ fd„Z	 	 	 	 dd„Zˆ xZS )Ú
SEWEncoderc                 óò  •‡— t          ¦   «                              ¦   «          ‰| _        t          ‰¦  «        | _        t          j        ‰j        ‰j        ¦  «        | _        t          j	        ‰j
        ‰j        ¬¦  «        | _        t          j        ‰j        ¦  «        | _        t          j        ˆfd„t#          ‰j        ¦  «        D ¦   «         ¦  «        | _        t)          ‰¦  «        | _        d| _        d S )N©Úepsc                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r!   )rl   )Ú.0Ú_rM   s     €r#   ú
<listcomp>z'SEWEncoder.__init__.<locals>.<listcomp>™   s!   ø€ Ð$fÐ$fÐ$fÀ¥_°VÑ%<Ô%<Ð$fÐ$fÐ$fr"   F)r7   r8   rM   r*   Úpos_conv_embedr   Ú	AvgPool1dr=   ÚpoolÚ	LayerNormr:   Úlayer_norm_epsÚ
layer_normÚDropoutÚhidden_dropoutÚdropoutÚ
ModuleListÚrangeÚnum_hidden_layersÚlayersrX   ÚupsampleÚgradient_checkpointingr\   s    `€r#   r8   zSEWEncoder.__init__’   sÈ   øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ8¸Ñ@Ô@ˆÔÝ”L Ô!6¸Ô8MÑNÔNˆŒ	Ýœ, vÔ'9¸vÔ?TÐUÑUÔUˆŒÝ”z &Ô"7Ñ8Ô8ˆŒÝ”mÐ$fÐ$fÐ$fÐ$fÅeÈFÔLdÑFeÔFeÐ$fÑ$fÔ$fÑgÔgˆŒÝ% fÑ-Ô-ˆŒØ&+ˆÔ#Ð#Ð#r"   NFTc           	      ó^  — |rdnd }|rdnd }|��-|                      d¦  «                             dd|j        d         ¦  «        }t          | j        ¦  «        rd|| <   |�d|v r|nd }nÓd|| <   t
          j                             |                     ¦   «                               d¦  «        | j        j	        | j        j	        ¬¦  «         
                    d¦  «                             ¦   «         }d|d d …d d d d …f                              |j        ¬	¦  «        z
  }|t          j        |j        ¦  «        j        z  }|j        d         }	|                     dd¦  «        }|                      |¦  «        }
|                      |¦  «        }t!          |
                     d¦  «        |                     d¦  «        ¦  «        }|d
d |…f         |
d
d |…f         z   }|                     dd¦  «        }|                      |¦  «        }|                      |¦  «        }t/          ¦   «         pt1          | ¦  «        }| j        D ]a}|r||fz   }t          j        g ¦  «        }| j        o|| j        j        k     }|r|r ||||¬¦  «        }|d         }|rd}|r||d         fz   }Œb|r||fz   }|                      |¦  «        }|j        d         |	k     r2t
          j                             |ddd|	|j        d         z
  f¦  «        }|st?          d„ |||fD ¦   «         ¦  «        S tA          |||¬¦  «        S )Nr!   éÿÿÿÿr   r   ç        r   )r,   r/   g      ð?)Údtype.)Úattention_maskÚoutput_attentions©NNc              3   ó   K  — | ]}|®|V — Œ	d S rP   r!   )rs   Úvs     r#   ú	<genexpr>z%SEWEncoder.forward.<locals>.<genexpr>ï   s(   è è € ÐmÐm˜qÐ_`Ð_l˜Ð_lÐ_lÐ_lÐ_lÐmÐmr"   ©Úlast_hidden_staterQ   Ú
attentions)!Ú	unsqueezeÚrepeatÚshaper   rM   r   Ú
functionalÚ
max_pool1dÚfloatr=   ÚsqueezeÚlongÚtorˆ   ÚtorchÚfinfoÚminÚ	transposerv   rx   r^   r{   r~   r   r   r‚   ÚrandÚtrainingÚ	layerdroprƒ   ÚpadÚtupler	   )rL   rQ   r‰   rŠ   Úoutput_hidden_statesÚreturn_dictÚall_hidden_statesÚall_self_attentionsÚexpand_attention_maskÚn_input_timestepsÚposition_embeddingsÚpooled_hidden_statesÚ
min_lengthÚsynced_gpusÚlayerÚdropout_probabilityÚskip_the_layerÚlayer_outputss                     r#   rR   zSEWEncoder.forward�   s»  € ð #7Ð@˜B˜B¸DÐØ$5Ð?˜b˜b¸4ÐàÑ%Ø$2×$<Ò$<¸RÑ$@Ô$@×$GÒ$GÈÈ1ÈmÔNaÐbcÔNdÑ$eÔ$eÐ!Ý+¨D¬KÑ8Ô8ð Wà8;�Ð4Ð4Ñ5à4BÐ4NÐSTÐXfÐSfÐSf  Ðmq��ð 9<�Ð4Ð4Ñ5õ
 ”M×,Ò,Ø&×,Ò,Ñ.Ô.×8Ò8¸Ñ;Ô;Ø$(¤KÔ$>Ø#œ{Ô9ð -ñ ô ÷
 ’W˜Q‘Z”Zß’T‘V”Vð ð "% ~°a°a°a¸¸tÀQÀQÀQÐ6FÔ'G×'JÒ'JÐQ^ÔQdÐ'JÑ'eÔ'eÑ!e�Ø!/µ%´+¸mÔ>QÑ2RÔ2RÔ2VÑ!V�à)Ô/°Ô2Ðà%×/Ò/°°1Ñ5Ô5ˆØ"×1Ò1°-Ñ@Ô@ÐØ#Ÿyšy¨Ñ7Ô7ÐÝÐ,×1Ò1°"Ñ5Ô5Ð7K×7PÒ7PÐQSÑ7TÔ7TÑUÔUˆ
Ø,¨S°+°:°+Ð-=Ô>ÐATÐUXÐZeÐ[eÐZeÐUeÔAfÑfˆØ%×/Ò/°°1Ñ5Ô5ˆàŸš¨Ñ6Ô6ˆØŸš ]Ñ3Ô3ˆå0Ñ2Ô2ÐRÕ6LÈTÑ6RÔ6Rˆà”[ð 	Pð 	PˆEØ#ð IØ$5¸Ð8HÑ$HÐ!õ #(¤*¨R¡.¤.Ðà!œ]ÐZÐ/BÀTÄ[ÔEZÒ/ZˆNØ!ð 1 [ð 1à % Ø!°.ÐTeð!ñ !ô !�ð !.¨aÔ 0�àð -Ø ,�à ð PØ&9¸]È1Ô=MÐ<OÑ&OÐ#øàð 	EØ 1°]Ð4DÑ DÐàŸš mÑ4Ô4ˆØÔ˜qÔ!Ð$5Ò5Ð5ÝœM×-Ò-¨m¸aÀÀAÐGXÐ[hÔ[nÐopÔ[qÑGqÐ=rÑsÔsˆMàð 	nÝÐmÐm ]Ð4EÐGZÐ$[ÐmÑmÔmÑmÔmÐmÝØ+Ø+Ø*ð
ñ 
ô 
ð 	
r"   )NFFTrS   rU   s   @r#   rn   rn   ‘   sb   ø€ € € € € ð	,ð 	,ð 	,ð 	,ð 	,ð ØØ"ØðW
ð W
ð W
ð W
ð W
ð W
ð W
ð W
r"   rn   c                   ó¦   ‡ — e Zd ZU eed<   dZdZdZdZdZ	dZ
dZ ej        ¦   «         ˆ fd„¦   «         Zdej        ez  fd	„Zd
edej        fd„Zˆ xZS )ÚSEWPreTrainedModelrM   ÚsewÚinput_valuesÚaudioTFc           
      ó’  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rwt	          j        |j        j        ddt          j	        d|j        j
        d         |j        j        z  z  ¦  «        z  ¬¦  «         t	          j        |j        j        d¦  «         dS t          |t          j        ¦  «        rþt!          ¦   «         rÕddl}t%          |d¦  «        rlt%          |d¦  «        r\|j                             |j        |j        gd¬¦  «        5  t	          j        |j        ¦  «         ddd¦  «         dS # 1 swxY w Y   dS |j                             |j        d¬¦  «        5  t	          j        |j        ¦  «         ddd¦  «         dS # 1 swxY w Y   dS t	          j        |j        ¦  «         dS dS )	zInitialize the weightsr   r   r   )ÚmeanÚstdNrG   rF   r1   )r7   Ú_init_weightsÚ
isinstancer*   ÚinitÚnormal_r>   r3   ÚmathÚsqrtr,   Úin_channelsÚ	constant_Úbiasr   r9   r   rA   r@   rB   rC   rG   rF   Úkaiming_normal_)rL   ÚmodulerA   rN   s      €r#   rº   z SEWPreTrainedModel._init_weights  sG  ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ8Ñ9Ô9ð 	4ÝŒLØ”Ô"ØØ�œ	 ! v¤{Ô'>¸qÔ'AÀFÄKÔD[Ñ'[Ñ"\Ñ]Ô]Ñ]ðñ ô ð õ
 ŒN˜6œ;Ô+¨QÑ/Ô/Ð/Ð/Ð/Ý˜¥¤	Ñ*Ô*ð 	4Ý)Ñ+Ô+ð 
4Ø Ð Ð Ð å˜6 :Ñ.Ô.ð <µ7¸6À:Ñ3NÔ3Nð <Ø"œ×:Ò:¸F¼OÈVÌ_Ð;]ÐmnÐ:ÑoÔoð <ð <ÝÔ,¨V¬]Ñ;Ô;Ð;ð<ð <ð <ñ <ô <ð <ð <ð <ð <ð <ð <ð <øøøð <ð <ð <ð <ð <ð <ð #œ×:Ò:¸6¼=ÐXYÐ:ÑZÔZð <ð <ÝÔ,¨V¬]Ñ;Ô;Ð;ð<ð <ð <ñ <ô <ð <ð <ð <ð <ð <ð <ð <øøøð <ð <ð <ð <ð <ð <õ Ô$ V¤]Ñ3Ô3Ð3Ð3Ð3ð	4ð 	4s$   Ä#E
Å
EÅEÅ8FÆF#Æ&F#Úinput_lengthsc                 óz   — d„ }t          | j        j        | j        j        ¦  «        D ]\  }} ||||¦  «        }Œ|S )zH
        Computes the output length of the convolutional layers
        c                 ó<   — t          j        | |z
  |d¬¦  «        dz   S )NÚfloor)Úrounding_moder   )r›   Údiv)Úinput_lengthr,   r/   s      r#   Ú_conv_out_lengthzMSEWPreTrainedModel._get_feat_extract_output_lengths.<locals>._conv_out_length  s&   € õ ”9˜\¨KÑ7¸ÈwÐWÑWÔWÐZ[Ñ[Ð[r"   )ÚziprM   Úconv_kernelÚconv_stride)rL   rÅ   rÌ   r,   r/   s        r#   Ú _get_feat_extract_output_lengthsz3SEWPreTrainedModel._get_feat_extract_output_lengths  s\   € ð
	\ð 	\ð 	\õ
 $' t¤{Ô'>ÀÄÔ@WÑ#XÔ#Xð 	Qð 	QÑˆK˜Ø,Ð,¨]¸KÈÑPÔPˆMˆMàÐr"   Úfeature_vector_lengthr‰   c                 ó  — |                       |                     d¦  «        ¦  «                             t          j        ¦  «        }t          j        ||j        ¬¦  «        }|                     d¦  «        |                     d¦  «        k     S )Nr†   )Údevicer   r   )rÐ   Úsumrš   r›   r™   ÚarangerÓ   r’   )rL   rÑ   r‰   Úoutput_lengthsÚattention_idss        r#   Ú"_get_feature_vector_attention_maskz5SEWPreTrainedModel._get_feature_vector_attention_mask)  sw   € Ø×>Ò>¸~×?QÒ?QÐRTÑ?UÔ?UÑVÔV×YÒYÕZ_ÔZdÑeÔeˆõ œÐ%:À>ÔCXÐYÑYÔYˆØ×&Ò& qÑ)Ô)¨N×,DÒ,DÀQÑ,GÔ,GÒGÐGr"   )r   r   r    r   Ú__annotations__Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnr›   Úno_gradrº   Ú
LongTensorÚintrÐ   rØ   rT   rU   s   @r#   r³   r³   ÷   sÔ   ø€ € € € € € àÐÐÑØÐØ$€OØÐØ&*Ð#ØÐØ€NØÐà€U„]�_„_ð4ð 4ð 4ð 4ñ „_ð4ð.¸eÔ>NÐQTÑ>Tð ð ð ð ðHÈð HÐ]bÔ]mð Hð Hð Hð Hð Hð Hð Hð Hr"   r³   c                   óð   ‡ — e Zd Zdefˆ fd„Z	 	 ddej        dej        dz  dej        dz  fd„Ze		 	 	 	 	 ddej
        dz  dej
        dz  dej        dz  d	edz  d
edz  dedz  deez  fd„¦   «         Zˆ xZS )ÚSEWModelrM   c                 óª  •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        t          j        |j        d         |j        ¬¦  «        | _	        |j        d         |j
        k    | _        | j        r*t          j        |j        d         |j
        ¦  «        | _        t          j        |j        ¦  «        | _        |j        dk    s|j        dk    rBt          j        t)          j        |j
        ¦  «                             ¦   «         ¦  «        | _        t1          |¦  «        | _        |                      ¦   «          d S )Nr†   rp   r‡   )r7   r8   rM   rf   Úfeature_extractorr   ry   Úconv_dimrz   r{   r:   Úproject_featuresrZ   Úfeature_projectionr|   Úfeat_proj_dropoutÚfeature_dropoutÚmask_time_probÚmask_feature_probÚ	Parameterr›   ÚTensorÚuniform_Úmasked_spec_embedrn   ÚencoderÚ	post_initr\   s     €r#   r8   zSEWModel.__init__4  s  ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ!2°6Ñ!:Ô!:ˆÔÝœ, v¤°rÔ':ÀÔ@UÐVÑVÔVˆŒà &¤°Ô 3°vÔ7IÒ IˆÔØÔ ð 	YÝ&(¤i°´ÀÔ0CÀVÔEWÑ&XÔ&XˆDÔ#Ý!œz¨&Ô*BÑCÔCˆÔàÔ  3Ò&Ð&¨&Ô*BÀSÒ*HÐ*HÝ%'¤\µ%´,¸vÔ?QÑ2RÔ2R×2[Ò2[Ñ2]Ô2]Ñ%^Ô%^ˆDÔ"å! &Ñ)Ô)ˆŒð 	�ŠÑÔÐÐÐr"   NrQ   Úmask_time_indicesr‰   c                 ó  — t          | j        dd¦  «        s|S |                     ¦   «         \  }}}|�#| j                             |j        ¦  «        ||<   n“| j        j        dk    rƒ| j        r|t          ||f| j        j        | j        j	        || j        j
        ¬¦  «        }t          j        ||j        t          j        ¬¦  «        }| j                             |j        ¦  «        ||<   | j        j        dk    r†| j        rt          ||f| j        j        | j        j        | j        j        ¬¦  «        }t          j        ||j        t          j        ¬¦  «        }|dd…df                              d|d¦  «        }d||<   |S )	z¢
        Masks extracted features along time axis and/or along feature axis according to
        [SpecAugment](https://huggingface.co/papers/1904.08779).
        Úapply_spec_augmentTNr   )Ú	mask_probÚmask_lengthr‰   Ú	min_masks)rÓ   rˆ   )rø   rù   rú   r†   )ÚgetattrrM   r^   rò   rš   rˆ   rí   r    r   Úmask_time_lengthÚmask_time_min_masksr›   ÚtensorrÓ   Úboolrî   Úmask_feature_lengthÚmask_feature_min_masksÚexpand)rL   rQ   rõ   r‰   Ú
batch_sizeÚsequence_lengthr:   Úmask_feature_indicess           r#   Ú_mask_hidden_stateszSEWModel._mask_hidden_statesH  s—  € õ �t”{Ð$8¸$Ñ?Ô?ð 	!Ø Ð ð 4A×3EÒ3EÑ3GÔ3GÑ0ˆ
�O [àÐ(à/3Ô/E×/HÒ/HÈÔI\Ñ/]Ô/]ˆMÐ+Ñ,Ð,ØŒ[Ô'¨!Ò+Ð+°´Ð+Ý 5Ø˜_Ð-Øœ+Ô4Ø œKÔ8Ø-Øœ+Ô9ð!ñ !ô !Ðõ !&¤Ð->À}ÔG[ÕchÔcmÐ nÑ nÔ nÐØ/3Ô/E×/HÒ/HÈÔI\Ñ/]Ô/]ˆMÐ+Ñ,àŒ;Ô(¨1Ò,Ð,°´Ð,å#8Ø˜[Ð)Øœ+Ô7Ø œKÔ;Øœ+Ô<ð	$ñ $ô $Ð õ $)¤<Ð0DÈ]ÔMaÕinÔisÐ#tÑ#tÔ#tÐ Ø#7¸¸¸¸4¸Ô#@×#GÒ#GÈÈOÐ]_Ñ#`Ô#`Ð Ø23ˆMÐ.Ñ/àÐr"   rµ   rŠ   r¤   r¥   Úreturnc                 óT  — |�|n| j         j        }|�|n| j         j        }|�|n| j         j        }|                      |¦  «        }|                     dd¦  «        }|                      |¦  «        }| j        r|                      |¦  «        }|  	                    |¦  «        }	|�!|  
                    |	j        d         |¦  «        }|                      |	|¬¦  «        }	|                      |	||||¬¦  «        }
|
d         }	|s|	f|
dd…         z   S t          |	|
j        |
j        ¬¦  «        S )a/  
        mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict
            masked extracted features in *config.proj_codevector_dim* space.
        Nr   r   )rõ   )r‰   rŠ   r¤   r¥   r   r�   )rM   rŠ   r¤   r¥   rç   rž   r{   ré   rê   rì   rØ   r”   r  ró   r	   rQ   r‘   )rL   rµ   r‰   rõ   rŠ   r¤   r¥   ÚkwargsÚextract_featuresrQ   Úencoder_outputss              r#   rR   zSEWModel.forwardv  sr  € ð  2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆà×1Ò1°,Ñ?Ô?ÐØ+×5Ò5°a¸Ñ;Ô;ÐØŸ?š?Ð+;Ñ<Ô<ÐàÔ ð 	IØ#×6Ò6Ð7GÑHÔHÐØ×,Ò,Ð-=Ñ>Ô>ˆàÐ%à!×DÒDÀ]ÔEXÐYZÔE[Ð]kÑlÔlˆNà×0Ò0°ÐRcÐ0ÑdÔdˆàŸ,š,ØØ)Ø/Ø!5Ø#ð 'ñ 
ô 
ˆð (¨Ô*ˆàð 	:Ø!Ð# o°a°b°bÔ&9Ñ9Ð9åØ+Ø)Ô7Ø&Ô1ð
ñ 
ô 
ð 	
r"   r‹   )NNNNN)r   r   r    r   r8   r›   ÚFloatTensorrâ   r  r   rð   rÿ   r£   r	   rR   rT   rU   s   @r#   rå   rå   2  s6  ø€ € € € € ð˜yð ð ð ð ð ð ð. 7;Ø26ð	,ð ,àÔ(ð,ð !Ô,¨tÑ3ð,ð Ô(¨4Ñ/ð	,ð ,ð ,ð ,ð\ ð /3Ø6:Ø)-Ø,0Ø#'ð4
ð 4
à”l TÑ)ð4
ð œ tÑ+ð4
ð !Ô,¨tÑ3ð	4
ð
   $™;ð4
ð # T™kð4
ð ˜D‘[ð4
ð 
�Ñ	 ð4
ð 4
ð 4
ñ „^ð4
ð 4
ð 4
ð 4
ð 4
r"   rå   c                   ó   — e Zd ZdS )Ú	SEWForCTCNr   r!   r"   r#   r  r  ®  r$   r"   r  c                   ó   — e Zd ZdS )ÚSEWForSequenceClassificationNr   r!   r"   r#   r  r  ²  r$   r"   r  )r  r  rå   r³   )5Ú__doc__r¾   r›   r   Ú r   r¼   Úactivationsr   Úintegrations.deepspeedr   Úintegrations.fsdpr   Úmodeling_outputsr	   Úmodeling_utilsr
   r?   r   Úutils.genericr   Úwav2vec2.modeling_wav2vec2r   r   r   r   r   r   r   r   r   r   r   Úconfiguration_sewr   Ú_HIDDEN_STATES_START_POSITIONr   r&   r(   ÚModuler*   rI   rX   rf   rh   rj   rl   rn   r³   rå   r  r  Ú__all__r!   r"   r#   ú<module>r     s+  ðð Ð à €€€à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø @Ð @Ð @Ð @Ð @Ð @Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø /Ð /Ð /Ð /Ð /Ð /Ø -Ð -Ð -Ð -Ð -Ð -Ø #Ð #Ð #Ð #Ð #Ð #Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð )Ð (Ð (Ð (Ð (Ð (ð !"Ð ð	ð 	ð 	ð 	ð 	Ð:ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð6ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð6ñ 	ô 	ð 	ð(ð (ð (ð (ð ( ¤ñ (ô (ð (ðV	ð 	ð 	ð 	ð 	Ð*ñ 	ô 	ð 	ðð ð ð ð �B”Iñ ô ð ð,	ð 	ð 	ð 	ð 	Ð.ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð$ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð(ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð*ñ 	ô 	ð 	ðc
ð c
ð c
ð c
ð c
�”ñ c
ô c
ð c
ðL ð7Hð 7Hð 7Hð 7Hð 7H˜ñ 7Hô 7Hñ „ð7Hðt ðx
ð x
ð x
ð x
ð x
Ð!ñ x
ô x
ñ „ðx
ðv	ð 	ð 	ð 	ð 	�ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð#Dñ 	ô 	ð 	ð ZÐ
YÐ
Y€€€r"   