§
    ‚ŠtjYC  ã                   óˆ  — d Z ddlZddlmZ ddlZddlmZ ddlmZ	 ddl
mZmZ ddlmZ ddlmZmZ d	d
lmZmZmZmZmZmZmZmZmZ ddlmZ  ej        e¦  «        Z  ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z! G d„ de¦  «        Z" G d„ de¦  «        Z# G d„ de¦  «        Z$ G d„ de¦  «        Z% G d„ de¦  «        Z& G d„ de¦  «        Z'e G d„ de¦  «        ¦   «         Z(eZ) G d„ d e(e¦  «        Z* ed!¬¦  «         G d"„ d#e(¦  «        ¦   «         Z+ G d$„ d%e¦  «        Z, G d&„ d'e¦  «        Z-g d(¢Z.dS ))zPyTorch UniSpeech model.é    N)Ú	dataclassé   )Úinitialization)ÚModelOutputÚWav2Vec2BaseModelOutput)ÚPreTrainedModel)Úauto_docstringÚloggingé   )	ÚWav2Vec2EncoderÚWav2Vec2EncoderStableLayerNormÚWav2Vec2FeatureEncoderÚWav2Vec2FeatureProjectionÚWav2Vec2ForCTCÚ!Wav2Vec2ForSequenceClassificationÚWav2Vec2GumbelVectorQuantizerÚWav2Vec2ModelÚWav2Vec2PositionalConvEmbeddingé   )ÚUniSpeechConfigzh
    Output type of [`UniSpeechForPreTrainingOutput`], with potential hidden states and attentions.
    )Úcustom_introc                   óà   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	ej        dz  ed<   dZ
ej        dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dS )	ÚUniSpeechForPreTrainingOutputaÝ  
    loss (*optional*, returned when model is in train mode, `torch.FloatTensor` of shape `(1,)`):
        Total loss as the sum of the contrastive loss (L_m) and the diversity loss (L_d) as stated in the [official
        paper](https://huggingface.co/papers/2006.11477).
    projected_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.proj_codevector_dim)`):
        Hidden-states of the model projected to *config.proj_codevector_dim* that can be used to predict the masked
        projected quantized states.
    projected_quantized_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.proj_codevector_dim)`):
        Quantized extracted feature vectors projected to *config.proj_codevector_dim* representing the positive
        target vectors for contrastive loss.
    codevector_perplexity (`torch.FloatTensor` of shape `(1,)`):
        The perplexity of the codevector distribution, used to measure the diversity of the codebook.
    NÚlossÚprojected_statesÚprojected_quantized_statesÚcodevector_perplexityÚhidden_statesÚ
attentions)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚtorchÚFloatTensorÚ__annotations__r   r   r   r   Útupler   © ó    úm/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/unispeech/modular_unispeech.pyr   r   +   s»   € € € € € € ðð ð &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø15Ð�eÔ'¨$Ñ.Ð5Ð5Ñ5Ø;?Ð Ô 1°DÑ 8Ð?Ð?Ñ?Ø6:Ð˜5Ô,¨tÑ3Ð:Ð:Ñ:Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ð6Ð6r)   r   c                   ó   — e Zd ZdS )Ú UniSpeechPositionalConvEmbeddingN©r    r!   r"   r(   r)   r*   r,   r,   H   ó   € € € € € Ø€Dr)   r,   c                   ó   — e Zd ZdS )ÚUniSpeechFeatureEncoderNr-   r(   r)   r*   r0   r0   L   r.   r)   r0   c                   ó   — e Zd ZdS )ÚUniSpeechFeatureProjectionNr-   r(   r)   r*   r2   r2   P   r.   r)   r2   c                   ó   — e Zd ZdS )ÚUniSpeechEncoderNr-   r(   r)   r*   r4   r4   T   r.   r)   r4   c                   ó   — e Zd ZdS )ÚUniSpeechEncoderStableLayerNormNr-   r(   r)   r*   r6   r6   X   r.   r)   r6   c                   ó*   — e Zd Zed„ ¦   «         Zd„ ZdS )ÚUniSpeechGumbelVectorQuantizerc                 óÎ   — |                       d¬¦  «        }t          j        t          j        t          j        ||¦  «        d¬¦  «         ¦  «                             ¦   «         }|S )Nr   ©Údiméÿÿÿÿ)Úmeanr$   ÚexpÚsumÚxlogy)ÚprobsÚmarginal_probsÚ
perplexitys      r*   Ú_compute_perplexityz2UniSpeechGumbelVectorQuantizer._compute_perplexity]   sU   € àŸš¨˜Ñ*Ô*ˆÝ”Y¥¤	­%¬+°nÀnÑ*UÔ*UÐ[]Ð ^Ñ ^Ô ^Ð^Ñ_Ô_×cÒcÑeÔeˆ
ØÐr)   c                 óö  — |j         \  }}}|                      |¦  «        }|                     ||z  | j        z  d¦  «        }| j        r§t
          j                             |                     ¦   «         | j	        d¬¦  «         
                    |¦  «        }t          j        |                     ||z  | j        d¦  «                             ¦   «         d¬¦  «        }|                      |¦  «        }n‚|                     d¬¦  «        } |j        |j         Ž                      d|                     dd¦  «        d¦  «        }|                     ||z  | j        d¦  «        }|                      |¦  «        }|                     ||z  d¦  «        }|                     d¦  «        | j        z  }	|	                     ||z  | j        | j        d¦  «        }
|
                     d¦  «                             ||d¦  «        }
|
|fS )Nr<   T)ÚtauÚhardr:   r   g      ð?éþÿÿÿ)ÚshapeÚweight_projÚviewÚ
num_groupsÚtrainingÚnnÚ
functionalÚgumbel_softmaxÚfloatÚtemperatureÚtype_asr$   ÚsoftmaxrD   ÚargmaxÚ	new_zerosÚscatter_Ú	unsqueezeÚcodevectorsÚnum_varsr?   )Úselfr   Ú
batch_sizeÚsequence_lengthÚhidden_sizeÚcodevector_probsÚcodevector_soft_distrC   Úcodevector_idxÚcodevectors_per_grouprY   s              r*   Úforwardz&UniSpeechGumbelVectorQuantizer.forwardc   s  € Ø3@Ô3FÑ0ˆ
�O [ð ×(Ò(¨Ñ7Ô7ˆØ%×*Ò*¨:¸Ñ+GÈ$Ì/Ñ+YÐ[]Ñ^Ô^ˆàŒ=ð 	Då!œ}×;Ò;Ø×#Ò#Ñ%Ô%¨4Ô+;À$ð  <ñ  ô  çŠg�mÑ$Ô$ð õ
 $)¤=Ø×"Ò" :°Ñ#?ÀÄÐRTÑUÔU×[Ò[Ñ]Ô]Ðceð$ñ $ô $Ð ð ×1Ò1Ð2FÑGÔGˆJˆJð +×1Ò1°bÐ1Ñ9Ô9ˆNØ6˜}Ô6¸Ô8KÐL×UÒUØ�N×'Ò'¨¨AÑ.Ô.°ñ ô  Ðð  0×4Ò4°ZÀ/Ñ5QÐSWÔSbÐdfÑgÔgÐà×1Ò1Ð2BÑCÔCˆJà+×0Ò0°¸oÑ1MÈrÑRÔRÐà 0× :Ò :¸2Ñ >Ô >ÀÔAQÑ QÐØ+×0Ò0°¸oÑ1MÈtÌÐ`dÔ`mÐoqÑrÔrˆØ!—o’o bÑ)Ô)×.Ò.¨z¸?ÈBÑOÔOˆà˜JÐ&Ð&r)   N)r    r!   r"   ÚstaticmethodrD   rc   r(   r)   r*   r8   r8   \   s<   € € € € € Øðð ñ „\ðð
#'ð #'ð #'ð #'ð #'r)   r8   c                   ó¦   ‡ — e Zd ZU eed<   dZdZdZdZdZ	dZ
dZ ej        ¦   «         ˆ fd„¦   «         Zdej        ez  fd„Zd	ed
ej        fd„Zˆ xZS )ÚUniSpeechPreTrainedModelÚconfigÚ	unispeechÚinput_valuesÚaudioTc           
      óL  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rZt	          j        |j        j        dd¬¦  «         t	          j        |j        j	        ¦  «         t	          j
        |j        ¦  «         dS t          |t          ¦  «        rwt	          j        |j        j        ddt          j        d|j        j        d         |j        j        z  z  ¦  «        z  ¬¦  «         t	          j        |j        j	        d¦  «         dS t          |t&          ¦  «        rgt          j        d|j        j        z  ¦  «        }t	          j
        |j        j        | |¬¦  «         t	          j
        |j        j	        | |¬¦  «         dS t          |t,          j        ¦  «        rnt	          j        |j        ¦  «         |j	        �Pt          j        |j        |j        |j        d         z  z  ¦  «        }t	          j
        |j	        | |¬¦  «         dS dS dS )zInitialize the weightsç        r   )r=   Ústdr   r   )ÚaÚbN)ÚsuperÚ_init_weightsÚ
isinstancer8   ÚinitÚnormal_rJ   ÚweightÚzeros_ÚbiasÚuniform_rY   r,   ÚconvÚmathÚsqrtÚkernel_sizeÚin_channelsÚ	constant_r2   Ú
projectionÚin_featuresrN   ÚConv1dÚkaiming_normal_Úgroups)r[   ÚmoduleÚkÚ	__class__s      €r*   rq   z&UniSpeechPreTrainedModel._init_weights”   s÷  ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%å�fÕ<Ñ=Ô=ð 	6ÝŒL˜Ô+Ô2¸À!ÐDÑDÔDÐDÝŒK˜Ô*Ô/Ñ0Ô0Ð0ÝŒM˜&Ô,Ñ-Ô-Ð-Ð-Ð-Ý˜Õ @ÑAÔAð 	6ÝŒLØ”Ô"ØØ�œ	 ! v¤{Ô'>¸qÔ'AÀFÄKÔD[Ñ'[Ñ"\Ñ]Ô]Ñ]ðñ ô ð õ
 ŒN˜6œ;Ô+¨QÑ/Ô/Ð/Ð/Ð/Ý˜Õ :Ñ;Ô;ð 		6Ý”	˜!˜fÔ/Ô;Ñ;Ñ<Ô<ˆAÝŒM˜&Ô+Ô2°q°b¸AÐ>Ñ>Ô>Ð>ÝŒM˜&Ô+Ô0°Q°B¸!Ð<Ñ<Ô<Ð<Ð<Ð<Ý˜¥¤	Ñ*Ô*ð 	6ÝÔ  ¤Ñ/Ô/Ð/àŒ{Ð&Ý”I˜fœm¨vÔ/AÀFÔDVÐWXÔDYÑ/YÑZÑ[Ô[�Ý”˜fœk¨a¨R°1Ð5Ñ5Ô5Ð5Ð5Ð5ð	6ð 	6ð 'Ð&r)   Úinput_lengthsc                 óz   — d„ }t          | j        j        | j        j        ¦  «        D ]\  }} ||||¦  «        }Œ|S )zH
        Computes the output length of the convolutional layers
        c                 ó<   — t          j        | |z
  |d¬¦  «        dz   S )NÚfloor)Úrounding_moder   )r$   Údiv)Úinput_lengthr|   Ústrides      r*   Ú_conv_out_lengthzSUniSpeechPreTrainedModel._get_feat_extract_output_lengths.<locals>._conv_out_length´   s&   € õ ”9˜\¨KÑ7¸ÈwÐWÑWÔWÐZ[Ñ[Ð[r)   )Úziprg   Úconv_kernelÚconv_stride)r[   r‡   r�   r|   rŽ   s        r*   Ú _get_feat_extract_output_lengthsz9UniSpeechPreTrainedModel._get_feat_extract_output_lengths¯   s\   € ð
	\ð 	\ð 	\õ
 $' t¤{Ô'>ÀÄÔ@WÑ#XÔ#Xð 	Qð 	QÑˆK˜Ø,Ð,¨]¸KÈÑPÔPˆMˆMàÐr)   Úfeature_vector_lengthÚattention_maskc                 ó  — |                      d¬¦  «        d d …df         }|                      |¦  «                             t          j        ¦  «        }|j        d         }t          j        ||f|j        |j        ¬¦  «        }d|t          j	        |j        d         |j        ¬¦  «        |dz
  f<   | 
                    dg¦  «                              d¦  «         
                    dg¦  «                             ¦   «         }|S )Nr<   r:   r   )ÚdtypeÚdevicer   )r˜   )Úcumsumr“   Útor$   ÚlongrI   Úzerosr—   r˜   ÚarangeÚflipÚbool)r[   r”   r•   Únon_padded_lengthsÚoutput_lengthsr\   s         r*   Ú"_get_feature_vector_attention_maskz;UniSpeechPreTrainedModel._get_feature_vector_attention_mask¾   sþ   € ð ,×2Ò2°rÐ2Ñ:Ô:¸1¸1¸1¸b¸5ÔAÐØ×>Ò>Ð?QÑRÔR×UÒUÕV[ÔV`ÑaÔaˆØ#Ô)¨!Ô,ˆ
åœØÐ.Ð/°~Ô7KÐTbÔTið
ñ 
ô 
ˆð uvˆ�œ ^Ô%9¸!Ô%<À^ÔEZÐ[Ñ[Ô[Ð]kÐnoÑ]oÐpÑqØ'×,Ò,¨b¨TÑ2Ô2×9Ò9¸"Ñ=Ô=×BÒBÀBÀ4ÑHÔH×MÒMÑOÔOˆØÐr)   )r    r!   r"   r   r&   Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnr$   Úno_gradrq   Ú
LongTensorÚintr“   r¢   Ú__classcell__©r†   s   @r*   rf   rf   ‰   sÊ   ø€ € € € € € àÐÐÑØ#ÐØ$€OØÐØ&*Ð#ØÐØ€NØÐà€U„]�_„_ð6ð 6ð 6ð 6ñ „_ð6ð4¸eÔ>NÐQTÑ>Tð ð ð ð ðÈð Ð]bÔ]mð ð ð ð ð ð ð ð r)   rf   c                   ó˜   — e Zd Zdefd„Zd„ Z	 	 	 	 	 ddej        dz  dej        dz  dej        dz  de	dz  d	e	dz  d
e	dz  de
ez  fd„ZdS )ÚUniSpeechModelrg   c                 óÒ  — t                                | |¦  «         || _        t          |¦  «        | _        t          |¦  «        | _        |j        dk    s|j        dk    rBt          j
        t          j        |j        ¦  «                             ¦   «         ¦  «        | _        |j        rt#          |¦  «        | _        nt'          |¦  «        | _        |                      ¦   «          d S )Nrl   )rf   Ú__init__rg   r0   Úfeature_extractorr2   Úfeature_projectionÚmask_time_probÚmask_feature_probrN   Ú	Parameterr$   ÚTensorr^   rx   Úmasked_spec_embedÚdo_stable_layer_normr6   Úencoderr4   Ú	post_init)r[   rg   s     r*   r²   zUniSpeechModel.__init__Ò   sÈ   € Ý ×)Ò)¨$°Ñ7Ô7Ð7ØˆŒÝ!8¸Ñ!@Ô!@ˆÔÝ"<¸VÑ"DÔ"DˆÔàÔ  3Ò&Ð&¨&Ô*BÀSÒ*HÐ*HÝ%'¤\µ%´,¸vÔ?QÑ2RÔ2R×2[Ò2[Ñ2]Ô2]Ñ%^Ô%^ˆDÔ"àÔ&ð 	4Ý:¸6ÑBÔBˆDŒLˆLå+¨FÑ3Ô3ˆDŒLð 	�ŠÑÔÐÐÐr)   c                 ó    — t          d¦  «        ‚)NzNot needed for UniSpeech)ÚAttributeError©r[   s    r*   Úfreeze_feature_encoderz%UniSpeechModel.freeze_feature_encoderã   s   € ÝÐ7Ñ8Ô8Ð8r)   Nri   r•   Úmask_time_indicesÚoutput_attentionsÚoutput_hidden_statesÚreturn_dictÚreturnc                 óþ  — |�|n| j         j        }|�|n| j         j        }|�|n| j         j        }|                      |¦  «        }|                     dd¦  «        }|�!|                      |j        d         |¦  «        }|                      |¦  «        \  }	}|  	                    |	||¬¦  «        }	|  
                    |	||||¬¦  «        }
|
d         }	|s|	|f|
dd…         z   S t          |	||
j        |
j        ¬¦  «        S )a/  
        mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict
            masked extracted features in *config.proj_codevector_dim* space.
        Nr   r   )rÁ   r•   ©r•   rÂ   rÃ   rÄ   r   )Úlast_hidden_stateÚextract_featuresr   r   )rg   rÂ   rÃ   rÄ   r³   Ú	transposer¢   rI   r´   Ú_mask_hidden_statesr»   ÚUniSpeechBaseModelOutputr   r   )r[   ri   r•   rÁ   rÂ   rÃ   rÄ   ÚkwargsrÉ   r   Úencoder_outputss              r*   rc   zUniSpeechModel.forwardæ   sW  € ð 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆà×1Ò1°,Ñ?Ô?ÐØ+×5Ò5°a¸Ñ;Ô;ÐàÐ%à!×DÒDÐEUÔE[Ð\]ÔE^Ð`nÑoÔoˆNà*.×*AÒ*AÐBRÑ*SÔ*SÑ'ˆÐ'Ø×0Ò0ØÐ->È~ð 1ñ 
ô 
ˆð Ÿ,š,ØØ)Ø/Ø!5Ø#ð 'ñ 
ô 
ˆð (¨Ô*ˆàð 	KØ!Ð#3Ð4°ÀqÀrÀrÔ7JÑJÐJå'Ø+Ø-Ø)Ô7Ø&Ô1ð	
ñ 
ô 
ð 	
r)   )NNNNN)r    r!   r"   r   r²   rÀ   r$   r¸   r%   rŸ   r'   rÌ   rc   r(   r)   r*   r°   r°   Ñ   sÔ   € € € € € ð˜ð ð ð ð ð"9ð 9ð 9ð /3Ø6:Ø)-Ø,0Ø#'ð3
ð 3
à”l TÑ)ð3
ð œ tÑ+ð3
ð !Ô,¨tÑ3ð	3
ð
   $™;ð3
ð # T™kð3
ð ˜D‘[ð3
ð 
Ð)Ñ	)ð3
ð 3
ð 3
ð 3
ð 3
ð 3
r)   r°   zZ
    UniSpeech Model with a vector-quantization module and ctc loss for pre-training.
    c                   óò   ‡ — e Zd Zdefˆ fd„Zdefd„Zd„ Ze	 dde	j
        de	j
        d	e	j
        defd
„¦   «         Ze	 	 	 	 dde	j        dz  de	j        dz  dedz  dedz  dedz  deez  fd„¦   «         Zˆ xZS )ÚUniSpeechForPreTrainingrg   c                 ó  •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        ¦  «        | _        t          |¦  «        | _	        t	          j
        |j        |j        ¦  «        | _        t	          j
        |j        |j        ¦  «        | _        t	          j
        |j        |j        ¦  «        | _        t	          j        |j        ¦  «        | _        |                      ¦   «          d S )N)rp   r²   r°   rh   rN   ÚDropoutÚfeat_quantizer_dropoutÚdropout_featuresr8   Ú	quantizerÚLinearÚcodevector_dimÚproj_codevector_dimÚ	project_qr^   Úproject_hidÚnum_ctc_classesÚctc_projÚfinal_dropoutÚdropoutr¼   )r[   rg   r†   s     €r*   r²   z UniSpeechForPreTraining.__init__"  sÆ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý'¨Ñ/Ô/ˆŒÝ "¤
¨6Ô+HÑ IÔ IˆÔå7¸Ñ?Ô?ˆŒÝœ 6Ô#8¸&Ô:TÑUÔUˆŒÝœ9 VÔ%?ÀÔASÑTÔTˆÔåœ	 &Ô"4°fÔ6LÑMÔMˆŒÝ”z &Ô"6Ñ7Ô7ˆŒð 	�ŠÑÔÐÐÐr)   rR   c                 ó   — || j         _        dS )zb
        Set the Gumbel softmax temperature to a given value. Only necessary for training
        N)rÕ   rR   )r[   rR   s     r*   Úset_gumbel_temperaturez.UniSpeechForPreTraining.set_gumbel_temperature1  s   € ð &1ˆŒÔ"Ð"Ð"r)   c                 óB   — | j         j                             ¦   «          dS )z¨
        Calling this function will disable the gradient computation for the feature encoder so that its parameter will
        not be updated during training.
        N)rh   r³   Ú_freeze_parametersr¿   s    r*   rÀ   z.UniSpeechForPreTraining.freeze_feature_encoder7  s!   € ð
 	ŒÔ(×;Ò;Ñ=Ô=Ð=Ð=Ð=r)   r   Útarget_featuresÚnegative_featuresÚpredicted_featuresc                 óà   — t          j        | |gd¬¦  «        } t          j        |                     ¦   «         |                      ¦   «         d¬¦  «        }|                     | ¦  «        }||z  }|S )zé
        Compute logits for contrastive loss based using cosine similarity as the distance measure between
        `[positive_feature, negative_features]` and `[predicted_features]`. Additionally, temperature can be applied.
        r   r:   r<   )r$   ÚcatÚcosine_similarityrQ   rS   )rã   rä   rå   rR   Úlogitss        r*   Úcompute_contrastive_logitsz2UniSpeechForPreTraining.compute_contrastive_logits>  sq   € õ  œ) _Ð6GÐ$HÈaÐPÑPÔPˆåÔ(Ð);×)AÒ)AÑ)CÔ)CÀ_×EZÒEZÑE\ÔE\ÐbdÐeÑeÔeˆØ—’ Ñ0Ô0ˆð ˜+Ñ%ˆØˆr)   Nri   r•   rÂ   rÃ   rÄ   rÅ   c                 óN  — |�|n| j         j        }|                      |||||¬¦  «        }|d         }|                      |d         ¦  «        }	|                      |	¦  «        \  }
}|                      |
                     | j        j        j        ¦  «        ¦  «        }
|  	                    |
¦  «        }
t          j        |                     d¦  «        |                     d¦  «        ¦  «                             | j         j        ¦  «        }|                     dd¦  «        }t          j        |¦  «                             ¦   «                              |j        ¦  «        }|                     dd¦  «        }|                     d¦  «        }|                     |d¦  «        |
                     | d¦  «        z   }|                      |¦  «        }|                      |¦  «        }d}|s#|�|||
|f|dd…         z   S ||
|f|dd…         z   S t/          |||
||j        |j        ¬¦  «        S )	a›  
        Example:

        ```python
        >>> import torch
        >>> from transformers import AutoFeatureExtractor, UniSpeechForPreTraining

        >>> feature_extractor = AutoFeatureExtractor.from_pretrained("microsoft/unispeech-large-1500h-cv")
        >>> model = UniSpeechForPreTraining.from_pretrained("microsoft/unispeech-large-1500h-cv")
        >>> # TODO: Add full pretraining example
        ```NrÇ   r   r   r<   rl   r   )r   r   r   r   r   r   )rg   rÄ   rh   rÔ   rÕ   rÙ   rš   ru   r—   rÚ   r$   ÚemptyÚsizeÚfill_Úreplace_probrÊ   Ú	bernoullirŸ   r˜   rX   Úmasked_fillrÞ   rÜ   r   r   r   )r[   ri   r•   rÂ   rÃ   rÄ   rÍ   ÚoutputsÚtransformer_featuresrÉ   Úquantized_featuresr   Úprob_replace_matrixÚsampled_replace_matrixré   r   s                   r*   rc   zUniSpeechForPreTraining.forwardR  sW  € ð, &1Ð%<�k�kÀ$Ä+ÔBYˆà—.’.ØØ)Ø/Ø!5Ø#ð !ñ 
ô 
ˆð  ' qœzÐð  ×0Ò0°¸´Ñ<Ô<ÐØ48·N²NÐCSÑ4TÔ4TÑ1ÐÐ1ð "Ÿ^š^Ð,>×,AÒ,AÀ$Ä.ÔBWÔB]Ñ,^Ô,^Ñ_Ô_ÐØ!×-Ò-Ð.@ÑAÔAÐå#œkÐ*>×*CÒ*CÀAÑ*FÔ*FÐH\×HaÒHaÐbcÑHdÔHdÑeÔe×kÒkØŒKÔ$ñ
ô 
Ðð 2×;Ò;¸A¸qÑAÔAÐÝ!&¤Ð1DÑ!EÔ!E×!JÒ!JÑ!LÔ!L×!OÒ!OÐPdÔPkÑ!lÔ!lÐØ!7×!AÒ!AÀ!ÀQÑ!GÔ!GÐØ!7×!AÒ!AÀ"Ñ!EÔ!EÐØ%×1Ò1Ð2HÈ#ÑNÔNØ×*Ò*Ð,BÐ+BÀCÑHÔHñ
ˆð
 —’˜fÑ%Ô%ˆØ—’˜vÑ&Ô&ˆð ˆØð 	cØÐØÐ2Ð4FÐH]Ð^ÐahÐijÐikÐikÔalÑlÐlØ(Ð*<Ð>SÐTÐW^Ð_`Ð_aÐ_aÔWbÑbÐbå,ØØ1Ø'9Ø"7Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r)   )r   )NNNN)r    r!   r"   r   r²   r¬   rà   rÀ   rd   r$   r%   rê   r	   r¸   rŸ   r'   r   rc   r­   r®   s   @r*   rÐ   rÐ     sa  ø€ € € € € ð˜ð ð ð ð ð ð ð1°#ð 1ð 1ð 1ð 1ð>ð >ð >ð ð
 ð	ð ØÔ*ðà Ô,ðð "Ô-ðð ð	ð ð ñ „\ðð& ð /3Ø)-Ø,0Ø#'ðE
ð E
à”l TÑ)ðE
ð œ tÑ+ðE
ð   $™;ð	E
ð
 # T™kðE
ð ˜D‘[ðE
ð 
Ð.Ñ	.ðE
ð E
ð E
ñ „^ðE
ð E
ð E
ð E
ð E
r)   rÐ   c                   ó   — e Zd ZdS )ÚUniSpeechForCTCNr-   r(   r)   r*   rø   rø   ›  r.   r)   rø   c                   ó   — e Zd ZdS )Ú"UniSpeechForSequenceClassificationNr-   r(   r)   r*   rú   rú   Ÿ  r.   r)   rú   )rø   rÐ   rú   r°   rf   )/r#   rz   Údataclassesr   r$   Útorch.nnrN   Ú r   rs   Úmodeling_outputsr   r   Úmodeling_utilsr   Úutilsr	   r
   Úwav2vec2.modeling_wav2vec2r   r   r   r   r   r   r   r   r   Úconfiguration_unispeechr   Ú
get_loggerr    Úloggerr   r,   r0   r2   r4   r6   r8   rf   rÌ   r°   rÐ   rø   rú   Ú__all__r(   r)   r*   ú<module>r     sÊ  ðð Ð à €€€Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø DÐ DÐ DÐ DÐ DÐ DÐ DÐ DØ -Ð -Ð -Ð -Ð -Ð -Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 5Ð 4Ð 4Ð 4Ð 4Ð 4ð 
ˆÔ	˜HÑ	%Ô	%€ð €ððñ ô ð
 ð7ð 7ð 7ð 7ð 7 Kñ 7ô 7ñ „ñô ð7ð.	ð 	ð 	ð 	ð 	Ð'Fñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð4ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð!:ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	�ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð&Dñ 	ô 	ð 	ð*'ð *'ð *'ð *'ð *'Ð%Bñ *'ô *'ð *'ðZ ðAð Að Að Að A˜ñ Aô Añ „ðAðH 3Ð ðH
ð H
ð H
ð H
ð H
Ð-¨}ñ H
ô H
ð H
ðV €ððñ ô ð
w
ð w
ð w
ð w
ð w
Ð6ñ w
ô w
ñô ð
w
ðt	ð 	ð 	ð 	ð 	�nñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð)Jñ 	ô 	ð 	ðð ð €€€r)   