§
    ‚Štj¿,  ã                   óà  — d Z ddlZddlmZ ddlmZ ddlmZ ddl	m
Z
 ddlmZ ddlmZ dd	lmZ d
dlmZmZmZmZmZmZmZ ddlmZ dZ G d„ dej        ¦  «        Z G d„ de¦  «        Z G d„ de¦  «        Z G d„ dej        ¦  «        Z  G d„ de¦  «        Z! G d„ de¦  «        Z"e G d„ de¦  «        ¦   «         Z# G d„ dee#¦  «        Z$ G d„ de¦  «        Z% G d „ d!e¦  «        Z&g d"¢Z'dS )#zPyTorch Hubert model.é    Né   )Úinitialization)ÚACT2FN)Úis_deepspeed_zero3_enabled)ÚBaseModelOutput)ÚPreTrainedModel)Úauto_docstringé   )ÚWav2Vec2EncoderÚWav2Vec2EncoderStableLayerNormÚWav2Vec2FeatureEncoderÚWav2Vec2ForCTCÚ!Wav2Vec2ForSequenceClassificationÚWav2Vec2ModelÚWav2Vec2SamePadLayeré   )ÚHubertConfigc                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚHubertPositionalConvEmbeddingc                 ó:  •— t          ¦   «                              ¦   «          t          j        |j        |j        |j        |j        dz  |j        ¬¦  «        | _        d | _        t          |dd¦  «        r t          j
        |j        ¦  «        | _        �nWt          j        j        }t          t          j        j        d¦  «        rt          j        j        j        }t          ¦   «         rëdd l}|j                             | j        j        d¬¦  «        5   || j        dd¬	¦  «        | _        d d d ¦  «         n# 1 swxY w Y   t          | j        d
¦  «        r-| j        j        j        j        }| j        j        j        j        }n| j        j        }| j        j        }|j                             | |¦  «         |j                             | |¦  «         n || j        dd¬	¦  «        | _        t3          |j        ¦  «        | _        t6          |j                 | _        d S )Nr
   )Úkernel_sizeÚpaddingÚgroupsÚconv_pos_batch_normFÚweight_normr   ©Úmodifier_rankÚweight)ÚnameÚdimÚparametrizations)ÚsuperÚ__init__ÚnnÚConv1dÚhidden_sizeÚnum_conv_pos_embeddingsÚnum_conv_pos_embedding_groupsÚconvÚ
batch_normÚgetattrÚBatchNorm1dÚutilsr   Úhasattrr!   r   Ú	deepspeedÚzeroÚGatheredParametersr   Ú	original0Ú	original1Úweight_gÚweight_vÚregister_external_parameterÚHubertSamePadLayerr   r   Úfeat_extract_activationÚ
activation)ÚselfÚconfigr   r/   r4   r5   Ú	__class__s         €úg/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/hubert/modular_hubert.pyr#   z&HubertPositionalConvEmbedding.__init__)   s1  ø€ Ý‰Œ×ÒÑÔÐÝ”IØÔØÔØÔ6ØÔ2°aÑ7ØÔ7ð
ñ 
ô 
ˆŒ	ð ˆŒÝ�6Ð0°%Ñ8Ô8ð 	IÝ œn¨VÔ-?Ñ@Ô@ˆDŒO‰Oåœ(Ô.ˆKÝ•r”xÔ0°-Ñ@Ô@ð DÝ œhÔ7ÔC�å)Ñ+Ô+ð IØ Ð Ð Ð à”^×6Ò6°t´yÔ7GÐWXÐ6ÑYÔYð Mð MØ + ¨D¬I¸HÈ!Ð LÑ LÔ L�D”IðMð Mð Mñ Mô Mð Mð Mð Mð Mð Mð Møøøð Mð Mð Mð Må˜4œ9Ð&8Ñ9Ô9ð 2Ø#œyÔ9Ô@ÔJ�HØ#œyÔ9Ô@ÔJ�H�Hà#œyÔ1�HØ#œyÔ1�HØ”×:Ò:¸4ÀÑJÔJÐJØ”×:Ò:¸4ÀÑJÔJÐJÐJà'˜K¨¬	¸ÀaÐHÑHÔH�”	å)¨&Ô*HÑIÔIˆŒÝ  Ô!?Ô@ˆŒˆˆs   ÄD7Ä7D;Ä>D;c                 ó  — |                      dd¦  «        }| j        �|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      dd¦  «        }|S )Nr   r
   )Ú	transposer*   r)   r   r9   ©r:   Úhidden_statess     r=   Úforwardz%HubertPositionalConvEmbedding.forwardN   s~   € Ø%×/Ò/°°1Ñ5Ô5ˆØŒ?Ð&Ø ŸOšO¨MÑ:Ô:ˆMØŸ	š	 -Ñ0Ô0ˆØŸš ]Ñ3Ô3ˆØŸš¨Ñ6Ô6ˆà%×/Ò/°°1Ñ5Ô5ˆØÐó    ©Ú__name__Ú
__module__Ú__qualname__r#   rB   Ú__classcell__©r<   s   @r=   r   r   (   sM   ø€ € € € € ð#Að #Að #Að #Að #AðJ	ð 	ð 	ð 	ð 	ð 	ð 	rC   r   c                   ó   — e Zd ZdS )r7   N©rE   rF   rG   © rC   r=   r7   r7   Z   ó   € € € € € Ø€DrC   r7   c                   ó   — e Zd ZdS )ÚHubertFeatureEncoderNrK   rL   rC   r=   rO   rO   ^   rM   rC   rO   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚHubertFeatureProjectionc                 óT  •— t          ¦   «                              ¦   «          |j        | _        | j        r+t          j        |j        d         |j        ¬¦  «        | _        t          j        |j        d         |j	        ¦  «        | _
        t          j        |j        ¦  «        | _        d S )Néÿÿÿÿ)Úeps)r"   r#   Úfeat_proj_layer_normr$   Ú	LayerNormÚconv_dimÚlayer_norm_epsÚ
layer_normÚLinearr&   Ú
projectionÚDropoutÚfeat_proj_dropoutÚdropout©r:   r;   r<   s     €r=   r#   z HubertFeatureProjection.__init__c   s…   ø€ Ý‰Œ×ÒÑÔÐØ$*Ô$?ˆÔ!ØÔ$ð 	[Ý œl¨6¬?¸2Ô+>ÀFÔDYÐZÑZÔZˆDŒOÝœ) F¤O°BÔ$7¸Ô9KÑLÔLˆŒÝ”z &Ô":Ñ;Ô;ˆŒˆˆrC   c                 ó’   — | j         r|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S )N)rU   rY   r[   r^   r@   s     r=   rB   zHubertFeatureProjection.forwardk   sF   € àÔ$ð 	;Ø ŸOšO¨MÑ:Ô:ˆMØŸš¨Ñ6Ô6ˆØŸš ]Ñ3Ô3ˆØÐrC   rD   rI   s   @r=   rQ   rQ   b   sG   ø€ € € € € ð<ð <ð <ð <ð <ðð ð ð ð ð ð rC   rQ   c                   ó   — e Zd ZdS )ÚHubertEncoderNrK   rL   rC   r=   rb   rb   t   rM   rC   rb   c                   ó   — e Zd ZdS )ÚHubertEncoderStableLayerNormNrK   rL   rC   r=   rd   rd   x   rM   rC   rd   c                   ó®   ‡ — e Zd ZU eed<   dZdZdZddgZdZ	dZ
dZdZ ej        ¦   «         ˆ fd„¦   «         Zd	ej        ez  fd
„Zdedej        fd„Zˆ xZS )ÚHubertPreTrainedModelr;   ÚhubertÚinput_valuesÚaudioÚHubertEncoderLayerÚParametrizedConv1dTc                 óâ  •— t          ¦   «                              |¦  «         t          |t          j        ¦  «        �rt          ¦   «         rÑddl}t          |d¦  «        rjt          |d¦  «        rZ|j         	                    |j
        |j        gd¬¦  «        5  t          j        |j        ¦  «         ddd¦  «         n# 1 swxY w Y   nl|j         	                    |j        d¬¦  «        5  t          j        |j        ¦  «         ddd¦  «         n# 1 swxY w Y   nt          j        |j        ¦  «         |j        �t          j        |j        ¦  «         dS dS t          |t"          ¦  «        r-t          |d¦  «        rt          j        |j        ¦  «         dS dS t          |t(          ¦  «        r<t          |d¦  «        r.t          j        |j        d| j        j        d	z   z  ¦  «         dS dS dS )
zInitialize the weightsr   Nr5   r4   r   Úmasked_spec_embedÚlayer_weightsg      ð?r   )r"   Ú_init_weightsÚ
isinstancer$   r%   r   r/   r.   r0   r1   r5   r4   ÚinitÚkaiming_normal_r   ÚbiasÚzeros_ÚHubertModelÚuniform_rm   ÚHubertForSequenceClassificationÚ	constant_rn   r;   Únum_hidden_layers)r:   Úmoduler/   r<   s      €r=   ro   z#HubertPreTrainedModel._init_weightsˆ   sv  ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�f�bœiÑ(Ô(ñ 	`Ý)Ñ+Ô+ð 
4Ø Ð Ð Ð å˜6 :Ñ.Ô.ð <µ7¸6À:Ñ3NÔ3Nð <Ø"œ×:Ò:¸F¼OÈVÌ_Ð;]ÐmnÐ:ÑoÔoð <ð <ÝÔ,¨V¬]Ñ;Ô;Ð;ð<ð <ð <ñ <ô <ð <ð <ð <ð <ð <ð <øøøð <ð <ð <ð <øð #œ×:Ò:¸6¼=ÐXYÐ:ÑZÔZð <ð <ÝÔ,¨V¬]Ñ;Ô;Ð;ð<ð <ð <ñ <ô <ð <ð <ð <ð <ð <ð <øøøð <ð <ð <ð <øõ Ô$ V¤]Ñ3Ô3Ð3àŒ{Ð&Ý”˜FœKÑ(Ô(Ð(Ð(Ð(ð 'Ð&å˜¥Ñ,Ô,ð 	`Ý�vÐ2Ñ3Ô3ð 8Ý”˜fÔ6Ñ7Ô7Ð7Ð7Ð7ð8ð 8å˜Õ ?Ñ@Ô@ð 	`Ý�v˜Ñ/Ô/ð `Ý”˜vÔ3°S¸D¼KÔ<YÐ\]Ñ<]Ñ5^Ñ_Ô_Ð_Ð_Ð_ð	`ð 	`ð`ð `s$   ÂB>Â>CÃCÃ+DÄDÄDÚinput_lengthsc                 óz   — d„ }t          | j        j        | j        j        ¦  «        D ]\  }} ||||¦  «        }Œ|S )zH
        Computes the output length of the convolutional layers
        c                 ó<   — t          j        | |z
  |d¬¦  «        dz   S )NÚfloor)Úrounding_moder   )ÚtorchÚdiv)Úinput_lengthr   Ústrides      r=   Ú_conv_out_lengthzPHubertPreTrainedModel._get_feat_extract_output_lengths.<locals>._conv_out_length§   s&   € õ ”9˜\¨KÑ7¸ÈwÐWÑWÔWÐZ[Ñ[Ð[rC   )Úzipr;   Úconv_kernelÚconv_stride)r:   r{   r„   r   rƒ   s        r=   Ú _get_feat_extract_output_lengthsz6HubertPreTrainedModel._get_feat_extract_output_lengths¢   s\   € ð
	\ð 	\ð 	\õ
 $' t¤{Ô'>ÀÄÔ@WÑ#XÔ#Xð 	Qð 	QÑˆK˜Ø,Ð,¨]¸KÈÑPÔPˆMˆMàÐrC   Úfeature_vector_lengthÚattention_maskc                 óê  — |                       |                     d¦  «        ¦  «                             t          j        ¦  «        }|j        d         }t          j        ||f|j        |j        ¬¦  «        }d|t          j	        |j        d         |j        ¬¦  «        |dz
  f<   | 
                    dg¦  «                             d¦  «         
                    dg¦  «                             ¦   «         }|S )NrS   r   )ÚdtypeÚdevicer   )r�   )rˆ   ÚsumÚtor€   ÚlongÚshapeÚzerosrŒ   r�   ÚarangeÚflipÚcumsumÚbool)r:   r‰   rŠ   Úoutput_lengthsÚ
batch_sizes        r=   Ú"_get_feature_vector_attention_maskz8HubertPreTrainedModel._get_feature_vector_attention_mask±   sæ   € Ø×>Ò>¸~×?QÒ?QÐRTÑ?UÔ?UÑVÔV×YÒYÕZ_ÔZdÑeÔeˆØ#Ô)¨!Ô,ˆ
åœØÐ.Ð/°~Ô7KÐTbÔTið
ñ 
ô 
ˆð uvˆ�œ ^Ô%9¸!Ô%<À^ÔEZÐ[Ñ[Ô[Ð]kÐnoÑ]oÐpÑqØ'×,Ò,¨b¨TÑ2Ô2×9Ò9¸"Ñ=Ô=×BÒBÀBÀ4ÑHÔH×MÒMÑOÔOˆØÐrC   )rE   rF   rG   r   Ú__annotations__Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚ_no_split_modulesÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnr€   Úno_gradro   Ú
LongTensorÚintrˆ   r™   rH   rI   s   @r=   rf   rf   |   sÛ   ø€ € € € € € àÐÐÑØ ÐØ$€OØÐØ-Ð/CÐDÐØ&*Ð#ØÐØ€NØÐà€U„]�_„_ð`ð `ð `ð `ñ „_ð`ð2¸eÔ>NÐQTÑ>Tð ð ð ð ð
Èð 
Ð]bÔ]mð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
rC   rf   c                   ó¢   ‡ — e Zd Zdefˆ fd„Zd„ Z	 	 	 	 	 ddej        dz  dej        dz  dej        dz  de	dz  d	e	dz  d
e	dz  de
ez  fd„Zˆ xZS )ru   r;   c                 óä  •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        t          |¦  «        | _        |j        dk    s|j        dk    rBt          j
        t          j        |j        ¦  «                             ¦   «         ¦  «        | _        |j        rt#          |¦  «        | _        nt'          |¦  «        | _        |                      ¦   «          | `d S )Ng        )r"   r#   r;   rO   Úfeature_extractorrQ   Úfeature_projectionÚmask_time_probÚmask_feature_probr$   Ú	Parameterr€   ÚTensorr&   rv   rm   Údo_stable_layer_normrd   Úencoderrb   Ú	post_initÚadapterr_   s     €r=   r#   zHubertModel.__init__¿   sÎ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ!5°fÑ!=Ô!=ˆÔÝ"9¸&Ñ"AÔ"AˆÔàÔ  3Ò&Ð&¨&Ô*BÀSÒ*HÐ*HÝ%'¤\µ%´,¸vÔ?QÑ2RÔ2R×2[Ò2[Ñ2]Ô2]Ñ%^Ô%^ˆDÔ"àÔ&ð 	1Ý7¸Ñ?Ô?ˆDŒLˆLå(¨Ñ0Ô0ˆDŒLð 	�ŠÑÔÐàˆLˆLˆLrC   c                 ó    — t          d¦  «        ‚)NzNot needed for Hubert)ÚAttributeError)r:   s    r=   Úfreeze_feature_encoderz"HubertModel.freeze_feature_encoderÒ   s   € ÝÐ4Ñ5Ô5Ð5rC   Nrh   rŠ   Úmask_time_indicesÚoutput_attentionsÚoutput_hidden_statesÚreturn_dictÚreturnc                 óò  — |�|n| j         j        }|�|n| j         j        }|�|n| j         j        }|                      |¦  «        }|                     dd¦  «        }|�!|                      |j        d         |¦  «        }|                      |¦  «        }	|  	                    |	|¬¦  «        }	|  
                    |	||||¬¦  «        }
|
d         }	|s|	f|
dd…         z   S t          |	|
j        |
j        ¬¦  «        S )a1  
        mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict
            masked extracted features in *config.proj_codevector_dim* space.

        Example:

        ```python
        >>> from transformers import AutoProcessor, HubertModel
        >>> from datasets import load_dataset

        >>> processor = AutoProcessor.from_pretrained("facebook/hubert-large-ls960-ft")
        >>> model = HubertModel.from_pretrained("facebook/hubert-large-ls960-ft")


        >>> def map_to_array(example):
        ...     example["speech"] = example["audio"]["array"]
        ...     return example


        >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        >>> ds = ds.map(map_to_array)

        >>> input_values = processor(ds["speech"][0], return_tensors="pt").input_values  # Batch size 1
        >>> hidden_states = model(input_values).last_hidden_state
        ```Nr   r
   )rµ   )rŠ   r¶   r·   r¸   r   )Úlast_hidden_staterA   Ú
attentions)r;   r¶   r·   r¸   r¨   r?   r™   r‘   r©   Ú_mask_hidden_statesr¯   r   rA   r¼   )r:   rh   rŠ   rµ   r¶   r·   r¸   ÚkwargsÚextract_featuresrA   Úencoder_outputss              r=   rB   zHubertModel.forwardÕ   s@  € ðH 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆà×1Ò1°,Ñ?Ô?ÐØ+×5Ò5°a¸Ñ;Ô;ÐàÐ%à!×DÒDÐEUÔE[Ð\]ÔE^Ð`nÑoÔoˆNà×/Ò/Ð0@ÑAÔAˆØ×0Ò0°ÐRcÐ0ÑdÔdˆàŸ,š,ØØ)Ø/Ø!5Ø#ð 'ñ 
ô 
ˆð (¨Ô*ˆàð 	:Ø!Ð# o°a°b°bÔ&9Ñ9Ð9åØ+Ø)Ô7Ø&Ô1ð
ñ 
ô 
ð 	
rC   )NNNNN)rE   rF   rG   r   r#   r´   r€   r­   ÚFloatTensorr–   Útupler   rB   rH   rI   s   @r=   ru   ru   ¾   sø   ø€ € € € € ð˜|ð ð ð ð ð ð ð&6ð 6ð 6ð /3Ø6:Ø)-Ø,0Ø#'ðE
ð E
à”l TÑ)ðE
ð œ tÑ+ðE
ð !Ô,¨tÑ3ð	E
ð
   $™;ðE
ð # T™kðE
ð ˜D‘[ðE
ð 
�Ñ	 ðE
ð E
ð E
ð E
ð E
ð E
ð E
ð E
rC   ru   c                   ó   — e Zd ZdS )ÚHubertForCTCNrK   rL   rC   r=   rÄ   rÄ     rM   rC   rÄ   c                   ó   — e Zd ZdS )rw   NrK   rL   rC   r=   rw   rw   !  rM   rC   rw   )rÄ   rw   ru   rf   )(Ú__doc__r€   Útorch.nnr$   Ú r   rq   Úactivationsr   Úintegrations.deepspeedr   Úmodeling_outputsr   Úmodeling_utilsr   r-   r	   Úwav2vec2.modeling_wav2vec2r   r   r   r   r   r   r   Úconfiguration_hubertr   Ú_HIDDEN_STATES_START_POSITIONÚModuler   r7   rO   rQ   rb   rd   rf   ru   rÄ   rw   Ú__all__rL   rC   r=   ú<module>rÒ      sâ  ðð Ð à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø @Ð @Ð @Ð @Ð @Ð @Ø /Ð /Ð /Ð /Ð /Ð /Ø -Ð -Ð -Ð -Ð -Ð -Ø #Ð #Ð #Ð #Ð #Ð #ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð /Ð .Ð .Ð .Ð .Ð .ð !"Ð ð/ð /ð /ð /ð / B¤Iñ /ô /ð /ðd	ð 	ð 	ð 	ð 	Ð-ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð1ñ 	ô 	ð 	ðð ð ð ð ˜bœiñ ô ð ð$	ð 	ð 	ð 	ð 	�Oñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð#Añ 	ô 	ð 	ð ð>ð >ð >ð >ð >˜Oñ >ô >ñ „ð>ðB\
ð \
ð \
ð \
ð \
�-Ð!6ñ \
ô \
ð \
ð~	ð 	ð 	ð 	ð 	�>ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð&Gñ 	ô 	ð 	ð fÐ
eÐ
e€€€rC   