tj ddlZddlZddlZddlZddlmZddlmcmZ ddlm Z ddl m Z ddlmZddlmZddlmZddlmZdd lmZmZmZmZmZmZdd lmZmZdd l m!Z!m"Z"m#Z#d d l$m%Z%e#j&e'Z(Gddej)Z*Gddej)Z+Gddej)Z,Gddej)Z-Gddej)Z.GddeZ/GddeZ0Gddej)Z1Gddej)Z2Gd d!ej)Z3e!Gd"d#eZ4Gd$d%eZ5Gd&d'eZ6Gd(d)eZ7Gd*d+ej)Z8Gd,d-ej)Z9Gd.d/ej)Z: dKd0e;edzd4e;Gd?d@e4ZEe!GdAdBe4ZFGdCdDej)ZGGdEdFej)ZHe!dG;GdHdIe4ZIgdJZJdS)LN)CrossEntropyLoss)initialization)ACT2FN)is_deepspeed_zero3_enabled)is_fsdp_managed_module)GradientCheckpointingLayer)BaseModelOutputCausalLMOutputSequenceClassifierOutputTokenClassifierOutputWav2Vec2BaseModelOutput XVectorOutput)PreTrainedModel*get_torch_context_manager_or_global_device)auto_docstringis_peft_availablelogging) WavLMConfigc$eZdZfdZdZxZS)WavLMSamePadLayerclt|dzdkrdnd|_dSNrr)super__init__num_pad_remove)selfnum_conv_pos_embeddings __class__s f/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/wavlm/modeling_wavlm.pyrzWavLMSamePadLayer.__init__&s: #:Q#>!#C#CaacJ|jdkr|ddddd|j f}|SNr)rr hidden_statess r"forwardzWavLMSamePadLayer.forward*s;   " ")!!!QQQ0F43F2F0F*FGMr#__name__ __module__ __qualname__rr( __classcell__r!s@r"rr%sLKKKKKr#rc$eZdZfdZdZxZS)WavLMPositionalConvEmbeddingcttj|j|j|j|jdz|j|_tjj }ttjj drtjj j }trddl }|j|jjd5||jdd|_dddn #1swxYwYt|jdr-|jj jj}|jj jj}n|jj}|jj}|j|||j||n||jdd|_t-|j|_t0|j|_dS) Nr) kernel_sizepaddinggroups weight_normr) modifier_rankweight)namedimparametrizations)rrnnConv1d hidden_sizer num_conv_pos_embedding_groupsconvutilsr5hasattrr:r deepspeedzeroGatheredParametersr7 original0 original1weight_gweight_vregister_external_parameterrr3rfeat_extract_activation activation)rconfigr5rBrGrHr!s r"rz%WavLMPositionalConvEmbedding.__init__1s I    62a77     h* 28,m < < @(3?K % ' ' E    22493CST2UU I I'K aHHH  I I I I I I I I I I I I I I Ity"455 .95<F95<F9-9- N 6 6tX F F F N 6 6tX F F F F# DIH!DDDDI()GHH  !?@sC??DDc|dd}||}||}||}|dd}|SNrr) transposer?r3rKr&s r"r(z$WavLMPositionalConvEmbedding.forwardRse%//155  -00  ]33  66 %//155 r#r)r.s@r"r0r00sMAAAAABr#r0c$eZdZfdZdZxZS)WavLMFeatureProjectionc.ttj|jd|j|_tj|jd|j|_ tj |j |_ dS)Neps) rrr; LayerNormconv_dimlayer_norm_eps layer_normLinearr= projectionDropoutfeat_proj_dropoutdropoutrrLr!s r"rzWavLMFeatureProjection.__init__^sn ,vr':@UVVV)FOB$79KLLz&":;; r#c||}||}||}||fSN)rYr[r^)rr'norm_hidden_statess r"r(zWavLMFeatureProjection.forwarddsC!__];;(:;;  ]33 000r#r)r.s@r"rQrQ]sG<<<<< 1111111r#rQceZdZdZ ddededeezd ed ed ef fd Z ddej dej d zdej d zdede ej ej d ze ej d zff dZ dej dej ejzdej dede ej ej ff dZdededej fdZdej dej fdZxZS)WavLMAttentionz=Multi-headed attention from 'Attention Is All You Need' paper@ T embed_dim num_headsr^ num_buckets max_distancehas_relative_position_biasct||_||_||_||z|_|j|z|jkrt d|jd|d|jdz|_tj |||_ tj |||_ tj |||_ tj |||_ ||_||_tjt#jd|jdd|_tj |jd|_|r&tj|j|j|_dSdS)Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).gr)rrrhrir^head_dim ValueErrorscalingr;rZk_projv_projq_projout_projrjrk Parametertorchonesgru_rel_pos_constgru_rel_pos_linear Embeddingrel_attn_embed)rrhrir^rjrkrlr!s r"rzWavLMAttention.__init__osa "" !Y. MI %$. 8 83dn33%.333 }d* i 955 i 955 i 955  )Y77 &(!#ejDNAq.Q.Q!R!R"$)DM1"="= % Q"$,t/?"P"PD    Q Qr#NFrr'attention_mask position_biasoutput_attentionsreturncF|\}}}|^|||}|d|ddd||jz||}||jdd|jdfz} | dddd} || } | | jdddz d} tj |  dd\} } | | |j zd z zd z} | ||jzdd|z}|d||f}|||||\}}|||fS) z'Attention layer with relative attentionNrrrSrr)rr9?g@)size compute_bias unsqueezerepeatviewrishapepermuterzsumrwsigmoidchunkrytorch_multi_head_self_attention)rr'r}r~rindexbsztgt_len_gated_hidden_statesrelative_position_projgate_agate_b gate_outputgated_position_bias attn_output attn_weightss r"r(zWavLMAttention.forwards(,,..Wa   --gw??M''**11#q!Q??DDS4>EY[bdkll  ,001DSbS1IT^]_L`1`aa199!Q1EE"&!8!89L!M!M!7!>DDQBDOO)? ?# EFL *..sT^/CRKKm[166GW7MNN$($H$H >+>@Q% % ! \L-77r#rc|ddx}x}}||dnd}dx} } d} tj||||j|jt jdgt j|j j |j j |j j f| | | |j |jj|jj |j|||d|j j|j j|j j\} } | dd} | E| dddf| jdd|jfz| jddz} | | fS)zCsimple wrapper around torch's multi_head_attention_forward functionrrNFT)use_separate_proj_weight q_proj_weight k_proj_weight v_proj_weight)rOneFmulti_head_attention_forwardrhrirwemptycatrtbiasrrrsr^rur7training broadcast_tor)rr'r}rrquerykeyvaluekey_padding_maskbias_kbias_v add_zero_attnrrs r"rz.WavLMAttention.torch_multi_head_self_attentionsx,55a;;;;e3A3M>,,Q///SW %&$B    N N K   It{')94;;KL M M    L M M  M   %)+,+,+,+% % % ! \2"++Aq11  #(40=="2A2&$.)::\=OPQPRPR=SSLL((r# query_length key_lengthctj|tjdddf}tj|tjdddf}||z }||}||jjj}||}|gd}|S)Ndtype)rrr) rwarangelong_relative_positions_buckettor|r7devicer)rrrcontext_positionmemory_positionrelative_positionrelative_position_bucketvaluess r"rzWavLMAttention.compute_biass < EJGGG4P,zDDDT111WM+.>>#'#B#BCT#U#U #;#>#>t?R?Y?`#a#a $$%=>> ** r#relative_positionsc|jdz}|dktj|z}tj|}|dz}||k}tj||z }|tj|j|z z }|||z z}||ztj}tj |tj ||dz }|tj |||z }|Sr) rjrrwrabslogfloatmathrkmin full_likewhere)rrrjrelative_buckets max_exactis_smallrelative_positions_if_largerelative_position_if_larges r"rz)WavLMAttention._relative_positions_buckets&!+ .266uzBB[P"Y'9::1$ % 1&+i0B0H0H0J0JY0V&W&W#&ADHTM^ajMjDkDk&k#&A[S\E\&]#&/2M&M%Q%QRWR\%]%]"%*Y &8RT_bcTc(d(d& & " EK2DF`aaar#)rerfrgTNNFr)r*r+r,__doc__intrboolrrwTensortupler( FloatTensor LongTensor BoolTensorrrrr-r.s@r"rdrdlsGG #+/"Q"Q"Q"Q "Q  "Q  "Q%)"Q"Q"Q"Q"Q"QN/3-1"' '8'8|'8 t+'8|d* '8  '8 u|U\D0% 2E2LL M'8'8'8'8R5)(5)(5+;;5)#. 5)  5) u %"33 4 5)5)5)5)n#%BS U=N SXSd        r#rdc$eZdZfdZdZxZS)WavLMFeedForwardcttj|j|_tj|j|j|_ t|j trt|j |_n |j |_tj|j|j|_tj|j|_dSra)rrr;r\activation_dropoutintermediate_dropoutrZr=intermediate_sizeintermediate_dense isinstance hidden_actstrrintermediate_act_fn output_densehidden_dropoutoutput_dropoutr_s r"rzWavLMFeedForward.__init__s $&Jv/H$I$I!"$)F,>@X"Y"Y f' - - 9'-f.?'@D $ $'-'8D $If&>@RSS j)>??r#c||}||}||}||}||}|Sra)rrrrrr&s r"r(zWavLMFeedForward.forward sg// >> 00?? 11-@@ ))-88 ++M:: r#r)r.s@r"rrsL @ @ @ @ @r#rc2eZdZd dedeffd Zd dZxZS) WavLMEncoderLayerTrLrlctt|j|j|j|j|j||_tj |j |_ tj |j|j|_t!||_tj |j|j|_dSN)rhrir^rjrkrlrTrrrdr=num_attention_headsattention_dropoutrjmax_bucket_distance attentionr;r\rr^rVrXrYr feed_forwardfinal_layer_normrrLrlr!s r"rzWavLMEncoderLayer.__init__+ '(0,*3'A    z&"788 ,v'9v?TUUU,V44 " V-?VEZ [ [ [r#NFrc|}||||||\}}}||}||z}||}|||z}||}||f}|r||fz }|S)Nr}r~rr)rr^rYrr) rr'r}r~rr attn_residualroutputss r"r(zWavLMEncoderLayer.forward:s% 59^^ )'/ 6D6 6 2 |] ]33 % 5  66 %(9(9-(H(HH --m<<  -0  '   &Gr#Trr*r+r,rrrr(r-r.s@r"rr*sm \ \{ \ \ \ \ \ \ \r#rc2eZdZddedeffd Zd dZxZS) WavLMEncoderLayerStableLayerNormTrLrlctt|j|j|j|j|j||_tj |j |_ tj |j|j|_t!||_tj |j|j|_dSrrrs r"rz)WavLMEncoderLayerStableLayerNorm.__init__Trr#NFc|}||}|||||\}}}||}||z}||||z}||f}|r||fz }|S)N)r}r~r)rYrr^rr)rr'r}r~rrrrs r"r(z(WavLMEncoderLayerStableLayerNorm.forwardcs%  66 59^^ )'/ 6D6 6 2 |]  ]33 % 5 %(9(9$:O:OP]:^:^(_(__  -0  '   &Gr#r)NNFrr.s@r"rrSsm \ \{ \ \ \ \ \ \ \r#rc.eZdZfdZ ddZxZS) WavLMEncoderct|_t|_t jjj|_ t j j |_ t j fdtjD|_d|_dS)NrTc:g|]}t|dkSr)rl)r.0irLs r" z)WavLMEncoder.__init__..s, u u uPQ v16 K K K u u ur#FrrrLr0pos_conv_embedr;rVr=rXrYr\rr^ ModuleListrangenum_hidden_layerslayersgradient_checkpointingr_s `r"rzWavLMEncoder.__init__ys  :6BB,v'9v?TUUUz&"788 m u u u uUZ[a[sUtUt u u u   ',###r#NFTc|rdnd}|rdnd}|;|ddd|jd}d||<||} || z}||}||}t pt|} d} t|j D]q\} } |r||fz}tj g}|j o| dko||j jk}|r| r| ||| || }|dd\}} |rd}|r ||dfz}r|r||fz}|std|||fDSt!||| S) NrSrrrrNNNc3K|]}||V dSrar rvs r" z'WavLMEncoder.forward..(mmq_`_l_l_l_l_lmmr#last_hidden_stater' attentions)rrrrrYr^rr enumerater rwrandrrL layerdroprr rr'r}routput_hidden_states return_dictall_hidden_statesall_self_attentionsexpand_attention_maskposition_embeddings synced_gpusr~rlayerdropout_probabilityskip_the_layer layer_outputss r"r(zWavLMEncoder.forwards#7@BBD$5?bb4  %$2$<$.s=   1UVZ[U[]]]   r#Frr_s `r"rz$WavLMEncoderStableLayerNorm.__init__s  :6BB,v'9v?TUUUz&"788 m    v788      ',###r#NFTc|rdnd}|rdnd}|;|ddd|jd}d||<||} || z}||}t pt |} d} t|jD]p\} } |r||fz}tj g}|j o| dko||j j k}|r| r| |||| }|dd\}} |rd}|r ||dfz}q||}|r||fz}|std|||fDSt!||| S) Nr rSrrr)r}rr~rc3K|]}||V dSrar rs r"rz6WavLMEncoderStableLayerNorm.forward..rr#r)rrrrr^rrrr rwrrrLrrYrr rs r"r(z#WavLMEncoderStableLayerNorm.forwards#7@BBD$5?bb4  %$2$<$eZdZdZfdZedZdZxZS)WavLMGumbelVectorQuantizerz Vector quantization using gumbel softmax. See [CATEGORICAL REPARAMETERIZATION WITH GUMBEL-SOFTMAX](https://huggingface.co/papers/1611.01144) for more information. ct|j|_|j|_|j|jzdkr td|jd|jdtj tj d|j|jz|j|jz|_ tj |jd|j|jz|_d|_dS)Nrz`config.codevector_dim z5 must be divisible by `config.num_codevector_groups` z for concatenation.rrSr)rrnum_codevector_groups num_groupsnum_codevectors_per_groupnum_varscodevector_dimrpr;rvrwr codevectorsrZrW weight_proj temperaturer_s r"rz#WavLMGumbelVectorQuantizer.__init__s  68  4? 2a 7 7%&*?%%6:o%%% <  a4=!@&BW[_[jBj k k  9V_R%8$/DM:YZZr#c|d}tjtjtj||d }|S)NrrrS)meanrwexprxlogy)probsmarginal_probs perplexitys r"_compute_perplexityz.WavLMGumbelVectorQuantizer._compute_perplexity(sU**Y %+nn*U*U[] ^ ^ ^^__ccee r#c|j\}}}||}|||z|jzd}|jrt j||j d}| |}tj |||z|jdd}| |}n|d}|j|jd|ddd}|||z|jd}| |}|||zd}|d|jz} | ||z|j|jd} | d||d} | |fS)NrST)tauhardrrr)rr6rr1rr; functionalgumbel_softmaxrr7type_asrwsoftmaxr?argmax new_zerosscatter_rr5r3r) rr' batch_sizesequence_lengthr=codevector_probscodevector_soft_distr>codevector_idxcodevectors_per_groupr5s r"r(z"WavLMGumbelVectorQuantizer.forward.s3@3F0 O[((77 %**:+G$/+Y[]^^ = D!};;M >AQ Q+00o1Mt`d`moqrr !oob))..z?BOO J&&r#) r*r+r,rr staticmethodr?r(r-r.s@r"r.r. sl *\ "'"'"'"'"'"'"'r#r.ceZdZUeed<dZdZdZdZdZ dZ dZ e j fdZdd e jezd edzfd Z dd ed e jfdZxZS)WavLMPreTrainedModelrLwavlm input_valuesaudioTFc Lt|t|trZt j|jjddt j|jj t j |j dSt|trwt j|j jddtjd|j jd|j jzz zt j|j j ddSt|t&rgtjd|jjz }t j |jj| |t j |jj | |dSt|t,jrnt j|j|j Ptj|j|j|jdzz }t j |j | |dSdSdS)zInitialize the weightsrer)r9stdrr)abN)r _init_weightsrr.initnormal_r6r7zeros_runiform_r5r0r?rsqrtr2 in_channels constant_rQr[ in_featuresr;r<kaiming_normal_r4)rmodulekr!s r"r[z"WavLMPreTrainedModel._init_weights^s f%%% f8 9 9 6 L+2! D D D D K*/ 0 0 0 M&, - - - - -  < = = 6 L " !v{'>q'AFKD['["\]]]     N6;+Q / / / / /  6 7 7 6 !f/;;< > > > M&+0QB! < < < < < <  * * 6   / / /{&Ifmv/AFDVWXDY/YZ[[ fkaR1555555  6 6'&r#N input_lengths add_adapterc| |jjn|}d}t|jj|jjD]\}}||||}|r3t |jjD]}||d|jj}|S)zH Computes the output length of the convolutional layers Nc<tj||z |ddzS)Nfloor) rounding_moder)rwdiv input_lengthr2strides r"_conv_out_lengthzOWavLMPreTrainedModel._get_feat_extract_output_lengths.._conv_out_lengths&9\K7wWWWZ[[ [r#r)rLrhzip conv_kernel conv_stridernum_adapter_layersadapter_stride)rrgrhrqr2rprs r" _get_feat_extract_output_lengthsz5WavLMPreTrainedModel._get_feat_extract_output_lengthsys 2=1Ddk--+  \ \ \ $'t{'> @W#X#X Q Q K,,]KPPMM  _4;9:: _ _ 0 04;C] ^ ^ r#feature_vector_lengthr}c |ddddf}|||}|tj}|jd}tj||f|j|j}d|tj |jd|j|dz f<| dgd dg }|S)NrSrrhr)rrr)r) cumsumrwrrwrrzerosrrrflipr)rrxr}rhnon_padded_lengthsoutput_lengthsrKs r""_get_feature_vector_attention_maskz7WavLMPreTrainedModel._get_feature_vector_attention_masks ,22r2::111b5A>>?Q_j>kk'**5:66#)!,  . /~7KTbTi   uv ^%9!%<^EZ[[[]kno]opq',,bT2299"==BBB4HHMMOOr#ra)r*r+r,r__annotations__base_model_prefixmain_input_nameinput_modalitiessupports_gradient_checkpointing_supports_flash_attn_supports_sdpa_supports_flex_attnrwno_gradr[rrrrwrr-r.s@r"rSrSSs $O&*# NU]__6666_64e>NQT>Tcgjncn,Y]%(:?:Jr#rSc&eZdZdfd ZdZxZS)WavLMNoLayerNormConvLayerrcZt|dkr|j|dz nd|_|j||_t j|j|j|j||j||j |_ t|j |_ dS)Nrrr2rpr)rrrW in_conv_dim out_conv_dimr;r<rsrt conv_biasr?rrJrKrrLlayer_idr!s r"rz"WavLMNoLayerNormConvLayer.__init__s .s>KKKFG)&1q5AAAKKKr#rr"c2g|]}t|S)r)rrs r"rz0WavLMFeatureEncoder.__init__..s'vvv126AFFFvvvr#z`config.feat_extract_norm` is z), but has to be one of ['group', 'layer']FT) rrfeat_extract_normrrnum_feat_extract_layersrpr;r conv_layersr _requires_grad)rrLrr!s ` r"rzWavLMFeatureEncoder.__init__s   #w . .26AFFFGKKKKKPQWQorsQsKtKtKKKKK % 0 0vvvvPUV\VtPuPuvvvKKt1Ittt =55&+#"r#cP|D] }d|_ d|_dS)NF) parameters requires_gradrrparams r"_freeze_parametersz&WavLMFeatureEncoder._freeze_parameterss4__&& ( (E"'E  #r#cr|dddf}|jr|jrd|_|jD] }||}|S)NT)rrrr)rrUr' conv_layers r"r(zWavLMFeatureEncoder.forwards[$QQQW-    /4= /*.M '* 6 6J&J}55MMr#)r*r+r,rrrr(r-r.s@r"rrs\88#####"$$$       r#rc$eZdZfdZdZxZS)WavLMAdapterLayercttj|jd|jz|j|jd|_dS)Nrr)rpr3)rrr;r<output_hidden_sizeadapter_kernel_sizervr?r_s r"rzWavLMAdapterLayer.__init__sU I  % ) )  &(     r#cr||}tj|d}|S)Nrr)r?r;rDglur&s r"r(zWavLMAdapterLayer.forwards3 -00  ))-Q)?? r#r)r.s@r"rrsG     r#rc$eZdZfdZdZxZS) WavLMAdapterctjjkrCt jjj|_t jj|_ndx|_|_t j fdtj D|_ j |_ dS)Nc36K|]}tVdSra)r)rrrLs r"rz(WavLMAdapter.__init__...s,#h#h!$5f$=$=#h#h#h#h#h#hr#)rrrr=r;rZprojrVproj_layer_normrrrur rr_s `r"rzWavLMAdapter.__init__$s   $(: : : &"4f6OPPDI#%<0I#J#JD /3 3DI,m#h#h#h#huVMfGgGg#h#h#hhh )r#cX|j1|j*||}||}|dd}|jD]=}tj}|jr ||jkr ||}>|dd}|SrN)rrrOr nprandomrr)rr'r"layerdrop_probs r"r(zWavLMAdapter.forward1s 9 T%9%E IIm44M 00??M%//155 [ 5 5EY--//N= 5^dn%D%D %m 4 4 %//155 r#r)r.s@r"rr#sG * * * * *r#rr mask_prob mask_lengthr} min_masksrc@|\}dkrtdkrtdddtjdfd}|9|dnfd t|D}tj |ft }g} |} | d kr|S|D]} || } tj tj | dz z | d } t| d krdz }n| d }tj| tj| | z tj |zg} | | tj| } tj| dddddf|| f} | || z} tj ddddf}tj||| f|| z}| |z} | dz krdz | | dz k<tj|| dd|S)an Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for ASR](https://huggingface.co/papers/1904.08779). Note that this method is not optimized to run on TPU and should be run on CPU as part of the preprocessing during training. Args: shape: The shape for which to compute masks. This should be of a tuple of size 2 where the first element is the batch size and the second element is the length of the axis to span. mask_prob: The percentage of the whole axis (between 0 and 1) which will be masked. The number of independently generated mask spans of length `mask_length` is computed by `mask_prob*shape[1]/mask_length`. Note that due to overlaps, `mask_prob` is an upper bound and the actual percentage will be smaller. mask_length: size of the mask min_masks: minimum number of masked spans attention_mask: A (right-padded) attention mask which independently shortens the feature axis of each batch dimension. rz&`mask_length` has to be bigger than 0.zO`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: z and `sequence_length`: `ct|zz z}t|}|zkrz}|dz z |krt|dz z d}|S)z;Given input length, compute how many spans should be maskedrr)rmax)ronum_masked_spanepsilonrrrrLs r"compute_num_masked_spanz6_compute_mask_indices..compute_num_masked_spanhs~i,6DwNOOoy99 [ (? : :-.{s 9 9 9!o 9 9 9r#rrF)replace)rprrritemdetachrtolistrr|rchoicerlen concatenaterxint32appendarrayrreshaperput_along_axis)rrrr}rrKrrg spec_aug_maskspec_aug_mask_idxsmax_num_masked_spanrorspec_aug_mask_idxdummy_mask_idxoffsetsrrLs `` ` @@r"_compute_mask_indicesrBsP0#(JQABBB_$$ :^i : :'6 : : :   innQ$$&&G         $  % ##B''..000 9 9 9 9uZ'8'8 9 9 9Hj/:$GGGM11/BBa%55 11,??I,, IlkAo6 7 7RW-    ! !Q & &-q0NN.q1NN (;o(MUWU] ^ ^ ^ao o p   !!"34444"455111aaa:&5H+(V,33J@SVa@abbi $$T4]3Gog 4G'UVV^^'+5G,g5/A"555GVYZGZ-!0CCDm%7B??? r#ceZdZdeffd ZdZ ddejdejdzdejdzfdZ e dd ej dzdej dzdejdzd e dzd e dzd e dzd e ezfdZxZS) WavLMModelrLct|||_t||_t ||_|jdks |jdkrBtj tj |j |_|jrt#||_nt'||_|jrt+|nd|_|dS)Nre)rrrLrfeature_extractorrQfeature_projectionmask_time_probmask_feature_probr;rvrwrr=r_masked_spec_embeddo_stable_layer_normr(encoderrrhradapter post_initr_s r"rzWavLMModel.__init__s     !4V!>DLL'//DL/5/AK|F+++t  r#c8|jdSz Calling this function will disable the gradient computation for the feature encoder so that its parameter will not be updated during training. N)rrrs r"freeze_feature_encoderz!WavLMModel.freeze_feature_encoders 1133333r#Nr'mask_time_indicesr}ct|jdds|S|\}}}|#|j|j||<n|jjdkr|jr|t||f|jj|jj ||jj }tj ||j tj}|j|j||<|jjdkr|jrt||f|jj|jj|jj}tj ||j tj}|dddfd|d}d||<|S) z Masks extracted features along time axis and/or along feature axis according to [SpecAugment](https://huggingface.co/papers/1904.08779). apply_spec_augmentTNr)rrr}r)rr)rrrrS)getattrrLrrrrrrrmask_time_lengthmask_time_min_masksrwtensorrrrmask_feature_lengthmask_feature_min_masksexpand)rr'rr}rKrLr=mask_feature_indicess r"_mask_hidden_stateszWavLMModel._mask_hidden_statesst{$8$?? ! 4A3E3E3G3G0 O[  (/3/E/H/HI\/]/]M+ , , [ '! + + + 5_-+4 K8-+9 !!! !& ->}G[chcm n n n /3/E/H/HI\/]/]M+ , ; (1 , , ,#8[)+7 K;+< $$$ $)<0D]Mainis#t#t#t #74#@#G#GO]_#`#` 23M. /r#rUrrrrc :||n |jj}||n |jj}||n |jj}||}|dd}|#||jd|d}||\} }| | ||} | | ||||} | d} |j | | } |s| |f| ddzSt| || j | j S) a/ mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*): Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict masked extracted features in *config.proj_codevector_dim* space. NrrFrz)rr}r}rrrr)rextract_featuresr'r)rLrrrrrOrrrrrrWavLMBaseModelOutputr'r) rrUr}rrrrkwargsrr'encoder_outputss r"r(zWavLMModel.forwards| 2C1N--TXT_Tq$8$D $+Jj &1%uEN+/*A*ABR*S*S' '00 ->~1  ,, )/!5# '  (* < # LL77M K!#34qrr7JJ J#+-)7&1     r#)NNNNNNN)r*r+r,rrrrwrrrrrrrrr(r-r.s@r"rrsF{(4447;26 ,,(,!,t3,(4/ ,,,,\/36:)-,0#'8 8 lT)8  t+8 !,t3 8  $; 8 #Tk 8 D[8  % %8 8 8 ^8 8 8 8 8 r#rrzm WavLM Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). ) custom_introceZdZddedzffd ZdZdZdZe dde j dzde j dzd e dzd e dzd e dzd e j dzd e e zfdZxZS) WavLMForCTCN target_langct|t||_t j|j|_||_|j td|j dt|dr|j r|jn|j}t j||j |_|dS)a/ target_lang (`str`, *optional*): Language id of adapter weights. Adapter weights are stored in the format adapter..safetensors or adapter..bin. Only relevant when using an instance of [`WavLMForCTC`] with adapters. Uses 'eng' by default. NzYou are trying to instantiate z with a configuration that does not define the vocabulary size of the language model head. Please instantiate the model as follows: `WavLMForCTC.from_pretrained(..., vocab_size=vocab_size)`. or define `vocab_size` of your model's configuration.rh)rrrrTr;r\ final_dropoutr^r  vocab_sizerpr!rArhrr=rZlm_headr)rrLr rr!s r"rzWavLMForCTC.__init__Ls    '' z&"677 &   $HHHH *1)G)G vFL^ vF % %djdv y!3V5FGG  r#c TttjdkrdS|j}|)t |jddt d|d|2t |jddtddS|| |ddSdS) a' This method overwrites [`~PreTrainedModel.tie_weights`] so that adapter weights can be correctly loaded when passing `target_lang=...` to `from_pretrained(...)`. This method is **not** supposed to be called by the user and is prone to be changed in the future. metaNadapter_attn_dimzCannot pass `target_lang`: z- if `config.adapter_attn_dim` is not defined.z)By default `target_lang` is set to 'eng'.T) force_load) rrwrr rrLrploggerinfo load_adapter)rrr s r" tie_weightszWavLMForCTC.tie_weightsis 6 7 75<;O;O O O F &  "wt{||jjkrt d|jj||||||}|d} || } || } d} |Z||ntj |tj }| | d tj } |dk} | d}|| }tj| dtjdd}tjjd 5tj||| ||jj|jj|jj } dddn #1swxYwY|s| f|t6dz}| | f|zn|St9| | |j|j S) a labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*): Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size - 1]`. Nz$Label values must be <= vocab_size: rrrrS)r9rrF)enabled)blank reduction zero_infinitylosslogitsr'r)rLrrr rprTr^r rw ones_likerrwrr masked_selectr;rD log_softmaxfloat32rObackendscudnnflagsctc_loss pad_token_idctc_loss_reductionctc_zero_infinity_HIDDEN_STATES_START_POSITIONr r'r)rrUr}rrrrrrr'r&r%rg labels_masktarget_lengthsflattened_targets log_probsoutputs r"r(zWavLMForCTC.forwards$&1% 1` a classification loss is computed (Cross-Entropy). NTrrrrSrrrer$)rLrr;rTr2rwstackr;rDrGr<rrr>r9rrrrr@rr?r r'r)rrUr}rrrrrrr' norm_weights pooled_output padding_maskexpand_padding_maskr&r%loss_fctr7s r"r(z&WavLMForSequenceClassification.forwardsW0&1%->r1a-H-HHMMRSMTTMM#AJM}55  !)..1.55MMBB=CVWXCY[ijjL"."8"8"<"<"C"CAq-J]^_J`"a"a 25M.. /)--!-44|7G7GA7G7N7N7S7STVXY7Z7ZZM//  '))H8FKKDK,BCCV[[QS__UUD FY)F)G)G!HHF)-)9TGf$$v E'!/)     r#r)r*r+r,rrrrrwrrrr r(r-r.s@r"r9r9s":::(((/3)-,0#'&*C C lT)C  t+C  $; C #Tk C D[ C  t#C  ) )C C C ^C C C C C r#r9ceZdZfdZdZdZe d dejdzdejdzdejdzde dzd e dzd e dzd e e zfd Z xZ S) WavLMForAudioFrameClassificationct|t|dr|jrt dt ||_|jdz}|jr.tj tj ||z |_ tj|j|j|_|j|_|dS)Nrhz_Audio frame classification does not support the use of WavLM adapters (config.add_adapter=True)r)rrrArhrprrTr r;r;rvrwrxr<rZr=r?r@rrAs r"rz)WavLMForAudioFrameClassification.__init__Is    6= ) ) f.@ q  '' -1  ( S!#ej.D.Dz.Q!R!RD )F$68IJJ + r#cB|jjdSrrrs r"rz7WavLMForAudioFrameClassification.freeze_feature_encoderYrr#cL|jD] }d|_ dSrrrs r"rz2WavLMForAudioFrameClassification.freeze_base_model`rr#NrUr}rrrrrc ||n |jj}|jjrdn|}||||||}|jjrx|t} t j| d} tj |j d} | | dddz d} n|d} | | } d} |`t} | | d|jt j| d|jd} |s| f|tdz}|St#| | |j|j S) rFNTrrrrSr)axisr$)rLrr;rTr2rwrGr;rDrGr<rrr@rr?rHr r'r)rrUr}rrrrrrr'rHr&r%rLr7s r"r(z(WavLMForAudioFrameClassification.forwardhs0&1%->r1a-H-HHMMRSMTTMM#AJM//  '))H8FKKDO<t>t>tuuD Y)F)G)G!HHFM$!/)     r#r)r*r+r,rrrrrwrrrr r(r-r.s@r"rNrNGs :::(((/3&*)-,0#': : lT):  t+:  t# :  $; : #Tk : D[:  & &: : : ^: : : : : r#rNc&eZdZdfd ZdZxZS) AMSoftmaxLoss>@皙?ct||_||_||_t jtj||d|_ t j |_ dS)NT)r) rrscalemarginr?r;rvrwrandnr7rr%)r input_dimr?rYrZr!s r"rzAMSoftmaxLoss.__init__se   $l5;y*#E#EUYZZZ ')) r#c|}tj|jd}tj|d}t j||}||jz }tj||j }|j t j | ||z}| ||}|S)Nrrr)flattenr;rD normalizer7rwmmrZone_hotr?rYrrr%) rr'rr7 cos_thetapsionehotr&r%s r"r(zAMSoftmaxLoss.forwards!!((!(<< // 1/EE H]F33 $+%&&vt??ek&++--iHHHyy(( r#)rVrWr)r.s@r"rUrUsL******       r#rUcDeZdZdfd ZdejdejfdZxZS) TDNNLayerrct|dkr|j|dz n |j||_|j||_|j||_|j||_tj |j|jz|j|_ tj |_ dS)Nrr)rrtdnn_dimrr tdnn_kernelr2 tdnn_dilationdilationr;rZkernelReLUrKrs r"rzTDNNLayer.__init__s .s#QQQy++QQQr#rSr)rrrrTr r;r;rvrwrxr<rZr=rhr>rrrtdnnxvector_output_dimrr@rUr? objectiver)rrLrB tdnn_layersr!s ` r"rzWavLMForXVector.__init__s'    '' -1  ( S!#ej.D.Dz.Q!R!RD 6#5vq7IJJQQQQU3v;O;O5P5PQQQ M+.. !#6?2+>+BFD]!^!^)F$=v?XYY&v'@&BSTT r#cB|jjdSrrrs r"rz&WavLMForXVector.freeze_feature_encoderrr#cL|jD] }d|_ dSrrrs r"rz!WavLMForXVector.freeze_base_modelrr#rgcDd}|jjD]}|||d}|S)z? Computes the output length of the TDNN layers c||z |zdzS)Nrr rns r"rqzBWavLMForXVector._get_tdnn_output_lengths.._conv_out_length s!;.69A= =r#r)rLri)rrgrqr2s r"_get_tdnn_output_lengthsz(WavLMForXVector._get_tdnn_output_lengthssE  > > >  ;2 L LK,,]KKKMMr#NrUr}rrrrrc >||n |jj}|jjrdn|}||||||}|jjrx|t} t j| d} tj |j d} | | dddz d} n|d} | | } |jD] } | | } |-| d} | d} n|| d}||}g} g} t'|D]k\}}| | |d|fd| | |d|fdlt j| } t j| } t j| | gd}||}||}d}||||}|s||f|tdz}||f|zn|St3||||j|jS) rFNTrrrrSr)r%r& embeddingsr'r)rLrr;rTr2rwrGr;rDrGr<rrr>rxr9rXrwrrrrrr@rzrr'r)rrUr}rrrrrrr'rH tdnn_layer mean_features std_featuresfeat_extract_output_lengthstdnn_output_lengthsrlengthstatistic_poolingoutput_embeddingsr&r%r7s r"r(zWavLMForXVector.forwards0&1%->r1a-H-HHMMRSMTTMM#AJM}55 ) 6 6J&J}55MM  !)..1.55M(,,,33LL*.*O*OP^PbPbghPbPiPi*j*j '"&"?"?@["\"\ ML&':;; J J 6$$]1gvg:%>%C%C%C%J%JKKK##M!WfW*$=$A$Aa$A$H$HIIII!K 66M ;|44L!I}l&CLLL 223DEE!233  >>&&11D F/07;X;Y;Y3ZZF)-)9TGf$$v E(!/)     r#r)r*r+r,rrrrwrrrrrrrrr(r-r.s@r"rurus) &:::((( e6F6L    /3)-,0#'&*P P lT)P  t+P  $; P #Tk P D[ P  t#P   P P P ^P P P P P r#ru)rNrr9rurrSr%)Krrqnumpyrrwtorch.nnr;torch.nn.functionalrDrrrr\ activationsrintegrations.deepspeedrintegrations.fsdprmodeling_layersr modeling_outputsr r r r rrmodeling_utilsrrr@rrrconfiguration_wavlmr get_loggerr*rModulerr0rQrdrrrrr(r.rSrrrrrrrrrrndarrayrrrr2rr9rNrUrfru__all__r r#r"rs  %%%%%%&&&&&&!!!!!!@@@@@@777777999999ZYYYYYYY??????????,,,,,,  H % % *****29***Z 1 1 1 1 1RY 1 1 1c c c c c RYc c c Lry0&&&&&2&&&R"""""'A"""JG G G G G 29G G G TH H H H H ")H H H VC'C'C'C'C'C'C'C'LLLLLL?LLL^ :*8680#####")###L $29F/3 tt c?ttt$t+ t  t Z ttttn/C C C C C %C C C L!" K K K K K &K K  K \ e e e e e %9e e  e P[ [ [ [ [ ';[ [ [ |BI. @ C C C C C *C C  C L   r#