§
    ‚Štjm²  ã                   ó0  — d dl Z d dlmZ d dlZd dlZd dlmZ d dlmZ ddl	m
Z ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZmZ ddlmZmZmZ ddlmZ ddl m!Z!m"Z"m#Z# ddl$m%Z% ddl&m'Z'  e#j(        e)¦  «        Z* G d„ de¦  «        Z+ G d„ de¦  «        Z, G d„ de¦  «        Z- G d„ dej.        ¦  «        Z/ G d„ dej.        ¦  «        Z0 G d„ dej.        ¦  «        Z1 G d„ d ej.        ¦  «        Z2	 	 dEd"ej.        d#ej3        d$ej3        d%ej3        d&ej3        dz  d'e4dz  d(e4d)ee!         fd*„Z5 G d+„ d,ej.        ¦  «        Z6 G d-„ d.ej.        ¦  «        Z7 G d/„ d0e¦  «        Z8 G d1„ d2ej.        ¦  «        Z9e" G d3„ d4e¦  «        ¦   «         Z:	 	 dFd5e;e<e<f         d6e4d7e<d&ej=        dz  d8e<d9ej>        fd:„Z?e" G d;„ d<e:¦  «        ¦   «         Z@dZA e"d=¬>¦  «         G d?„ d@e:¦  «        ¦   «         ZB e"dA¬>¦  «         G dB„ dCe:¦  «        ¦   «         ZCg dD¢ZDdS )Gé    N)ÚCallable)Únn)ÚCrossEntropyLossé   )Úinitialization)ÚACT2FN)Úis_deepspeed_zero3_enabled)Úis_fsdp_managed_module)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚCausalLMOutputÚSequenceClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModelÚ*get_torch_context_manager_or_global_device)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)Úis_flash_attention_requestedé   )Ú	SEWConfigc                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚSEWNoLayerNormConvLayerr   c                 óZ  •— t          ¦   «                              ¦   «          |dk    r|j        |dz
           nd| _        |j        |         | _        t          j        | j        | j        |j        |         |j        |         |j	        ¬¦  «        | _
        t          |j                 | _        d S )Nr   r   ©Úkernel_sizeÚstrideÚbias)ÚsuperÚ__init__Úconv_dimÚin_conv_dimÚout_conv_dimr   ÚConv1dÚconv_kernelÚconv_strideÚ	conv_biasÚconvr   Úfeat_extract_activationÚ
activation©ÚselfÚconfigÚlayer_idÚ	__class__s      €úb/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/sew/modeling_sew.pyr"   z SEWNoLayerNormConvLayer.__init__/   s�   ø€ Ý‰Œ×ÒÑÔÐØ<DÀqºL¸L˜6œ?¨8°a©<Ô8Ð8ÈaˆÔØ"œO¨HÔ5ˆÔå”IØÔØÔØÔ*¨8Ô4ØÔ% hÔ/ØÔ!ð
ñ 
ô 
ˆŒ	õ ! Ô!?Ô@ˆŒˆˆó    c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S ©N)r*   r,   ©r.   Úhidden_statess     r2   ÚforwardzSEWNoLayerNormConvLayer.forward=   s*   € ØŸ	š	 -Ñ0Ô0ˆØŸš¨Ñ6Ô6ˆØÐr3   ©r   ©Ú__name__Ú
__module__Ú__qualname__r"   r8   Ú__classcell__©r1   s   @r2   r   r   .   sR   ø€ € € € € ðAð Að Að Að Að Aðð ð ð ð ð ð r3   r   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚSEWLayerNormConvLayerr   c                 óš  •— t          ¦   «                              ¦   «          |dk    r|j        |dz
           nd| _        |j        |         | _        t          j        | j        | j        |j        |         |j        |         |j	        ¬¦  «        | _
        t          j        | j        d¬¦  «        | _        t          |j                 | _        d S )Nr   r   r   T)Úelementwise_affine)r!   r"   r#   r$   r%   r   r&   r'   r(   r)   r*   Ú	LayerNormÚ
layer_normr   r+   r,   r-   s      €r2   r"   zSEWLayerNormConvLayer.__init__D   s¶   ø€ Ý‰Œ×ÒÑÔÐØ<DÀqºL¸L˜6œ?¨8°a©<Ô8Ð8ÈaˆÔØ"œO¨HÔ5ˆÔå”IØÔØÔØÔ*¨8Ô4ØÔ% hÔ/ØÔ!ð
ñ 
ô 
ˆŒ	õ œ, tÔ'8ÈTÐRÑRÔRˆŒÝ  Ô!?Ô@ˆŒˆˆr3   c                 óÜ   — |                       |¦  «        }|                     dd¦  «        }|                      |¦  «        }|                     dd¦  «        }|                      |¦  «        }|S )Néþÿÿÿéÿÿÿÿ)r*   Ú	transposerE   r,   r6   s     r2   r8   zSEWLayerNormConvLayer.forwardS   se   € ØŸ	š	 -Ñ0Ô0ˆà%×/Ò/°°BÑ7Ô7ˆØŸš¨Ñ6Ô6ˆØ%×/Ò/°°BÑ7Ô7ˆàŸš¨Ñ6Ô6ˆØÐr3   r9   r:   r?   s   @r2   rA   rA   C   sR   ø€ € € € € ðAð Að Að Að Að Aðð ð ð ð ð ð r3   rA   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚSEWGroupNormConvLayerr   c                 ó¦  •— t          ¦   «                              ¦   «          |dk    r|j        |dz
           nd| _        |j        |         | _        t          j        | j        | j        |j        |         |j        |         |j	        ¬¦  «        | _
        t          |j                 | _        t          j        | j        | j        d¬¦  «        | _        d S )Nr   r   r   T)Ú
num_groupsÚnum_channelsÚaffine)r!   r"   r#   r$   r%   r   r&   r'   r(   r)   r*   r   r+   r,   Ú	GroupNormrE   r-   s      €r2   r"   zSEWGroupNormConvLayer.__init___   s½   ø€ Ý‰Œ×ÒÑÔÐØ<DÀqºL¸L˜6œ?¨8°a©<Ô8Ð8ÈaˆÔØ"œO¨HÔ5ˆÔå”IØÔØÔØÔ*¨8Ô4ØÔ% hÔ/ØÔ!ð
ñ 
ô 
ˆŒ	õ ! Ô!?Ô@ˆŒåœ,°$Ô2CÐRVÔRcÐlpÐqÑqÔqˆŒˆˆr3   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r5   )r*   rE   r,   r6   s     r2   r8   zSEWGroupNormConvLayer.forwardo   s;   € ØŸ	š	 -Ñ0Ô0ˆØŸš¨Ñ6Ô6ˆØŸš¨Ñ6Ô6ˆØÐr3   r9   r:   r?   s   @r2   rK   rK   ^   sR   ø€ € € € € ðrð rð rð rð rð rð ð ð ð ð ð ð r3   rK   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚSEWPositionalConvEmbeddingc                 óÖ  •— t          ¦   «                              ¦   «          t          j        |j        |j        |j        |j        dz  |j        |j        ¬¦  «        | _        t          j	        j
        }t          t          j	        j        d¦  «        rt          j	        j        j
        }t          ¦   «         rëdd l}|j                             | j        j        d¬¦  «        5   || j        dd¬¦  «        | _        d d d ¦  «         n# 1 swxY w Y   t          | j        d¦  «        r-| j        j        j        j        }| j        j        j        j        }n| j        j        }| j        j        }|j                             | |¦  «         |j                             | |¦  «         n || j        dd¬¦  «        | _        t/          |j        ¦  «        | _        t2          |j                 | _        d S )	Né   )r   ÚpaddingÚgroupsr   Úweight_normr   ©Úmodifier_rankÚweight)ÚnameÚdimÚparametrizations)r!   r"   r   r&   Úhidden_sizeÚnum_conv_pos_embeddingsÚnum_conv_pos_embedding_groupsÚsqueeze_factorr*   ÚutilsrX   Úhasattrr^   r	   Ú	deepspeedÚzeroÚGatheredParametersr[   Ú	original0Ú	original1Úweight_gÚweight_vÚregister_external_parameterÚSEWSamePadLayerrV   r   r+   r,   )r.   r/   rX   re   rj   rk   r1   s         €r2   r"   z#SEWPositionalConvEmbedding.__init__w   s  ø€ Ý‰Œ×ÒÑÔÐÝ”IØÔØÔØÔ6ØÔ2°aÑ7ØÔ7ØÔ(ð
ñ 
ô 
ˆŒ	õ ”hÔ*ˆÝ•2”8Ô,¨mÑ<Ô<ð 	@Ýœ(Ô3Ô?ˆKå%Ñ'Ô'ð 	EØÐÐÐà”×2Ò2°4´9Ô3CÐSTÐ2ÑUÔUð Ið IØ'˜K¨¬	¸ÀaÐHÑHÔH�”	ðIð Ið Iñ Iô Ið Ið Ið Ið Ið Ið Iøøøð Ið Ið Ið Iå�t”yÐ"4Ñ5Ô5ð .Øœ9Ô5Ô<ÔF�Øœ9Ô5Ô<ÔF��àœ9Ô-�Øœ9Ô-�ØŒN×6Ò6°t¸XÑFÔFÐFØŒN×6Ò6°t¸XÑFÔFÐFÐFà#˜ D¤I°HÀ!ÐDÑDÔDˆDŒIå& vÔ'EÑFÔFˆŒÝ  Ô!?Ô@ˆŒˆˆs   Ã DÄD	ÄD	c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r5   )r*   rV   r,   r6   s     r2   r8   z"SEWPositionalConvEmbedding.forward™   s;   € ØŸ	š	 -Ñ0Ô0ˆØŸš ]Ñ3Ô3ˆØŸš¨Ñ6Ô6ˆàÐr3   r:   r?   s   @r2   rS   rS   v   sM   ø€ € € € € ð Að  Að  Að  Að  AðDð ð ð ð ð ð r3   rS   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )rm   c                 ól   •— t          ¦   «                              ¦   «          |dz  dk    rdnd| _        d S )NrU   r   r   )r!   r"   Únum_pad_remove)r.   r`   r1   s     €r2   r"   zSEWSamePadLayer.__init__¢   s:   ø€ Ý‰Œ×ÒÑÔÐØ#:¸QÑ#>À!Ò#CÐ#C˜a˜aÈˆÔÐÐr3   c                 óJ   — | j         dk    r|d d …d d …d | j          …f         }|S ©Nr   )rq   r6   s     r2   r8   zSEWSamePadLayer.forward¦   s;   € ØÔ Ò"Ð"Ø)¨!¨!¨!¨Q¨Q¨QÐ0F°4Ô3FÐ2FÐ0FÐ*FÔGˆMØÐr3   r:   r?   s   @r2   rm   rm   ¡   sL   ø€ € € € € ðKð Kð Kð Kð Kðð ð ð ð ð ð r3   rm   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚSEWUpsamplingc                 óæ   •— t          ¦   «                              ¦   «          t          j        |j        |j        |j        z  ¦  «        | _        t          |j                 | _	        |j        | _        d S r5   )
r!   r"   r   ÚLinearr_   rb   Ú
projectionr   r+   r,   ©r.   r/   r1   s     €r2   r"   zSEWUpsampling.__init__­   sZ   ø€ Ý‰Œ×ÒÑÔÐÝœ) FÔ$6¸Ô8JÈVÔMbÑ8bÑcÔcˆŒÝ  Ô!?Ô@ˆŒØ$Ô3ˆÔÐÐr3   c                 ó0  — |                       |¦  «        }|                      |¦  «        }| j        dk    r`|                     ¦   «         \  }}}|| j        z  }|| j        z  }|                     ||| j        |¦  «        }|                     |||¦  «        }|S )Nr   )rx   r,   rb   ÚsizeÚreshape)r.   r7   ÚbszÚsrc_lenÚsrc_embed_dimÚtgt_lenÚtgt_embed_dims          r2   r8   zSEWUpsampling.forward³   sž   € ØŸš¨Ñ6Ô6ˆØŸš¨Ñ6Ô6ˆàÔ Ò"Ð"à*7×*<Ò*<Ñ*>Ô*>Ñ'ˆC�˜-Ø Ô 3Ñ3ˆGØ)¨TÔ-@Ñ@ˆMØ)×1Ò1°#°wÀÔ@SÐUbÑcÔcˆMØ)×1Ò1°#°wÀÑNÔNˆMàÐr3   r:   r?   s   @r2   ru   ru   ¬   sG   ø€ € € € € ð4ð 4ð 4ð 4ð 4ðð ð ð ð ð ð r3   ru   c                   ó.   ‡ — e Zd ZdZˆ fd„Zd„ Zd„ Zˆ xZS )ÚSEWFeatureEncoderz.Construct the features from raw audio waveformc                 ó¤  •‡— t          ¦   «                              ¦   «          ‰j        dk    r7t          ‰d¬¦  «        gˆfd„t	          ‰j        dz
  ¦  «        D ¦   «         z   }nD‰j        dk    r!ˆfd„t	          ‰j        ¦  «        D ¦   «         }nt          d‰j        › d	�¦  «        ‚t          j        |¦  «        | _	        d
| _
        d| _        d S )NÚgroupr   ©r0   c                 ó8   •— g | ]}t          ‰|d z   ¬¦  «        ‘ŒS )r   r†   )r   ©Ú.0Úir/   s     €r2   ú
<listcomp>z.SEWFeatureEncoder.__init__.<locals>.<listcomp>É   s>   ø€ ð Ið Ið IØDEÕ'¨¸¸Q¹Ð?Ñ?Ô?ðIð Ið Ir3   r   Úlayerc                 ó2   •— g | ]}t          ‰|¬ ¦  «        ‘ŒS )r†   )rA   rˆ   s     €r2   r‹   z.SEWFeatureEncoder.__init__.<locals>.<listcomp>Í   s'   ø€ ÐtÐtÐtÈÕ0°À!ÐDÑDÔDÐtÐtÐtr3   z`config.feat_extract_norm` is z), but has to be one of ['group', 'layer']FT)r!   r"   Úfeat_extract_normrK   ÚrangeÚnum_feat_extract_layersÚ
ValueErrorr   Ú
ModuleListÚconv_layersÚgradient_checkpointingÚ_requires_grad)r.   r/   r“   r1   s    ` €r2   r"   zSEWFeatureEncoder.__init__Å   s  øø€ Ý‰Œ×ÒÑÔÐàÔ# wÒ.Ð.Ý0°À!ÐDÑDÔDÐEð Ið Ið Ið IÝINÈvÔOmÐpqÑOqÑIrÔIrðIñ Iô Iñ ˆKˆKð Ô%¨Ò0Ð0ØtÐtÐtÐtÍeÐTZÔTrÑNsÔNsÐtÑtÔtˆKˆKåØt°Ô1IÐtÐtÐtñô ð õ œ=¨Ñ5Ô5ˆÔØ&+ˆÔ#Ø"ˆÔÐÐr3   c                 óP   — |                       ¦   «         D ]	}d|_        Œ
d| _        d S ©NF)Ú
parametersÚrequires_gradr•   ©r.   Úparams     r2   Ú_freeze_parametersz$SEWFeatureEncoder._freeze_parametersÖ   s4   € Ø—_’_Ñ&Ô&ð 	(ð 	(ˆEØ"'ˆEÔÐØ#ˆÔÐÐr3   c                 ór   — |d d …d f         }| j         r| j        rd|_        | j        D ]} ||¦  «        }Œ|S )NT)r•   Útrainingr™   r“   )r.   Úinput_valuesr7   Ú
conv_layers       r2   r8   zSEWFeatureEncoder.forwardÛ   s[   € Ø$ Q Q Q¨ WÔ-ˆð Ôð 	/ 4¤=ð 	/Ø*.ˆMÔ'àÔ*ð 	6ð 	6ˆJØ&˜J }Ñ5Ô5ˆMˆMàÐr3   )r;   r<   r=   Ú__doc__r"   rœ   r8   r>   r?   s   @r2   rƒ   rƒ   Â   s\   ø€ € € € € Ø8Ð8ð#ð #ð #ð #ð #ð"$ð $ð $ð

ð 
ð 
ð 
ð 
ð 
ð 
r3   rƒ   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 ó®  — |€|                      d¦  «        dz  }t          j        ||                     dd¦  «        ¦  «        |z  }|�||z   }t          j                             |d¬¦  «        }t          j                             ||| j        ¬¦  «        }t          j        ||¦  «        }	|	                     dd¦  «         	                    ¦   «         }	|	|fS )NrH   ç      à¿rU   r   ©r]   )Úprž   r   )
r{   ÚtorchÚmatmulrI   r   Ú
functionalÚsoftmaxr©   rž   Ú
contiguous)
r£   r¤   r¥   r¦   r§   r¨   r©   rª   Úattn_weightsÚattn_outputs
             r2   Úeager_attention_forwardr¶   è   sÈ   € ð €Ø—*’*˜R‘.”. DÑ(ˆõ ”<  s§}¢}°Q¸Ñ':Ô':Ñ;Ô;¸gÑE€LàÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2Ð(Ñ>Ô>€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r3   c                   óþ   ‡ — e Zd ZdZ	 	 	 	 	 ddededed	ed
edededz  fˆ fd„Z	 	 	 dde	j
        de	j
        dz  de	j
        dz  dedz  dee         dee	j
        e	j
        dz  ee	j
                 dz  f         fd„Zˆ xZS )ÚSEWAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr¢   FTNÚ	embed_dimÚ	num_headsr©   Ú
is_decoderr    Ú	is_causalr/   c                 ó
  •— t          ¦   «                              ¦   «          || _        || _        || _        ||z  | _        || _        | j        |z  | j        k    rt          d| j        › d|› d�¦  «        ‚| j        dz  | _        || _	        || _
        t          j        |||¬¦  «        | _        t          j        |||¬¦  «        | _        t          j        |||¬¦  «        | _        t          j        |||¬¦  «        | _        d S )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).r¬   )r    )r!   r"   r¹   rº   r©   Úhead_dimr/   r‘   r¨   r»   r¼   r   rw   Úk_projÚv_projÚq_projÚout_proj)	r.   r¹   rº   r©   r»   r    r¼   r/   r1   s	           €r2   r"   zSEWAttention.__init__  s  ø€ õ 	‰Œ×ÒÑÔÐØ"ˆŒØ"ˆŒØˆŒØ! YÑ.ˆŒØˆŒàŒM˜IÑ%¨$¬.Ò8Ð8Ýð3ÈdÌnð 3ð 3Ø%.ð3ð 3ð 3ñô ð ð ”} dÑ*ˆŒØ$ˆŒØ"ˆŒå”i 	¨9¸4Ð@Ñ@Ô@ˆŒÝ”i 	¨9¸4Ð@Ñ@Ô@ˆŒÝ”i 	¨9¸4Ð@Ñ@Ô@ˆŒÝœ	 )¨Y¸TÐBÑBÔBˆŒˆˆr3   r7   Úkey_value_statesr§   Úoutput_attentionsrª   Úreturnc                 óú  — |du}|j         dd…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }	|r|n|}
g |
j         dd…         ¢d‘| j        ‘R }|                      |
¦  «                             |¦  «                             dd¦  «        }|                      |
¦  «                             |¦  «                             dd¦  «        }t          j        | j	        j
        t          ¦  «        } || |	|||f| j        sdn| j        | j        |dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||dfS )z#Input shape: Batch x Time x ChannelNrH   r   rU   r¢   )r©   r¨   rÄ   )Úshaper¾   rÁ   ÚviewrI   r¿   rÀ   r   Úget_interfacer/   Ú_attn_implementationr¶   rž   r©   r¨   r|   r³   rÂ   )r.   r7   rÃ   r§   rÄ   rª   Úis_cross_attentionÚinput_shapeÚhidden_shapeÚquery_statesÚcurrent_statesÚkv_shapeÚ
key_statesÚvalue_statesÚattention_interfacerµ   r´   s                    r2   r8   zSEWAttention.forward&  s½  € ð .°TÐ9Ðð $Ô)¨#¨2¨#Ô.ˆà8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆð —{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆà-?ÐRÐ)Ð)À]ˆØB�^Ô)¨#¨2¨#Ô.ÐB°ÐB°D´MÐBÐBˆØ—[’[ Ñ0Ô0×5Ò5°hÑ?Ô?×IÒIÈ!ÈQÑOÔOˆ
Ø—{’{ >Ñ2Ô2×7Ò7¸ÑAÔA×KÒKÈAÈqÑQÔQˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð  $œ}Ð>�C�C°$´,Ø”LØ/ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—m’m KÑ0Ô0ˆà˜L¨$Ð.Ð.r3   )r¢   FTFN)NNF)r;   r<   r=   r¡   ÚintÚfloatÚboolr   r"   r¯   ÚTensorr   r   Útupler8   r>   r?   s   @r2   r¸   r¸     sJ  ø€ € € € € ØGÐGð Ø ØØØ#'ðCð CàðCð ðCð ð	Cð
 ðCð ðCð ðCð ˜DÑ ðCð Cð Cð Cð Cð CðD 15Ø.2Ø).ð0/ð 0/à”|ð0/ð  œ,¨Ñ-ð0/ð œ tÑ+ð	0/ð
   $™;ð0/ð Ð-Ô.ð0/ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	Mð0/ð 0/ð 0/ð 0/ð 0/ð 0/ð 0/ð 0/r3   r¸   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚSEWFeedForwardc                 óÌ  •— t          ¦   «                              ¦   «          t          j        |j        ¦  «        | _        t          j        |j        |j        ¦  «        | _	        t          |j        t          ¦  «        rt          |j                 | _        n|j        | _        t          j        |j        |j        ¦  «        | _        t          j        |j        ¦  «        | _        d S r5   )r!   r"   r   ÚDropoutÚactivation_dropoutÚintermediate_dropoutrw   r_   Úintermediate_sizeÚintermediate_denseÚ
isinstanceÚ
hidden_actÚstrr   Úintermediate_act_fnÚoutput_denseÚhidden_dropoutÚoutput_dropoutry   s     €r2   r"   zSEWFeedForward.__init__Z  s°   ø€ Ý‰Œ×ÒÑÔÐÝ$&¤J¨vÔ/HÑ$IÔ$IˆÔ!å"$¤)¨FÔ,>ÀÔ@XÑ"YÔ"YˆÔÝ�fÔ'­Ñ-Ô-ð 	9Ý'-¨fÔ.?Ô'@ˆDÔ$Ð$à'-Ô'8ˆDÔ$åœI fÔ&>ÀÔ@RÑSÔSˆÔÝ œj¨Ô)>Ñ?Ô?ˆÔÐÐr3   c                 óØ   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r5   )rà   rä   rÞ   rå   rç   r6   s     r2   r8   zSEWFeedForward.forwardg  sg   € Ø×/Ò/°Ñ>Ô>ˆØ×0Ò0°Ñ?Ô?ˆØ×1Ò1°-Ñ@Ô@ˆà×)Ò)¨-Ñ8Ô8ˆØ×+Ò+¨MÑ:Ô:ˆØÐr3   r:   r?   s   @r2   rÚ   rÚ   Y  sL   ø€ € € € € ð@ð @ð @ð @ð @ðð ð ð ð ð ð r3   rÚ   c                   ó&   ‡ — e Zd Zˆ fd„Zdd„Zˆ xZS )ÚSEWEncoderLayerc                 ó�  •— t          ¦   «                              ¦   «          t          |j        |j        |j        d|¬¦  «        | _        t          j        |j	        ¦  «        | _
        t          j        |j        |j        ¬¦  «        | _        t          |¦  «        | _        t          j        |j        |j        ¬¦  «        | _        d S )NF)r¹   rº   r©   r»   r/   ©Úeps)r!   r"   r¸   r_   Únum_attention_headsÚattention_dropoutÚ	attentionr   rÜ   ræ   r©   rD   Úlayer_norm_epsrE   rÚ   Úfeed_forwardÚfinal_layer_normry   s     €r2   r"   zSEWEncoderLayer.__init__r  s¬   ø€ Ý‰Œ×ÒÑÔÐÝ%ØÔ(ØÔ0ØÔ,ØØð
ñ 
ô 
ˆŒõ ”z &Ô"7Ñ8Ô8ˆŒÝœ, vÔ'9¸vÔ?TÐUÑUÔUˆŒÝ*¨6Ñ2Ô2ˆÔÝ "¤¨VÔ-?ÀVÔEZÐ [Ñ [Ô [ˆÔÐÐr3   NFc                 ó  — |}|                       |||¬¦  «        \  }}}|                      |¦  «        }||z   }|                      |¦  «        }||                      |¦  «        z   }|                      |¦  «        }|f}|r||fz  }|S )N©r§   rÄ   )rð   r©   rE   rò   ró   )r.   r7   r§   rÄ   Úattn_residualr´   Ú_Úoutputss           r2   r8   zSEWEncoderLayer.forward�  s¨   € Ø%ˆØ)-¯ªØ¨.ÐL]ð *8ñ *
ô *
Ñ&ˆ�| Qð Ÿš ]Ñ3Ô3ˆØ%¨Ñ5ˆàŸš¨Ñ6Ô6ˆØ%¨×(9Ò(9¸-Ñ(HÔ(HÑHˆØ×-Ò-¨mÑ<Ô<ˆà Ð"ˆàð 	'Ø˜�Ñ&ˆGàˆr3   r—   r:   r?   s   @r2   rê   rê   q  sQ   ø€ € € € € ð\ð \ð \ð \ð \ðð ð ð ð ð ð ð r3   rê   c                   ó.   ‡ — e Zd Zˆ fd„Z	 	 	 	 dd„Zˆ xZS )Ú
SEWEncoderc                 óò  •‡— t          ¦   «                              ¦   «          ‰| _        t          ‰¦  «        | _        t          j        ‰j        ‰j        ¦  «        | _        t          j	        ‰j
        ‰j        ¬¦  «        | _        t          j        ‰j        ¦  «        | _        t          j        ˆfd„t#          ‰j        ¦  «        D ¦   «         ¦  «        | _        t)          ‰¦  «        | _        d| _        d S )Nrì   c                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS © )rê   )r‰   r÷   r/   s     €r2   r‹   z'SEWEncoder.__init__.<locals>.<listcomp>�  s!   ø€ Ð$fÐ$fÐ$fÀ¥_°VÑ%<Ô%<Ð$fÐ$fÐ$fr3   F)r!   r"   r/   rS   Úpos_conv_embedr   Ú	AvgPool1drb   ÚpoolrD   r_   rñ   rE   rÜ   ræ   r©   r’   r�   Únum_hidden_layersÚlayersru   Úupsampler”   ry   s    `€r2   r"   zSEWEncoder.__init__–  sÈ   øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ8¸Ñ@Ô@ˆÔÝ”L Ô!6¸Ô8MÑNÔNˆŒ	Ýœ, vÔ'9¸vÔ?TÐUÑUÔUˆŒÝ”z &Ô"7Ñ8Ô8ˆŒÝ”mÐ$fÐ$fÐ$fÐ$fÅeÈFÔLdÑFeÔFeÐ$fÑ$fÔ$fÑgÔgˆŒÝ% fÑ-Ô-ˆŒØ&+ˆÔ#Ð#Ð#r3   NFTc           	      ó^  — |rdnd }|rdnd }|��-|                      d¦  «                             dd|j        d         ¦  «        }t          | j        ¦  «        rd|| <   |�d|v r|nd }nÓd|| <   t
          j                             |                     ¦   «                               d¦  «        | j        j	        | j        j	        ¬¦  «         
                    d¦  «                             ¦   «         }d|d d …d d d d …f                              |j        ¬	¦  «        z
  }|t          j        |j        ¦  «        j        z  }|j        d         }	|                     dd¦  «        }|                      |¦  «        }
|                      |¦  «        }t!          |
                     d¦  «        |                     d¦  «        ¦  «        }|d
d |…f         |
d
d |…f         z   }|                     dd¦  «        }|                      |¦  «        }|                      |¦  «        }t/          ¦   «         pt1          | ¦  «        }| j        D ]a}|r||fz   }t          j        g ¦  «        }| j        o|| j        j        k     }|r|r ||||¬¦  «        }|d         }|rd}|r||d         fz   }Œb|r||fz   }|                      |¦  «        }|j        d         |	k     r2t
          j                             |ddd|	|j        d         z
  f¦  «        }|st?          d„ |||fD ¦   «         ¦  «        S tA          |||¬¦  «        S )Nrý   rH   r   rU   r¢   r   )r   r   g      ð?©Údtype.rõ   ©NNc              3   ó   K  — | ]}|®|V — Œ	d S r5   rý   )r‰   Úvs     r2   ú	<genexpr>z%SEWEncoder.forward.<locals>.<genexpr>ó  s(   è è € ÐmÐm˜qÐ_`Ð_l˜Ð_lÐ_lÐ_lÐ_lÐmÐmr3   ©Úlast_hidden_stater7   Ú
attentions)!Ú	unsqueezeÚrepeatrÇ   r   r/   r   r±   Ú
max_pool1drÕ   rb   ÚsqueezeÚlongÚtor  r¯   ÚfinfoÚminrI   rþ   r   r{   rE   r©   r	   r
   r  Úrandrž   Ú	layerdropr  ÚpadrØ   r   )r.   r7   r§   rÄ   Úoutput_hidden_statesÚreturn_dictÚall_hidden_statesÚall_self_attentionsÚexpand_attention_maskÚn_input_timestepsÚposition_embeddingsÚpooled_hidden_statesÚ
min_lengthÚsynced_gpusrŒ   Údropout_probabilityÚskip_the_layerÚlayer_outputss                     r2   r8   zSEWEncoder.forward¡  s»  € ð #7Ð@˜B˜B¸DÐØ$5Ð?˜b˜b¸4ÐàÑ%Ø$2×$<Ò$<¸RÑ$@Ô$@×$GÒ$GÈÈ1ÈmÔNaÐbcÔNdÑ$eÔ$eÐ!Ý+¨D¬KÑ8Ô8ð Wà8;�Ð4Ð4Ñ5à4BÐ4NÐSTÐXfÐSfÐSf  Ðmq��ð 9<�Ð4Ð4Ñ5õ
 ”M×,Ò,Ø&×,Ò,Ñ.Ô.×8Ò8¸Ñ;Ô;Ø$(¤KÔ$>Ø#œ{Ô9ð -ñ ô ÷
 ’W˜Q‘Z”Zß’T‘V”Vð ð "% ~°a°a°a¸¸tÀQÀQÀQÐ6FÔ'G×'JÒ'JÐQ^ÔQdÐ'JÑ'eÔ'eÑ!e�Ø!/µ%´+¸mÔ>QÑ2RÔ2RÔ2VÑ!V�à)Ô/°Ô2Ðà%×/Ò/°°1Ñ5Ô5ˆØ"×1Ò1°-Ñ@Ô@ÐØ#Ÿyšy¨Ñ7Ô7ÐÝÐ,×1Ò1°"Ñ5Ô5Ð7K×7PÒ7PÐQSÑ7TÔ7TÑUÔUˆ
Ø,¨S°+°:°+Ð-=Ô>ÐATÐUXÐZeÐ[eÐZeÐUeÔAfÑfˆØ%×/Ò/°°1Ñ5Ô5ˆàŸš¨Ñ6Ô6ˆØŸš ]Ñ3Ô3ˆå0Ñ2Ô2ÐRÕ6LÈTÑ6RÔ6Rˆà”[ð 	Pð 	PˆEØ#ð IØ$5¸Ð8HÑ$HÐ!õ #(¤*¨R¡.¤.Ðà!œ]ÐZÐ/BÀTÄ[ÔEZÒ/ZˆNØ!ð 1 [ð 1à % Ø!°.ÐTeð!ñ !ô !�ð !.¨aÔ 0�àð -Ø ,�à ð PØ&9¸]È1Ô=MÐ<OÑ&OÐ#øàð 	EØ 1°]Ð4DÑ DÐàŸš mÑ4Ô4ˆØÔ˜qÔ!Ð$5Ò5Ð5ÝœM×-Ò-¨m¸aÀÀAÐGXÐ[hÔ[nÐopÔ[qÑGqÐ=rÑsÔsˆMàð 	nÝÐmÐm ]Ð4EÐGZÐ$[ÐmÑmÔmÑmÔmÐmÝØ+Ø+Ø*ð
ñ 
ô 
ð 	
r3   )NFFTr:   r?   s   @r2   rú   rú   •  sb   ø€ € € € € ð	,ð 	,ð 	,ð 	,ð 	,ð ØØ"ØðW
ð W
ð W
ð W
ð W
ð W
ð W
ð W
r3   rú   c                   ó¦   ‡ — e Zd ZU eed<   dZdZdZdZdZ	dZ
dZ ej        ¦   «         ˆ fd„¦   «         Zdej        ez  fd	„Zd
edej        fd„Zˆ xZS )ÚSEWPreTrainedModelr/   ÚsewrŸ   ÚaudioTFc           
      ó’  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rwt	          j        |j        j        ddt          j	        d|j        j
        d         |j        j        z  z  ¦  «        z  ¬¦  «         t	          j        |j        j        d¦  «         dS t          |t          j        ¦  «        rþt!          ¦   «         rÕddl}t%          |d¦  «        rlt%          |d¦  «        r\|j                             |j        |j        gd¬¦  «        5  t	          j        |j        ¦  «         ddd¦  «         dS # 1 swxY w Y   dS |j                             |j        d¬¦  «        5  t	          j        |j        ¦  «         ddd¦  «         dS # 1 swxY w Y   dS t	          j        |j        ¦  «         dS dS )	zInitialize the weightsr   rU   r   )ÚmeanÚstdNrk   rj   rY   )r!   Ú_init_weightsrá   rS   ÚinitÚnormal_r*   r[   ÚmathÚsqrtr   Úin_channelsÚ	constant_r    r   r&   r	   re   rd   rf   rg   rk   rj   Úkaiming_normal_)r.   r£   re   r1   s      €r2   r-  z SEWPreTrainedModel._init_weights  sG  ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ8Ñ9Ô9ð 	4ÝŒLØ”Ô"ØØ�œ	 ! v¤{Ô'>¸qÔ'AÀFÄKÔD[Ñ'[Ñ"\Ñ]Ô]Ñ]ðñ ô ð õ
 ŒN˜6œ;Ô+¨QÑ/Ô/Ð/Ð/Ð/Ý˜¥¤	Ñ*Ô*ð 	4Ý)Ñ+Ô+ð 
4Ø Ð Ð Ð å˜6 :Ñ.Ô.ð <µ7¸6À:Ñ3NÔ3Nð <Ø"œ×:Ò:¸F¼OÈVÌ_Ð;]ÐmnÐ:ÑoÔoð <ð <ÝÔ,¨V¬]Ñ;Ô;Ð;ð<ð <ð <ñ <ô <ð <ð <ð <ð <ð <ð <ð <øøøð <ð <ð <ð <ð <ð <ð #œ×:Ò:¸6¼=ÐXYÐ:ÑZÔZð <ð <ÝÔ,¨V¬]Ñ;Ô;Ð;ð<ð <ð <ñ <ô <ð <ð <ð <ð <ð <ð <ð <øøøð <ð <ð <ð <ð <ð <õ Ô$ V¤]Ñ3Ô3Ð3Ð3Ð3ð	4ð 	4s$   Ä#E
Å
EÅEÅ8FÆF#Æ&F#Úinput_lengthsc                 óz   — d„ }t          | j        j        | j        j        ¦  «        D ]\  }} ||||¦  «        }Œ|S )zH
        Computes the output length of the convolutional layers
        c                 ó<   — t          j        | |z
  |d¬¦  «        dz   S )NÚfloor)Úrounding_moder   )r¯   Údiv)Úinput_lengthr   r   s      r2   Ú_conv_out_lengthzMSEWPreTrainedModel._get_feat_extract_output_lengths.<locals>._conv_out_length#  s&   € õ ”9˜\¨KÑ7¸ÈwÐWÑWÔWÐZ[Ñ[Ð[r3   )Úzipr/   r'   r(   )r.   r5  r<  r   r   s        r2   Ú _get_feat_extract_output_lengthsz3SEWPreTrainedModel._get_feat_extract_output_lengths  s\   € ð
	\ð 	\ð 	\õ
 $' t¤{Ô'>ÀÄÔ@WÑ#XÔ#Xð 	Qð 	QÑˆK˜Ø,Ð,¨]¸KÈÑPÔPˆMˆMàÐr3   Úfeature_vector_lengthr§   c                 ó  — |                       |                     d¦  «        ¦  «                             t          j        ¦  «        }t          j        ||j        ¬¦  «        }|                     d¦  «        |                     d¦  «        k     S )NrH   )Údevicer   r   )r>  Úsumr  r¯   r  ÚarangerA  r  )r.   r?  r§   Úoutput_lengthsÚattention_idss        r2   Ú"_get_feature_vector_attention_maskz5SEWPreTrainedModel._get_feature_vector_attention_mask-  sw   € Ø×>Ò>¸~×?QÒ?QÐRTÑ?UÔ?UÑVÔV×YÒYÕZ_ÔZdÑeÔeˆõ œÐ%:À>ÔCXÐYÑYÔYˆØ×&Ò& qÑ)Ô)¨N×,DÒ,DÀQÑ,GÔ,GÒGÐGr3   )r;   r<   r=   r   Ú__annotations__Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnr¯   Úno_gradr-  Ú
LongTensorrÔ   r>  rF  r>   r?   s   @r2   r'  r'  û  sÔ   ø€ € € € € € àÐÐÑØÐØ$€OØÐØ&*Ð#ØÐØ€NØÐà€U„]�_„_ð4ð 4ð 4ð 4ñ „_ð4ð.¸eÔ>NÐQTÑ>Tð ð ð ð ðHÈð HÐ]bÔ]mð Hð Hð Hð Hð Hð Hð Hð Hr3   r'  rÇ   Ú	mask_probÚmask_lengthÚ	min_masksrÅ   c                 ó@  ‡‡‡‡‡— | \  }Š‰dk     rt          d¦  «        ‚‰‰k    rt          d‰› d‰› d�¦  «        ‚t          j                             d¦  «                             ¦   «         Šˆˆˆˆˆfd„}|�9|                     ¦   «                              d¦  «                             ¦   «         nˆfd	„t          |¦  «        D ¦   «         }t          j	        |‰ft          ¬
¦  «        }g }	 |‰¦  «        }
|
dk    r|S |D ]·} ||¦  «        }t          j                             t          j        |‰dz
  z
  ¦  «        |d¬¦  «        }t          |¦  «        dk    r‰dz
  }n|d         }t          j        |t          j        |
|z
  t          j        ¬
¦  «        |z  g¦  «        }|	                     |¦  «         Œ¸t          j        |	¦  «        }	t          j        |	dd…dd…df         ||
‰f¦  «        }	|	                     ||
‰z  ¦  «        }	t          j        ‰¦  «        dddd…f         }t          j        |||
‰f¦  «                             ||
‰z  ¦  «        }|	|z   }	|	                     ¦   «         ‰dz
  k    r‰dz
  |	|	‰dz
  k    <   t          j        ||	dd¦  «         |S )an  
    Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for
    ASR](https://huggingface.co/papers/1904.08779). Note that this method is not optimized to run on TPU and should be run on
    CPU as part of the preprocessing during training.

    Args:
        shape: The shape for which to compute masks. This should be of a tuple of size 2 where
               the first element is the batch size and the second element is the length of the axis to span.
        mask_prob:  The percentage of the whole axis (between 0 and 1) which will be masked. The number of
                    independently generated mask spans of length `mask_length` is computed by
                    `mask_prob*shape[1]/mask_length`. Note that due to overlaps, `mask_prob` is an upper bound and the
                    actual percentage will be smaller.
        mask_length: size of the mask
        min_masks: minimum number of masked spans
        attention_mask: A (right-padded) attention mask which independently shortens the feature axis of
                        each batch dimension.
    r   z&`mask_length` has to be bigger than 0.zO`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: z and `sequence_length`: ú`c                 ó¸   •— t          ‰| z  ‰z  ‰z   ¦  «        }t          |‰¦  «        }|‰z  ‰k    r‰‰z  }| ‰dz
  z
  |k     rt          | ‰dz
  z
  d¦  «        }|S )z;Given input length, compute how many spans should be maskedr   r   )rÔ   Úmax)r;  Únum_masked_spanÚepsilonrR  rQ  rS  Úsequence_lengths     €€€€€r2   Úcompute_num_masked_spanz6_compute_mask_indices.<locals>.compute_num_masked_span\  s~   ø€ å˜i¨,Ñ6¸ÑDÀwÑNÑOÔOˆÝ˜o¨yÑ9Ô9ˆð ˜[Ñ(¨?Ò:Ð:Ø-°Ñ<ˆOð ˜;¨™?Ñ+¨oÒ=Ð=Ý! ,°+À±/Ñ"BÀAÑFÔFˆOàÐr3   NrH   c                 ó   •— g | ]}‰‘ŒS rý   rý   )r‰   r÷   rZ  s     €r2   r‹   z)_compute_mask_indices.<locals>.<listcomp>o  s   ø€ Ð9Ð9Ð9 !ˆoÐ9Ð9Ð9r3   r  r   F)Úreplace)r‘   ÚnpÚrandomr  ÚitemÚdetachrB  Útolistr�   ÚzerosrÖ   ÚchoicerC  ÚlenÚconcatenateÚonesÚint32ÚappendÚarrayÚbroadcast_tor|   rW  Úput_along_axis)rÇ   rQ  rR  r§   rS  Ú
batch_sizer[  r5  Úspec_aug_maskÚspec_aug_mask_idxsÚmax_num_masked_spanr;  rX  Úspec_aug_mask_idxÚdummy_mask_idxÚoffsetsrY  rZ  s    `` `           @@r2   Ú_compute_mask_indicesrt  6  sP  øøøøø€ ð0 #(Ñ€J�à�Q‚€ÝÐAÑBÔBÐBà�_Ò$Ð$Ýð:Ð^ið :ð :Ø'6ð:ð :ð :ñ
ô 
ð 	
õ Œi�nŠn˜QÑÔ×$Ò$Ñ&Ô&€Gðð ð ð ð ð ð ð ð ð$ Ð%ð 	×ÒÑÔ×#Ò# BÑ'Ô'×.Ò.Ñ0Ô0Ð0à9Ð9Ð9Ð9¥u¨ZÑ'8Ô'8Ð9Ñ9Ô9ð õ ”H˜j¨/Ð:Å$ÐGÑGÔG€MØÐà1Ð1°/ÑBÔBÐà˜aÒÐØÐà%ð 5ð 5ˆà1Ð1°,Ñ?Ô?ˆõ œI×,Ò,ÝŒI�l k°A¡oÑ6Ñ7Ô7¸ÐRWð -ñ 
ô 
Ðõ Ð Ñ!Ô! QÒ&Ð&ð -¨qÑ0ˆNˆNà.¨qÔ1ˆNåœNØ¥¤Ð(;¸oÑ(MÕUWÔU]Ð ^Ñ ^Ô ^ÐaoÑ oÐpñ
ô 
Ðð 	×!Ò!Ð"3Ñ4Ô4Ð4Ð4åœÐ"4Ñ5Ô5Ðõ œØ˜1˜1˜1˜a˜a˜a ˜:Ô&¨Ð5HÈ+Ð(Vñô Ðð ,×3Ò3°JÐ@SÐVaÑ@aÑbÔbÐõ Œi˜Ñ$Ô$ T¨4°°° ]Ô3€GÝŒo˜g¨
Ð4GÈÐ'UÑVÔV×^Ò^ØÐ'¨+Ñ5ñô €Gð ,¨gÑ5Ðð ×ÒÑÔ /°AÑ"5Ò5Ð5ØGVÐYZÑGZÐÐ-°À!Ñ0CÒCÑDõ Ô�mÐ%7¸¸BÑ?Ô?Ð?àÐr3   c                   óð   ‡ — e Zd Zdefˆ fd„Z	 	 ddej        dej        dz  dej        dz  fd„Ze		 	 	 	 	 ddej
        dz  dej
        dz  dej        dz  d	edz  d
edz  dedz  deez  fd„¦   «         Zˆ xZS )ÚSEWModelr/   c                 óª  •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        t          j        |j        d         |j        ¬¦  «        | _	        |j        d         |j
        k    | _        | j        r*t          j        |j        d         |j
        ¦  «        | _        t          j        |j        ¦  «        | _        |j        dk    s|j        dk    rBt          j        t)          j        |j
        ¦  «                             ¦   «         ¦  «        | _        t1          |¦  «        | _        |                      ¦   «          d S )NrH   rì   r¢   )r!   r"   r/   rƒ   Úfeature_extractorr   rD   r#   rñ   rE   r_   Úproject_featuresrw   Úfeature_projectionrÜ   Úfeat_proj_dropoutÚfeature_dropoutÚmask_time_probÚmask_feature_probÚ	Parameterr¯   r×   Úuniform_Úmasked_spec_embedrú   ÚencoderÚ	post_initry   s     €r2   r"   zSEWModel.__init__¯  s  ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ!2°6Ñ!:Ô!:ˆÔÝœ, v¤°rÔ':ÀÔ@UÐVÑVÔVˆŒà &¤°Ô 3°vÔ7IÒ IˆÔØÔ ð 	YÝ&(¤i°´ÀÔ0CÀVÔEWÑ&XÔ&XˆDÔ#Ý!œz¨&Ô*BÑCÔCˆÔàÔ  3Ò&Ð&¨&Ô*BÀSÒ*HÐ*HÝ%'¤\µ%´,¸vÔ?QÑ2RÔ2R×2[Ò2[Ñ2]Ô2]Ñ%^Ô%^ˆDÔ"å! &Ñ)Ô)ˆŒð 	�ŠÑÔÐÐÐr3   Nr7   Úmask_time_indicesr§   c                 ó  — t          | j        dd¦  «        s|S |                     ¦   «         \  }}}|�#| j                             |j        ¦  «        ||<   n“| j        j        dk    rƒ| j        r|t          ||f| j        j        | j        j	        || j        j
        ¬¦  «        }t          j        ||j        t          j        ¬¦  «        }| j                             |j        ¦  «        ||<   | j        j        dk    r†| j        rt          ||f| j        j        | j        j        | j        j        ¬¦  «        }t          j        ||j        t          j        ¬¦  «        }|dd…df                              d|d¦  «        }d||<   |S )	z¢
        Masks extracted features along time axis and/or along feature axis according to
        [SpecAugment](https://huggingface.co/papers/1904.08779).
        Úapply_spec_augmentTNr   )rQ  rR  r§   rS  )rA  r  )rQ  rR  rS  rH   )Úgetattrr/   r{   r�  r  r  r}  rž   rt  Úmask_time_lengthÚmask_time_min_masksr¯   ÚtensorrA  rÖ   r~  Úmask_feature_lengthÚmask_feature_min_masksÚexpand)r.   r7   r„  r§   rm  rZ  r_   Úmask_feature_indicess           r2   Ú_mask_hidden_stateszSEWModel._mask_hidden_statesÃ  s—  € õ �t”{Ð$8¸$Ñ?Ô?ð 	!Ø Ð ð 4A×3EÒ3EÑ3GÔ3GÑ0ˆ
�O [àÐ(à/3Ô/E×/HÒ/HÈÔI\Ñ/]Ô/]ˆMÐ+Ñ,Ð,ØŒ[Ô'¨!Ò+Ð+°´Ð+Ý 5Ø˜_Ð-Øœ+Ô4Ø œKÔ8Ø-Øœ+Ô9ð!ñ !ô !Ðõ !&¤Ð->À}ÔG[ÕchÔcmÐ nÑ nÔ nÐØ/3Ô/E×/HÒ/HÈÔI\Ñ/]Ô/]ˆMÐ+Ñ,àŒ;Ô(¨1Ò,Ð,°´Ð,å#8Ø˜[Ð)Øœ+Ô7Ø œKÔ;Øœ+Ô<ð	$ñ $ô $Ð õ $)¤<Ð0DÈ]ÔMaÕinÔisÐ#tÑ#tÔ#tÐ Ø#7¸¸¸¸4¸Ô#@×#GÒ#GÈÈOÐ]_Ñ#`Ô#`Ð Ø23ˆMÐ.Ñ/àÐr3   rŸ   rÄ   r  r  rÅ   c                 óT  — |�|n| j         j        }|�|n| j         j        }|�|n| j         j        }|                      |¦  «        }|                     dd¦  «        }|                      |¦  «        }| j        r|                      |¦  «        }|  	                    |¦  «        }	|�!|  
                    |	j        d         |¦  «        }|                      |	|¬¦  «        }	|                      |	||||¬¦  «        }
|
d         }	|s|	f|
dd…         z   S t          |	|
j        |
j        ¬¦  «        S )a/  
        mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict
            masked extracted features in *config.proj_codevector_dim* space.
        Nr   rU   )r„  ©r§   rÄ   r  r  r   r  )r/   rÄ   r  r  rx  rI   rE   ry  rz  r|  rF  rÇ   r�  r‚  r   r7   r  )r.   rŸ   r§   r„  rÄ   r  r  rª   Úextract_featuresr7   Úencoder_outputss              r2   r8   zSEWModel.forwardñ  sr  € ð  2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆà×1Ò1°,Ñ?Ô?ÐØ+×5Ò5°a¸Ñ;Ô;ÐØŸ?š?Ð+;Ñ<Ô<ÐàÔ ð 	IØ#×6Ò6Ð7GÑHÔHÐØ×,Ò,Ð-=Ñ>Ô>ˆàÐ%à!×DÒDÀ]ÔEXÐYZÔE[Ð]kÑlÔlˆNà×0Ò0°ÐRcÐ0ÑdÔdˆàŸ,š,ØØ)Ø/Ø!5Ø#ð 'ñ 
ô 
ˆð (¨Ô*ˆàð 	:Ø!Ð# o°a°b°bÔ&9Ñ9Ð9åØ+Ø)Ô7Ø&Ô1ð
ñ 
ô 
ð 	
r3   r  ©NNNNN)r;   r<   r=   r   r"   r¯   ÚFloatTensorrP  r�  r   r×   rÖ   rØ   r   r8   r>   r?   s   @r2   rv  rv  ­  s6  ø€ € € € € ð˜yð ð ð ð ð ð ð. 7;Ø26ð	,ð ,àÔ(ð,ð !Ô,¨tÑ3ð,ð Ô(¨4Ñ/ð	,ð ,ð ,ð ,ð\ ð /3Ø6:Ø)-Ø,0Ø#'ð4
ð 4
à”l TÑ)ð4
ð œ tÑ+ð4
ð !Ô,¨tÑ3ð	4
ð
   $™;ð4
ð # T™kð4
ð ˜D‘[ð4
ð 
�Ñ	 ð4
ð 4
ð 4
ñ „^ð4
ð 4
ð 4
ð 4
ð 4
r3   rv  zk
    SEW Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).
    )Úcustom_introc                   óÆ   ‡ — e Zd Zddedz  fˆ fd„Zd„ Zd„ Zd„ Ze	 	 	 	 	 dde	j
        dz  de	j
        dz  d	edz  d
edz  dedz  de	j
        dz  deez  fd„¦   «         Zˆ xZS )Ú	SEWForCTCNÚtarget_langc                 óª  •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        ¦  «        | _        || _        |j	        €t          d| j        › d�¦  «        ‚t          |d¦  «        r|j        r|j        n|j        }t	          j        ||j	        ¦  «        | _        |                      ¦   «          dS )a-  
        target_lang (`str`, *optional*):
            Language id of adapter weights. Adapter weights are stored in the format adapter.<lang>.safetensors or
            adapter.<lang>.bin. Only relevant when using an instance of [`SEWForCTC`] with adapters. Uses 'eng' by
            default.
        NzYou are trying to instantiate zò with a configuration that does not define the vocabulary size of the language model head. Please instantiate the model as follows: `SEWForCTC.from_pretrained(..., vocab_size=vocab_size)`. or define `vocab_size` of your model's configuration.Úadd_adapter)r!   r"   rv  r(  r   rÜ   Úfinal_dropoutr©   r™  Ú
vocab_sizer‘   r1   rd   r›  Úoutput_hidden_sizer_   rw   Úlm_headrƒ  )r.   r/   r™  rž  r1   s       €r2   r"   zSEWForCTC.__init__2  sÝ   ø€ õ 	‰Œ×Ò˜Ñ Ô Ð å˜FÑ#Ô#ˆŒÝ”z &Ô"6Ñ7Ô7ˆŒà&ˆÔàÔÐ$ÝðH°´ð Hð Hð Hñô ð õ *1°¸Ñ)GÔ)GÐvÈFÔL^ÐvˆFÔ%Ð%ÐdjÔdvð 	õ ”yÐ!3°VÔ5FÑGÔGˆŒð 	�ŠÑÔÐÐÐr3   c                 óT  — t          ¦   «         t          j        d¦  «        k    rdS | j        }|�)t	          | j        dd¦  «        €t          d|› d�¦  «        ‚|€2t	          | j        dd¦  «        �t                               d¦  «         dS |�|  	                    |d¬¦  «         dS dS )	a'  
        This method overwrites [`~PreTrainedModel.tie_weights`] so that adapter weights can be correctly loaded when
        passing `target_lang=...` to `from_pretrained(...)`.

        This method is **not** supposed to be called by the user and is prone to be changed in the future.
        ÚmetaNÚadapter_attn_dimzCannot pass `target_lang`: z- if `config.adapter_attn_dim` is not defined.z)By default `target_lang` is set to 'eng'.T)Ú
force_load)
r   r¯   rA  r™  r‡  r/   r‘   ÚloggerÚinfoÚload_adapter)r.   rª   r™  s      r2   Útie_weightszSEWForCTC.tie_weightsO  sÃ   € õ 6Ñ7Ô7½5¼<ÈÑ;OÔ;OÒOÐOØˆFð Ô&ˆàÐ"¥w¨t¬{Ð<NÐPTÑ'UÔ'UÐ']ÝÐu¸;ÐuÐuÐuÑvÔvÐvØÐ ¥W¨T¬[Ð:LÈdÑ%SÔ%SÐ%_Ý�KŠKÐCÑDÔDÐDÐDÐDØÐ$Ø×Ò˜k°dÐÑ;Ô;Ð;Ð;Ð;ð %Ð$r3   c                 óB   — | j         j                             ¦   «          dS ©z¨
        Calling this function will disable the gradient computation for the feature encoder so that its parameter will
        not be updated during training.
        N©r(  rx  rœ   ©r.   s    r2   Úfreeze_feature_encoderz SEWForCTC.freeze_feature_encoderg  ó!   € ð
 	ŒÔ"×5Ò5Ñ7Ô7Ð7Ð7Ð7r3   c                 óL   — | j                              ¦   «         D ]	}d|_        Œ
dS ©zÒ
        Calling this function will disable the gradient computation for the base model so that its parameters will not
        be updated during training. Only the classification head will be updated.
        FN©r(  r˜   r™   rš   s     r2   Úfreeze_base_modelzSEWForCTC.freeze_base_modeln  ó6   € ð
 ”X×(Ò(Ñ*Ô*ð 	(ð 	(ˆEØ"'ˆEÔÐð	(ð 	(r3   rŸ   r§   rÄ   r  r  ÚlabelsrÅ   c           
      óp  — |�|n| j         j        }|�>|                     ¦   «         | j         j        k    rt	          d| j         j        › �¦  «        ‚|                      |||||¬¦  «        }|d         }	|                      |	¦  «        }	|                      |	¦  «        }
d}|��Z|�|nt          j	        |t          j
        ¬¦  «        }|                      |                     d¦  «        ¦  «                             t          j
        ¦  «        }|dk    }|                     d¦  «        }|                     |¦  «        }t          j                             |
dt          j        ¬¦  «                             dd¦  «        }t          j        j                             d	¬
¦  «        5  t          j                             ||||| j         j        | j         j        | j         j        ¬¦  «        }ddd¦  «         n# 1 swxY w Y   |s|
f|t6          d…         z   }|�|f|z   n|S t9          ||
|j        |j        ¬¦  «        S )aà  
        labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*):
            Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to
            the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`.
            All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ...,
            config.vocab_size - 1]`.
        Nz$Label values must be <= vocab_size: r‘  r   r  rH   )r]   r  r   F)Úenabled)ÚblankÚ	reductionÚzero_infinity©ÚlossÚlogitsr7   r  )r/   r  rW  r�  r‘   r(  r©   rŸ  r¯   Ú	ones_liker  r>  rB  r  Úmasked_selectr   r±   Úlog_softmaxÚfloat32rI   ÚbackendsÚcudnnÚflagsÚctc_lossÚpad_token_idÚctc_loss_reductionÚctc_zero_infinityÚ_HIDDEN_STATES_START_POSITIONr   r7   r  )r.   rŸ   r§   rÄ   r  r  r³  rª   rø   r7   r»  rº  r5  Úlabels_maskÚtarget_lengthsÚflattened_targetsÚ	log_probsÚoutputs                     r2   r8   zSEWForCTC.forwardv  s€  € ð$ &1Ð%<�k�kÀ$Ä+ÔBYˆàÐ &§*¢*¡,¤,°$´+Ô2HÒ"HÐ"HÝÐ\ÀDÄKÔDZÐ\Ð\Ñ]Ô]Ð]à—(’(ØØ)Ø/Ø!5Ø#ð ñ 
ô 
ˆð   œ
ˆØŸš ]Ñ3Ô3ˆà—’˜mÑ,Ô,ˆàˆØÑð #1Ð"<��Å%Ä/ÐR^ÕfkÔfpÐBqÑBqÔBqð ð !×AÒAÀ.×BTÒBTÐUWÑBXÔBXÑYÔY×\Ò\Õ]bÔ]gÑhÔhˆMð ! Aš+ˆKØ(Ÿ_š_¨RÑ0Ô0ˆNØ &× 4Ò 4°[Ñ AÔ AÐõ œ×1Ò1°&¸bÍÌÐ1ÑVÔV×`Ò`ÐabÐdeÑfÔfˆIå”Ô%×+Ò+°EÐ+Ñ:Ô:ð 	ð 	Ý”}×-Ò-ØØ%Ø!Ø"Øœ+Ô2Ø"œkÔ<Ø"&¤+Ô"?ð .ñ ô �ð	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	ð ð 	FØ�Y Õ)FÐ)GÐ)GÔ!HÑHˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEåØ˜f°GÔ4IÐV]ÔVhð
ñ 
ô 
ð 	
s   Æ AG1Ç1G5Ç8G5r5   r”  )r;   r<   r=   rã   r"   r§  r¬  r±  r   r¯   r×   rÖ   rØ   r   r8   r>   r?   s   @r2   r˜  r˜  ,  s*  ø€ € € € € ðð ¨C°$©Jð ð ð ð ð ð ð:<ð <ð <ð08ð 8ð 8ð(ð (ð (ð ð /3Ø)-Ø,0Ø#'Ø&*ðE
ð E
à”l TÑ)ðE
ð œ tÑ+ðE
ð   $™;ð	E
ð
 # T™kðE
ð ˜D‘[ðE
ð ”˜tÑ#ðE
ð 
�Ñ	ðE
ð E
ð E
ñ „^ðE
ð E
ð E
ð E
ð E
r3   r˜  z’
    SEW Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like
    SUPERB Keyword Spotting.
    c                   ó²   ‡ — e Zd Zˆ fd„Zd„ Zd„ Ze	 	 	 	 	 ddej        dz  dej        dz  de	dz  de	dz  d	e	dz  d
ej        dz  de
ez  fd„¦   «         Zˆ xZS )ÚSEWForSequenceClassificationc                 óô  •— t          ¦   «                              |¦  «         t          |d¦  «        r|j        rt	          d¦  «        ‚t          |¦  «        | _        |j        dz   }|j        r.t          j
        t          j        |¦  «        |z  ¦  «        | _        t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¦  «        | _        |                      ¦   «          d S )Nr›  zZSequence classification does not support the use of SEW adapters (config.add_adapter=True)r   )r!   r"   rd   r›  r‘   rv  r(  r  Úuse_weighted_layer_sumr   r  r¯   rg  Úlayer_weightsrw   r_   Úclassifier_proj_sizeÚ	projectorÚ
num_labelsÚ
classifierrƒ  )r.   r/   Ú
num_layersr1   s      €r2   r"   z%SEWForSequenceClassification.__init__Æ  sá   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å�6˜=Ñ)Ô)ð 	¨fÔ.@ð 	ÝØlñô ð õ ˜FÑ#Ô#ˆŒØÔ-°Ñ1ˆ
ØÔ(ð 	SÝ!#¤­e¬j¸Ñ.DÔ.DÀzÑ.QÑ!RÔ!RˆDÔÝœ 6Ô#5°vÔ7RÑSÔSˆŒÝœ) FÔ$?ÀÔARÑSÔSˆŒð 	�ŠÑÔÐÐÐr3   c                 óB   — | j         j                             ¦   «          dS r©  rª  r«  s    r2   r¬  z3SEWForSequenceClassification.freeze_feature_encoder×  r­  r3   c                 óL   — | j                              ¦   «         D ]	}d|_        Œ
dS r¯  r°  rš   s     r2   r±  z.SEWForSequenceClassification.freeze_base_modelÞ  r²  r3   NrŸ   r§   rÄ   r  r  r³  rÅ   c                 ód  — |�|n| j         j        }| j         j        rdn|}|                      |||||¬¦  «        }| j         j        rx|t                   }	t          j        |	d¬¦  «        }	t          j         	                    | j
        d¬¦  «        }
|	|
                     ddd¦  «        z                       d¬¦  «        }	n|d         }	|                      |	¦  «        }	|€|	                     d¬¦  «        }n�|                      |	j        d         |¦  «        }|                     d¦  «                             dd|	j        d         ¦  «        }d	|	| <   |	                     d¬¦  «        |                     d¬¦  «                             dd¦  «        z  }|                      |¦  «        }d}|�Kt)          ¦   «         } ||                     d| j         j        ¦  «        |                     d¦  «        ¦  «        }|s|f|t          d…         z   }|�|f|z   n|S t-          |||j        |j        ¬
¦  «        S )a  
        input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
            Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file
            into an array of type `list[float]`, a `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library
            (`pip install torchcodec`) or the soundfile library (`pip install soundfile`).
            To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and conversion
            into a tensor of type `torch.FloatTensor`. See [`SEWProcessor.__call__`] for details.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        NTr‘  r   r­   rH   r   rU   r¢   r¹  )r/   r  rÐ  r(  rÇ  r¯   Ústackr   r±   r²   rÑ  rÈ   rB  rÓ  r+  rF  rÇ   r  r  rÕ  r   rÔ  r   r7   r  )r.   rŸ   r§   rÄ   r  r  r³  rª   rø   r7   Únorm_weightsÚpooled_outputÚpadding_maskÚexpand_padding_maskr»  rº  Úloss_fctrÌ  s                     r2   r8   z$SEWForSequenceClassification.forwardæ  sW  € ð0 &1Ð%<�k�kÀ$Ä+ÔBYˆØ'+¤{Ô'IÐc˜t˜tÐOcÐà—(’(ØØ)Ø/Ø!5Ø#ð ñ 
ô 
ˆð Œ;Ô-ð 	'Ø#Õ$AÔBˆMÝ!œK¨¸1Ð=Ñ=Ô=ˆMÝœ=×0Ò0°Ô1CÈÐ0ÑLÔLˆLØ*¨\×->Ò->¸rÀ1ÀaÑ-HÔ-HÑH×MÒMÐRSÐMÑTÔTˆMˆMà# AœJˆMàŸš }Ñ5Ô5ˆØÐ!Ø)×.Ò.°1Ð.Ñ5Ô5ˆMˆMà×BÒBÀ=ÔCVÐWXÔCYÐ[iÑjÔjˆLØ".×"8Ò"8¸Ñ"<Ô"<×"CÒ"CÀAÀqÈ-ÔJ]Ð^_ÔJ`Ñ"aÔ"aÐØ25ˆMÐ.Ð.Ñ/Ø)×-Ò-°!Ð-Ñ4Ô4°|×7GÒ7GÈAÐ7GÑ7NÔ7N×7SÒ7SÐTVÐXYÑ7ZÔ7ZÑZˆMà—’ Ñ/Ô/ˆàˆØÐÝ'Ñ)Ô)ˆHØ�8˜FŸKšK¨¨D¬KÔ,BÑCÔCÀVÇ[Â[ÐQSÁ_Ä_ÑUÔUˆDàð 	FØ�Y Õ)FÐ)GÐ)GÔ!HÑHˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå'ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r3   r”  )r;   r<   r=   r"   r¬  r±  r   r¯   r×   rÖ   rØ   r   r8   r>   r?   s   @r2   rÎ  rÎ  ¿  s  ø€ € € € € ðð ð ð ð ð"8ð 8ð 8ð(ð (ð (ð ð /3Ø)-Ø,0Ø#'Ø&*ðC
ð C
à”l TÑ)ðC
ð œ tÑ+ðC
ð   $™;ð	C
ð
 # T™kðC
ð ˜D‘[ðC
ð ”˜tÑ#ðC
ð 
Ð)Ñ	)ðC
ð C
ð C
ñ „^ðC
ð C
ð C
ð C
ð C
r3   rÎ  )r˜  rÎ  rv  r'  )Nr¢   rs   )Er0  Úcollections.abcr   Únumpyr^  r¯   r   Útorch.nnr   Ú r   r.  Úactivationsr   Úintegrations.deepspeedr	   Úintegrations.fsdpr
   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_utilsr   r   r   Úprocessing_utilsr   rc   r   r   r   Úutils.genericr   Úconfiguration_sewr   Ú
get_loggerr;   r¤  r   rA   rK   ÚModulerS   rm   ru   rƒ   r×   rÕ   r¶   r¸   rÚ   rê   rú   r'  rØ   rÔ   rP  Úndarrayrt  rv  rÇ  r˜  rÎ  Ú__all__rý   r3   r2   ú<module>rò     s¨  ðð* €€€Ø $Ð $Ð $Ð $Ð $Ð $à Ð Ð Ð Ø €€€Ø Ð Ð Ð Ð Ð Ø %Ð %Ð %Ð %Ð %Ð %à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø @Ð @Ð @Ð @Ð @Ð @Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø BÐ BÐ BÐ BÐ BÐ BØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YØ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rØ &Ð &Ð &Ð &Ð &Ð &Ø @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø (Ð (Ð (Ð (Ð (Ð (ð 
ˆÔ	˜HÑ	%Ô	%€ðð ð ð ð Ð8ñ ô ð ð*ð ð ð ð Ð6ñ ô ð ð6ð ð ð ð Ð6ñ ô ð ð0(ð (ð (ð (ð ( ¤ñ (ô (ð (ðVð ð ð ð �b”iñ ô ð ðð ð ð ð �B”Iñ ô ð ð,#ð #ð #ð #ð #˜œ	ñ #ô #ð #ðX !Øð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð �T‰\ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð8R/ð R/ð R/ð R/ð R/�2”9ñ R/ô R/ð R/ðjð ð ð ð �R”Yñ ô ð ð0!ð !ð !ð !ð !Ð0ñ !ô !ð !ðHc
ð c
ð c
ð c
ð c
�”ñ c
ô c
ð c
ðL ð7Hð 7Hð 7Hð 7Hð 7H˜ñ 7Hô 7Hñ „ð7Hð| /3Øðtð tØ��c�Œ?ðtàðtð ðtð Ô$ tÑ+ð	tð
 ðtð „Zðtð tð tð tðn ðx
ð x
ð x
ð x
ð x
Ð!ñ x
ô x
ñ „ðx
ðv !"Ð ð €ððñ ô ð
K
ð K
ð K
ð K
ð K
Ð"ñ K
ô K
ñô ð
K
ð\ €ððñ ô ðe
ð e
ð e
ð e
ð e
Ð#5ñ e
ô e
ñô ðe
ðP ZÐ
YÐ
Y€€€r3   