tj\ddlZddlmZddlZddlmZddlmZmZm Z m Z m Z ddl m Z ddlmZddlmZmZdd lmZdd lmZdd lmZmZmZdd lmZmZdd lmZddlm Z m!Z!m"Z"m#Z#ddl$m%Z%ddl&m'Z'ddl(m)Z)m*Z*m+Z+m,Z,ddl-m.Z.m/Z/ddl0m1Z1m2Z2m3Z3m4Z4m5Z5ddl6m7Z7e#j8e9Z:Gdde7eZ;GddedZe!deGd"d#e.Z?Gd$d%ej@ZAGd&d'e*ZBGd(d)e)ZCGd*d+e2ZDGd,d-e1ZEGd.d/e5ZFGd0d1e3ZGe!d23Gd4d5eFZHGd6d7e4ZIgd8ZJdS)9N)Callable)strict) Tokenizerdecoders normalizerspre_tokenizers processors)Unigram)nn) AudioInputmake_list_of_audio)create_bidirectional_mask)ALL_ATTENTION_FUNCTIONS)ProcessingKwargsProcessorMixinUnpack)PreTokenizedInput TextInput)TokenizersBackend)TransformersKwargsauto_docstringcan_return_tuplelogging)merge_with_config_defaults)capture_outputs)LlamaAttentionLlamaRotaryEmbeddingapply_rotary_pos_embeager_attention_forward)ParakeetCTCConfigParakeetEncoderConfig)ParakeetEncoderBlock ParakeetEncoderConvolutionModuleParakeetEncoderModelOutputParakeetForCTCParakeetPreTrainedModel) T5Tokenizerc beZdZ ddZ dd eeezd ed edzd ed ef dZdS) LasrTokenizerNdc ||_|ld|D} t| dkr|dt|Dz }nK|dkr)|t| krtd|d|dndt|D} | }|||_not |d ft |d ft |d fd g|_t|dz d d D]"} |jd | d d f#tt|jdd|_ |tj ||j _ tjtjtjdddg|j _t%jddd|j _t)jd|||||d| t-jddggdd|jfg|j _dS)Nc4g|]}dt|v|S) z*LasrTokenizer.__init__..As,[[[!LTWXYTZTZDZDZADZDZDZcg|]}d|d Sr2>r4is r6r7z*LasrTokenizer.__init__..Cs$-Z-Z-ZA.?1.?.?.?-Z-Z-Zr8rzBoth extra_ids (z!) and additional_special_tokens (zm) are provided to LasrTokenizer. In this case the additional_special_tokens must include the extra_ids tokenscg|]}d|d Sr;r=r>s r6r7z*LasrTokenizer.__init__..Ks$HHH!----HHHr8)▁gr2r<r F)unk_id byte_fallbackrBalwaysT) replacementprepend_schemesplit) eos_token unk_token pad_token extra_idsadditional_special_tokens$Ar,)rOr,z$Br,)singlepairspecial_tokensr=) _extra_idslenrange ValueError _vocab_scoresr3appendrr _tokenizerr Precompiled normalizerrSequenceWhitespaceSplit Metaspace pre_tokenizerrdecoderr__init__r TemplateProcessing eos_token_idpost_processor) selfrJrKrL_spm_precompiled_charsmaprMrNvocab vocab_filekwargs extra_tokensr?s r6razLasrTokenizer.__init__1s$ % 0[['@[[<  1$$)-Z-ZyIYIY-Z-Z-ZZ))Q9L0A0A#A#A ySl IHuY7G7GHHHL(4 %  !&D  Y%Y%Y% "D  9q="b11 D D"))+<+<+<+---*+* * * &&&r8FT token_idsskip_special_tokensclean_up_tokenization_spaces group_tokensreturnc t|tr|g}|rdtj|D}fd|D}t jf|||d|S)Ncg|] }|d S)rr=)r4 token_groups r6r7z)LasrTokenizer._decode..sXXXKQXXXr8c*g|]}|jk |Sr=) pad_token_id)r4tokenres r6r7z)LasrTokenizer._decode..s&PPPuUd>O5O5OU5O5O5Or8)rkrlrm) isinstanceint itertoolsgroupbyr_decode)rerkrlrmrnris` r6rzzLasrTokenizer._decode~s i % % $" I  YXX9;LY;W;WXXXIQPPP PPP  (   3)E       r8)r,r-r.Nr/NNN)FNT) __name__ __module__ __qualname__rarwlistboolr3rzr=r8r6r+r+0s"&"&K K K K `%*48!   c? " '+Tk           r8r+c.eZdZdddddddddd id Zd S) LasrProcessorKwargsi>longestT) sampling_ratepaddingreturn_attention_maskrightF)r padding_sideadd_special_tokensreturn_tensorspt) audio_kwargs text_kwargs common_kwargsN)r{r|r} _defaultsr=r8r6rrsN# %)  #"'  +D1  IIIr8rF)totalc eZdZfdZe d dedeezeezeezdzde dzde e fdZ e dZxZS) LasrProcessorcLt||dSN)superra)refeature_extractor tokenizer __class__s r6razLasrProcessor.__init__s$ *I66666r8Naudiotextrric t|}|jtfd|jji|}|+t d|dddn4||ddkr"td|d|ddd ||j|fi|d}||j|fi|d }||S|d |d <|S) a sampling_rate (`int`, *optional*): The sampling rate of the input audio in Hz. This should match the sampling rate expected by the feature extractor (defaults to 16000 Hz). If provided, it will be validated against the processor's expected sampling rate, and an error will be raised if they don't match. If not provided, a warning will be issued and the default sampling rate will be assumed. tokenizer_init_kwargsNzUYou've provided audio without specifying the sampling rate. It will be assumed to be rrz$, which can result in silent errors.z The sampling rate of the audio (z5) does not match the sampling rate of the processor (zD). Please provide resampled the audio to the expected sampling rate.r input_idslabels) r _merge_kwargsrr init_kwargslogger warning_oncerVr)rerrrri output_kwargsinputs encodingss r6__call__zLasrProcessor.__call__s{#5))**   "&."<      }huwEiFGViW}}}    mN;OL L Ll=llxEFTxUVexflll   +T+ESS]>5RSSF  &tLL}]/KLLI <M(5F8 Mr8c&|jj}|dgzS)Nr)rmodel_input_names)refeature_extractor_input_namess r6rzLasrProcessor.model_input_namess(,(>(P%,z99r8NN)r{r|r}rarr rrr~rwrrrpropertyr __classcell__rs@r6rrs77777bf$( (((++d9o=EV@WWZ^^(Tz ( ,- (((^(T::X:::::r8rz google/medasr) checkpointcJeZdZUdZdZeed<dZeed<dZeed<dZ e ed <dZ e ed <d Z eed <d Z eed<dZeed<dZeed<dZeed<dZeeeedfzed<dZeeeedfzed<dZeed<dZedzed<eZeZdS)LasrEncoderConfiga convolution_bias (`bool`, *optional*, defaults to `False`): Whether to use bias in convolutions of the conformer's convolution module. conv_kernel_size (`int`, *optional*, defaults to 32): The kernel size of the convolution layers in the Conformer block. subsampling_conv_channels (`int`, *optional*, defaults to 256): The number of channels in the subsampling convolution layers. subsampling_conv_kernel_size (`int`, *optional*, defaults to 5): The kernel size of the subsampling convolution layers. subsampling_conv_stride (`int`, *optional*, defaults to 2): The stride of the subsampling convolution layers. dropout_positions (`float`, *optional*, defaults to 0.0): The dropout ratio for the positions in the input sequence. feed_forward_residual_weights (`tuple[float, float]`, *optional*, defaults to `[1.5, 0.5]`): The residual weights for the feed forward layers. conv_residual_weights (`tuple[float, float]`, *optional*, defaults to `[2.0, 1.0]`): The residual weights for the convolution layers. batch_norm_momentum (`float`, *optional*, defaults to 0.01): The momentum for the batch normalization layers Example: ```python >>> from transformers import LasrEncoderModel, LasrEncoderConfig >>> # Initializing a `LasrEncoder` configuration >>> configuration = LasrEncoderConfig() >>> # Initializing a model from the configuration >>> model = LasrEncoderModel(configuration) >>> # Accessing the model configuration >>> configuration = model.config ``` This configuration class is based on the LasrEncoder architecture from Google Health AI. You can find more details and pre-trained models at [google/medasr](https://huggingface.co/google/medasr).  hidden_sizenum_hidden_layersiintermediate_sizeFattention_biasconvolution_bias conv_kernel_sizesubsampling_conv_kernel_size num_mel_binsi'max_position_embeddingsgư>layer_norm_eps)g?g?.feed_forward_residual_weights)g@g?conv_residual_weightsg{Gz?batch_norm_momentumNrope_parameters)r{r|r}__doc__rrw__annotations__rrrrrrrrrrfloatrr~tuplerrrdictAttributeErrorsubsampling_factor scale_inputr=r8r6rrsI$$LKs!s!!! ND   "d"""c() #)))L##(S((( NE   EO!4;ucz1B#BOOO=G4;ucz)::GGG!%%%%#'OTD['''')) .""KKKr8rcFeZdZUdZdZeed<dZeed<edZ dS) LasrCTCConfigaE ctc_loss_reduction (`str`, *optional*, defaults to `"mean"`): Specifies the reduction to apply to the output of `torch.nn.CTCLoss`. Only relevant when training an instance of [`LasrForCTC`]. ctc_zero_infinity (`bool`, *optional*, defaults to `True`): Whether to zero infinite losses and the associated gradients of `torch.nn.CTCLoss`. Infinite losses mainly occur when the inputs are too short to be aligned to the targets. Only relevant when training an instance of [`LasrForCTC`]. Example: ```python >>> from transformers import LasrForCTC, LasrCTCConfig >>> # Initializing a Lasr configuration >>> configuration = LasrCTCConfig() >>> # Initializing a model from the configuration >>> model = LasrForCTC(configuration) >>> # Accessing the model configuration >>> configuration = model.config ``` This configuration class is based on the Lasr CTC architecture from Google Health AI. You can find more details and pre-trained models at [google/medasr](https://huggingface.co/google/medasr). r vocab_sizerrtc |jjdzS)Nr)encoder_configsubsampling_conv_stride)res r6inputs_to_logits_ratioz$LasrCTCConfig.inputs_to_logits_ratio6s":A==r8N) r{r|r}rrrwrrtrrr=r8r6rrs[.JL# >>X>>>r8rcHeZdZdeffd ZdejdejfdZxZS)LasrEncoderSubsamplingconfigcttj|j|j|_tj|j|j|j|j |_ tj|j|j |j|j |_ tj|j |j|_ tj|_dS)N) kernel_sizestride)rrar Linearrrdense_0Conv1drrconv_0subsampling_conv_channelsconv_1dense_1ReLUact_fnrerrs r6razLasrEncoderSubsampling.__init__<s y!4f6HII i    ;1     i    ,;1     y!A6CUVV gii r8input_featuresroct|||}|dd}|||}|||}|dd}||S)Nr9r)rr transposerrr)rer hidden_statess r6forwardzLasrEncoderSubsampling.forwardNs DLL$@$@AA %//155  DKK $>$>??  DKK $>$>?? %//155 ||M***r8) r{r|r}rratorchTensorrrrs@r6rr;sk 0      $+el+u|++++++++r8rceZdZdS)LasrEncoderRotaryEmbeddingNr{r|r}r=r8r6rrWsr8rc eZdZdedeffd Z d dejdeejejfdzdejdzde e d eejejff d Z xZ S) LasrEncoderAttentionr layer_idxcZt||d|_dS)NF)rra is_causalrerrrs r6razLasrEncoderAttention.__init__[s( +++r8Nrposition_embeddingsattention_maskriroc |jdd}g|d|jR}|||dd}|||dd}|||dd} |\} } t||| | \}}tj |j j t} | |||| |f|j sdn|j|jd|\} }| jg|dR} || } | |fS)NrCr9rrA)dropoutscaling)shapehead_dimq_projviewrk_projv_projr r get_interfacer_attn_implementationr!trainingattention_dropoutrreshape contiguouso_proj)rerrrri input_shape hidden_shape query_states key_states value_statescossinattention_interface attn_output attn_weightss r6rzLasrEncoderAttention.forward_s$)#2#. 88b8$-88 {{=1166|DDNNqRSTT [[//44\BBLLQPQRR {{=1166|DDNNqRSTT &S#7 jRUWZ#[#[ j(?(M K ,.E) ) %8$7      % $}HCC$2HL % %  % % ! \*k);;;;;;FFHH kk+.. L((r8r) r{r|r}rrwrarrrrrrrrs@r6rrZs0SIM.2 ")")|")#5<#=>E") t+ ") +, ") u|U\) * ")")")")")")")")r8rc&eZdZddeffd ZxZS)LasrEncoderConvolutionModuleNrct||d|_tj|j|j|_dS)Nsame)momentum)rrarr BatchNorm1drrnorm)rer module_configrs r6raz%LasrEncoderConvolutionModule.__init__sD /// N6#5@Z[[[ r8r)r{r|r}rrarrs@r6rrsV\\0\\\\\\\\\\r8rc eZdZdedeffd Z d dejdejdzdejdzdee d ejf d Z xZ S) LasrEncoderBlockrrct|||j|_|j|_t j|j|jd|_t j|j|jd|_ t j|j|jd|_ t j|j|jd|_ t j|j|jd|_ dS)NF)bias) rrarrr LayerNormrrnorm_feed_forward1 norm_self_att norm_convnorm_feed_forward2norm_outrs r6razLasrEncoderBlock.__init__s +++-3-Q*%+%A""$,v/A6CX_d"e"e"e\&*SZ_```f&8&:OV[\\\"$,v/A6CX_d"e"e"e V%79NUZ[[[ r8Nrrrriroc N|}|||}|jd|z|jd|zz}||}|jd|||d|\}}||z}||||} |jd|z|jd| zz}|}|| |}|jd|z|jd|zz}| |}|S)Nrr9)rrr)rr=) feed_forward1rrr self_attnconvrr feed_forward2rr) rerrrriresidualnormalized_hidden_statesr _ conv_outputs r6rzLasrEncoderBlock.forwardsX!**4+B+B=+Q+QRR  .q 1H .s$ b b bY fi 0 0 b b br8)epsr)rragradient_checkpointingrdropout_positions layerdropr subsamplerr rotary_embr ModuleListrUrlayersrrrout_norm post_initrs `r6razLasrEncoder.__init__s    &+#~ !'!9)0884V<<m b b b b%H`BaBa b b b    V%7V=RY^___  r8Nrroutput_attention_maskriroc V||}||tj|jd|jd\}}tj ||j |j }tj ||j |j }tj ||j |j }d}|$| ||jd}|}t|j||}|jD]<} d} |j r!tjg} | |jkrd } | s| |f|||fd |}=||}t'||r||nd S) a; output_attention_mask (`bool`, *optional*): Whether to return the output attention mask. Example: ```python >>> from transformers import AutoProcessor, LasrEncoder >>> from datasets import load_dataset, Audio >>> model_id = "google/medasr" >>> processor = AutoProcessor.from_pretrained(model_id) >>> encoder = ParakeetEncoder.from_pretrained(model_id) >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") >>> ds = ds.cast_column("audio", Audio(sampling_rate=processor.feature_extractor.sampling_rate)) >>> inputs = processor(ds[0]["audio"]["array"]) >>> encoder_outputs = encoder(**inputs) >>> print(encoder_outputs.last_hidden_state.shape) ``` r9)devicer)prN) target_length)r inputs_embedsrFT)rr)last_hidden_stater)r?r@rarangerrG unsqueezer functionalrrr=_get_output_attention_maskrrrBrandr>rCr4rw) rerrrErirr r  output_mask encoder_layerto_dropdropout_probabilitys r6rzLasrEncoder.forwardsF77 ?? 5< (;A(>}G[\\\ffghii  S --mt|VZVc-dd m##C4+ADM#ZZm##C4+ADM#ZZ  %99.XeXklmXn9ooK(N2;')    "[  MG} #&+jnn#&77"G  - !!#1),c !! !!  m44 %+0Ek+Ja;??,,,gk    r8r)r{r|r}rrbase_model_prefixrarrrrrrrrrr4rrrs@r6r7r7s !0"/3-1 H H  H  t+H  $d{ H +, H H H H _ ^H H H H H r8r7ceZdZfdZxZS) LasrForCTCc 6tjdi|S)a  Example: ```python >>> from transformers import AutoProcessor, LasrForCTC >>> from datasets import load_dataset, Audio >>> model_id = "google/medasr" >>> processor = AutoProcessor.from_pretrained(model_id) >>> model = LasrForCTC.from_pretrained(model_id) >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") >>> ds = ds.cast_column("audio", Audio(sampling_rate=processor.feature_extractor.sampling_rate)) >>> inputs = processor(ds[0]["audio"]["array"], text=ds[0]["text"]) >>> predicted_ids = model.generate(**inputs) >>> transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True) >>> print(transcription) ``` r=)rgenerate) super_kwargsrs r6rYzLasrForCTC.generate=s", uww//,///r8)r{r|r}rYrrs@r6rWrW<s8000000000r8rW)rWr7r*rrrr+)Krxcollections.abcrrhuggingface_hub.dataclassesr tokenizersrrrrr tokenizers.modelsr r audio_utilsr r masking_utilsrmodeling_utilsrprocessing_utilsrrrtokenization_utils_baserrtokenization_utils_tokenizersrutilsrrrr utils.genericrutils.output_capturingrllama.modeling_llamarrr r!parakeet.configuration_parakeetr"r#parakeet.modeling_parakeetr$r%r&r'r(t5.tokenization_t5r) get_loggerr{rr+rrrrModulerrrrrr*r4r7rW__all__r=r8r6ros$$$$$$ ......SSSSSSSSSSSSSS%%%%%%99999999666666555555HHHHHHHHHHCCCCCCCC>>>>>>RRRRRRRRRRRR777777555555vvvvvvvvvvvvVVVVVVVV-,,,,,  H % %d d d d d K!2d d d N     *%     2:2:2:2:2:N2:2:2:j?+++7#7#7#7#7#-7#7#,+7#t?+++>>>>>%>>,+>@+++++RY+++8<;;;;!5;;;')')')')')>')')')T\\\\\#C\\\.....+...b1&     7    a a a a a %a a  a H000000004   r8