tj{1T UddlZddlZddlZddlZddlZddlZddlZddlZddlZddl m Z ddlm Z ddl m Z mZddlmZddlmZmZddlmZmZddlmZdd lmZdd lmZmZmZmZmZdd l m!Z!ddl"Z"dd l#m$Z$m%Z%dd l&m'Z'ddl(m)Z)ddl*m+Z,ddl*m-Z.ddl"m/Z/m0Z0ddl1m2Z2ddl3m4Z4ddl5m6Z7ddl8m9Z9ddl:m;Z;ddlZ>m?Z?m@Z@ddlAmBZBddlCmDZDddlEmFZFmGZGddlHmIZImJZJmKZKmLZLmMZMddlNmOZOmPZPmQZQmRZRmSZSmTZTmUZUddlVmWZWddlXmYZYdd lZm[Z[dd!l\m]Z]dd"l^m_Z_dd#l`maZadd$lbmcZcmdZdmeZedd%lfmgZgdd&lhmiZidd'ljmkZkdd(llmmZmdd)lnmoZompZpmqZqmrZrmsZsmtZtmuZudd*lvmwZwdd+lxmyZymzZzm{Z{m|Z|dd,l}m~Z~dd-lmZmZdd.lmZdd/lmZdd0lmZdd1lmZdd2lmZdd3lmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZdd4lmZmZmZdd5lmZmZmZmZdd6lmZmZmZmZmZmZmZdd7lmZmZdd8lmZmZdd9lmZer dd:lmZdd;lmZer ddlmZe'jeʦe'jd?kZnd@ZejeΦZejdAdBZejdCdBZedDdEFZd@ad@aedGHGdIdJZdKefdLZdKefdMZdNZedOZedPZeddQe"jdRedzfdSZdTZdUZe"je"je"je"je"je"je"je"je"je"je"je"je"je"je"jdVZdWdXdKefdYZ dd[eejzd\ee"jzd]ed^edzdKeee"j/ff d_Zd`e"j/dKefdaZdbe0jdKeefdcZddeeedeeee"j/fdKeeeeeeffdfZddeeedeeee"j/fdKeeeeeeeffdgZdeeee"j/fdhdEdKeee"j/ffdiZdhdEdjed`e"j/fdkZddledmedzdKefdnZ ddoeejzdzdmedzdpedzdqedzdredzdsedtedzduedzdvedzdKeeedzedzffdwZ ddQee"jzezdzdxeedzdye9dzedzdeedzd]ed{edzdKee9e"jffd|ZGd}d~ZGddZGddEe0jeeeeIZ ee j e _ e j j 3e j j  dhdde j _ eddhe dedKe fdZ eddhe0jdedKe0jfdZ ddhe0jdedKe0jfdZ deeze"jzdKefdZ ddhe ded{edzfdZdhe ded{edzfdZGddeZeZeed<Gdde ZdS)N)abstractmethod) defaultdict)CallableIterator)contextmanager) dataclassfield)partialwraps)cycle)Thread) TYPE_CHECKINGAnyTypeVarget_type_hintsoverload) is_zipfile)is_offline_mode"split_torch_state_dict_into_shards)version) safe_open)load) save_file)Tensornn) constraints) checkpoint)initialization)PreTrainedConfig)get_model_conversion_mapping)WeightConverterWeightRenaming$convert_and_load_state_dict_in_modelrevert_weight_conversion)DistributedConfig)custom_object_save) CompileConfigGenerationConfig)PeftAdapterMixindeepspeed_config hub_kernelsis_deepspeed_zero3_enabledis_fsdp_enabled)_get_device_mapaccelerate_disk_offloadaccelerate_dispatchcheck_and_set_device_mapexpand_device_map get_deviceload_offloaded_parameter)!_load_state_dict_into_zero3_model)eager_paged_attention_forward)ALL_FP8_EXPERTS_FUNCTIONS)flash_attention_forward)paged_attention_forward)flex_attention_forward)allow_all_hub_kernels is_kernel kernelize)ALL_EXPERTS_FUNCTIONS)maybe_load_adapters)sdpa_attention_forward)sdpa_attention_paged_forward)ALL_PARALLEL_STYLES_get_parameter_tp_plandistribute_modelgather_state_dict_for_saveinitialize_tensor_parallelismshard_and_distribute_moduleverify_tp_plan) LOSS_MAPPING)$FLASH_ATTENTION_COMPATIBILITY_MATRIXFLASH_ATTN_KERNEL_FALLBACKlazy_import_flash_attention!lazy_import_paged_flash_attention)ROPE_INIT_FUNCTIONS) apply_patchespatch_output_recorders)id_tensor_storage) HfQuantizer)get_hf_quantizer)get_module_from_name)auto_conversion)ADAPTER_SAFE_WEIGHTS_NAME DUMMY_INPUTSSAFE_WEIGHTS_INDEX_NAMESAFE_WEIGHTS_NAMEWEIGHTS_INDEX_NAME WEIGHTS_NAMEContextManagers KernelConfigPushToHubMixin cached_filecheck_torch_load_is_safe copy_funchas_fileis_accelerate_availableis_bitsandbytes_availableis_env_variable_trueis_kernels_availableis_torch_flex_attn_availableis_torch_npu_availableis_torch_xpu_availablelogging)GeneralInterfaceis_flash_attention_requestedsplit_attention_implementation)DownloadKwargscreate_and_tag_model_cardget_checkpoint_shard_fileshf_api)KERNELS_MAX_VERSIONKERNELS_MIN_VERSIONis_flash_attn_greater_or_equal#is_huggingface_hub_greater_or_equalis_sagemaker_mp_enabledis_torch_cuda_available is_tracing)LoadStateDictInfolog_state_dict_report)_CAN_RECORD_REGISTRYOutputRecorder)QuantizationMethod)add_hook_to_module)extract_model_from_parallel)Mode)DeviceMeshLike) __version__z1.10F XLA_USE_BF160XLA_DOWNCAST_BF16SpecificPreTrainedModelTypePreTrainedModel)boundT)frozenceZdZUdZdZedzed<eeZ edzed<dZ e dzed<dZ e ed<dZ edzed <dZedzed <dZedzed <dZe ed <dZejdzed <eeZeed<dZedzed<dZded<dZe ed<dZeeezdzed<dZe dzed<ede fdZdS)LoadStateDictConfigze Config for loading weights. This allows bundling arguments that are just passed around. Npretrained_model_name_or_path)default_factorydownload_kwargsuse_safetensorsFignore_mismatched_sizessharded_metadata device_mapdisk_offload_folderoffload_buffersdtype dtype_plan hf_quantizerDeviceMeshLike | None device_meshT weights_onlyweight_mapping disable_mmapreturnc|jduSN)rselfs Y/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/modeling_utils.py is_quantizedz LoadStateDictConfig.is_quantizeds ,,) __name__ __module__ __qualname____doc__rstr__annotations__r rorrboolrrdictrrrrtorchrrrSrrrlistr"r#rpropertyrrrrrs 15!3:444-2U>-R-R-RO^d*RRR#'OTD['''$)T)))$(dTk((("Jt """&*t***!OT!!! $E5; $$$uT222J222'+L+$++++/K(///L$DHND>9:TAHHH $L$+$$$ -d---X---rrrcto7ttjdotjS)Nis_initialized)_torch_distributed_availablehasattrr distributedrrrr!_is_torch_distributed_initializedrs7$ / E%'7 8 8 /   , , . .rctrttjdsdStjS)Nget_world_sizer)rrrrrrrr!_get_torch_distributed_world_sizers? , . .ge>OQa6b6bq   + + - --rcto0ttjdddkS)N LOCAL_RANKz-1r)rintosenvirongetrrris_local_dist_rank_0rs2 , . . _3rz~~lTX7Y7Y3Z3Z^_3__rc#*Kda dVdadS#dawxYwNTF) _is_quantizedrrrset_quantized_staters5M   c#*Kda dVdadS#dawxYwr)_is_ds_init_calledrrrset_zero3_staters:# "U""""rrmodel_class_namec# K|js | |d|d}nd|d}t|tj} tj|dVtj|dS#tj|wxYw)z Locally change the torch default dtype to `dtype`, and restore the old one upon exiting the context. If `model_class_name` is provided, it's used to provide a more helpful error message if `dtype` is not valid. Nz% cannot be instantiated under `dtype=z$` as it's not a floating-point dtypez Cannot set `z7` as torch's default as it's not a floating-point dtype)is_floating_point ValueErrorrget_default_dtypeset_default_dtype)rr error_messageoriginal_dtypes rlocal_torch_dtypers  "(  '#uu%uuu Mj5iiiM''',..N0 &&&  /////////s A,,Bctjgj}tj}||kr|tjdkr|SdS|S)z Test if a device context manager is currently in use, or if it is not the case, check if the default device is not "cpu". This is used to infer the correct device to load the model on, in case `device_map` is not provided. cpuN)rtensordeviceget_default_device)device_in_contextdefault_devices r*get_torch_context_manager_or_global_devicersV  R((/-//NN** U\%00 0 0! !t rcf|D]K}|r5dt|jvrdt|jvr |jcSLt |dkr t jStt|jS)zt Returns the first found floating dtype in `state_dict` if there is one, otherwise returns the first dtype. float8_float4_r) valuesrrrlenrfloat32nextiter) state_dictts rget_state_dict_dtypers       Yc!'ll%B%ByX[\]\cXdXdGdGd7NNN :!} Z&&(()) * * 00r)BOOLU8I8I16U16F16BF16I32U32F32F64I64U64F8_E4M3F8_E5M2pathzstr | os.PathLikectjdsdS tjtj|}tdd5}tdd|DDdd }d d d n #1swxYwY|D]>\}}||ks+|| d d zr|d kcS?n#ttf$rYnwxYwdS)aTrue if `path` lives on an hf-mount FUSE filesystem (device string 'hf-mount'). hf-mount's mmap + readahead interaction deadlocks under parallel page-faults, so callers should load the file into memory instead. Linux-only; returns False on other platforms. linuxFz /proc/mountsutf-8encodingc3\K|]'}t|dk|d|dfV(dS)rrNr).0ps r z"_is_on_hf_mount..Es8NN!#a&&A++!A$!++++NNrc3>K|]}|VdSr)split)rls rrz"_is_on_hf_mount..Es*'>'>a '>'>'>'>'>'>rc,t|dS)Nrr)es rz!_is_on_hf_mount..Fsc!A$iirT)keyreverseN/zhf-mount) sysplatform startswithrrrealpathfspathopensortedrstripOSErrorr)rrealfhentriesdevmps r_is_on_hf_mountr8si < " "7 + +u  w $00 .7 3 3 3 rNN'>'>2'>'>'>NNN''G                 ) )GCrzzT__RYYs^^c-ABBzj(((( ) Z      5s=AC&%(B C&BC& B!AC&$C&&C:9C:rcheckpoint_file map_locationrrctj|}|t|}|drm|rwdkrqt |d5}t |}dddn #1swxYwYdkr fd|D}|St|d5}i}| D]}dkr| |} | } | tvrt| } ntd | tj| | d ||<||||<|cdddS#1swxYwY|rt'i} dkrt)|rd d i} tj|f|d | S)aW Reads a `safetensor` or a `.bin` checkpoint file. We load the checkpoint on "cpu" by default. When `disable_mmap` is True, safetensors files are read fully into memory instead of being memory-mapped. When `disable_mmap` is None (default), it is auto-detected to True on hf-mount FUSE filesystems (see `_is_on_hf_mount`). N .safetensorsmetarbrcBi|]\}}||Sr)to)rkvrs r z#load_state_dict..gs+SSS1al!3!3SSSrpt) frameworkz)Cannot load safetensors of unknown dtype )sizerrmmapTrr)rrrendswithr_safe_load_bytesreaditemsrkeys get_slice get_dtypestr_to_torch_dtyperrempty get_shape get_tensorr rarr) rrrrcheckpoint_path_fhrfr!_slicek_dtyper extra_argss ` rload_state_dictr:Qsi00O&77 //  LF22ot,, :-chhjj99  : : : : : : : : : : : : : : :u$$SSSS @P@P@R@RSSS   $ 7 7 7 1JVVXX E E6))[[^^F$..00G"444 27 ;()^U\)^)^___$)KV5E5E5G5Gu]c$d$d$dJqMM$%LLOO$6$6|$D$DJqMM                 # """Jv*_"="=d^ :o jL| j j_i j jjs%"BB BC FF#&F#rc|rC|dd|z}n|}|S)N)nelementviewdata_ptr element_size)rstops r_end_ptrrBs[ !{{2r"++--0C0C0E0EE   Krmodulecg}|D]K\}t|dipi}|fd|DL|S)N_tied_weights_keysc&g|] }rd|n|S.rrr!names r z)_get_tied_weight_keys..s, S S S!$!=D1A S S Sr) named_modulesgetattrextendr-)rCtied_weight_keys submoduletiedrJs @r_get_tied_weight_keysrRs{"$!//11UUiy"6;;Ar S S S Styy{{ S S STTTT rtensorsrcg}|D]}t|dkr||+g}|D]A}||}||t||fB||d\}}} || h|ddD]@\} } }| |kr||hn|d|| }Ag} g} |D]R}t|dkr(| |=| |S| | fS)Nrrrr<)rappendr?rBsortaddpop)rSrfiltered_tensorssharedareasrJr_ last_stop last_namestartrAdisjoint_tensorsshared_tensorss r_find_disjointrbs v;;??  # #F + + +  F FD%F LL&//++Xf-=-=tD E E E E "'(9i ,,,!&qrr   E4 !! ''//// $((...II   N#++ w<<1    # #GKKMM 2 2 2 2  ! !' * * * * + ++rcg}g}|D]}t|dkrtjt}|D]N}||}|j|t |f}|||Ot|dkr||||||fS)Nrr) r collectionsrsetrr?rBrWrU) rSrra identicalrZr[rJrareas r_find_identicalrhsN "I * * v;;?? ',, " "D%FM6??#4#4hv6F6FGD $KOOD ! ! ! ! u::??   V $ $ $ $  ! !& ) ) ) ) 9 $$rmodelc\tjt}|D]\}t |t js)|t|H|j j dkr>| }|t||t|d|D}tt|}g}t}|y|D]d}d} t!|D]Pt#fd|D} | r1|vr-| dz } | t%|kr|Qet)||\} } | D]||< t-| |\} } | D]\}||}|D]|=||}t%|dkr||]| r|| t%|dkrt5d|d|d |S) aX Remove all tied weights from the given `state_dict`, making sure to keep only the main weight that `model` will expect when reloading (even if we now tie weights symmetrically, it's better to keep the intended one). This is because `safetensors` does not allow tensor aliasing - so we're going to remove aliases before saving. rc@i|]\}}t|dk||S)rr)rptrnamess rr#z7remove_tied_weights_from_state_dict..s)OOO*#uE Q3rNrc3BK|]}tj|VdSrresearch)rpatrJs rrz6remove_tied_weights_from_state_dict..s/%f%fsbiT&:&:%f%f%f%f%f%frrz8The weights trying to be saved contained shared tensors z\ which are not properly defined. We found all the potential target tied weights keys to be: zo. This can also just mean that the module's tied weight keys are wrong vs the actual tied weights in the model.)rdrrr, isinstancerridrUrtypeget_parameter_or_bufferrRrerRrranyrrWrbclonerh intersection differencerN RuntimeError)rriptrsr shared_ptrsall_potential_tied_weights_keys error_namesto_delete_namesrmfoundmatches_pattern shared_namesdisjoint_namesidentical_namesinamesknownunknownrJs @r#remove_tied_weights_from_state_dictrs  "4 ( (D"((**99 f&%,// 9 F  # #D ) ) ) ) ] 6 ) )22488F F  # #D ) ) ) ) "6** + 2 24 8 8 8 8OO OOOK'**?*F*F&G&G#KeeO'2 '')) 2 2EEu  2 2"%%f%f%f%fFe%f%f%f"f"f"2tz'9'9QJEs5zz))'++D111  2$2+2D2D2F2F #S#S L.44%d+1133 4%4L*$M$M!L/!((##O44 ! !D4  ##O44 w<t#|dtt|dtt|dn#tt|}t)||fi|}|>|tt|kr%t)|tt |fi|}|d}|`|rtnt}|tt fvr?t-||fi|s1|r/t/t*|fddi|d n~|>t-|tfi|r+t#|dtt|d!|d"t#|dtt|dtt|dnI#t"$rt2$r2}t#d#|d$|d%tt|d|d }~wwxYw|r t4d&||}nat4d&|d'|n@|r>t j |r|}n| | | | | |||dd|d( }t)||fi|}d }|rt9||| | | | | |||||) \} }n||gnd } | |fS)*zGet all the checkpoint filenames based on `pretrained_model_name_or_path`, and optional metadata if the checkpoints are sharded. This function will download the data if necessary. cache_dirforce_downloadFproxieslocal_files_onlytokenrevisionmain subfolder commit_hashNrz.safetensors.index.jsonzadapter_model.binzThe transformers file in the config seems to be incorrect: it is neither a safetensors file (*.safetensors) nor a safetensors index file (*.safetensors.index.json): zM`transformers_weights` must reference a file inside the model directory, got TzError no file named z found in directory rHz, or z, found in directory )rrrrr)rrr _raise_exceptions_for_gated_repo%_raise_exceptions_for_missing_entries _commit_hashrDISABLE_SAFETENSORS_CONVERSIONz& does not appear to have a file named z or zX and thus cannot be loaded with `safetensors`. Please do not set `use_safetensors=True`.ignore_errors_during_conversionzThread-auto_conversion)targetargskwargsrJz) but there is a file without the variant z;. Use `variant=None` to load this model from those weights.zCan't load the model for 'z'. If you were trying to load it from 'https://huggingface.co/models', make sure you don't have a local directory with the same name. Otherwise, make sure 'z=' is the correct path to a directory containing a file named zloading weights file z from cache at ) rrrrrrrrrrr) rrrrrrrrrr)rorr)rrrrisdirjoinabspath commonpathisfilerrZrYr\r[rrrfr`rVrcr r_ Exceptionloggerinforq)!rrrrrrrrrrrrrrrrr is_shardedis_localbase_dir archive_fileabsolute_base_dirabsolute_archive_file containedfilenamehas_file_kwargscached_file_kwargscan_auto_convertresolved_archive_filesafe_weights_namerrcheckpoint_filess! r_get_resolved_checkpoint_filesr)s &9)9)9O##K00I$(()95AAN!!),,G&**+=uEE    ( (E"":..8&H##K44I!%%m44K%1-66~FF OmOvOv %P P  .1DDD 8588 J$0Y5F(+,I(J(J%7==!>?? z -97<<(EyQQ!w||H6TUU &(*(A(A%,.GOOL,I,I) " 2 24EG\3] ^ ^bs sII!&&& %III& $IiGII<DDE^__  --"'.. :I|TegnGoGopp33- "w||19lK\^e>f>f   !--"'.. :I|TkmtGuGuvv33- "w||19lKbdk>l>l   " $  :I|T`biGjGjkk**  "w||19la>a   %  :I|TfhoGpGpqq**  "w||19lK]_f>g>g   "  9<8I7+S+S995999 L<8I7+S+SLLZfgsu|Z}Z}LL+HLLLW^^BGLL4QRR S S@ 8LHH.99;DDE^__  --'(97CC' g>>%"&$4 O#1(&499> +( " "" " $%%%$,-MNNN$'&$O Y )44QS[(r(r_q(r(r%)0XN_ahAiAi5i5i,75$% >> ? ? ?$0 ! ! KK```I^`` a a a a p 7>>) $ $ p$- ! ! '"0"$4($&499> + " " %00My$o$o\n$o$o !j-G ) !)-!$! . . . ***7T6_122ei - --s,>A#G"" G10G1>G3]22^8-^33^8rconfigrrc|du}|t|tr;|dkrt|dr-|j&|j}td|dn|r d|vr |d}nc|t |}nQ|(|ddr tj }n't|dd| }t |}td |d n:tt|rtt|}ntd t|trtt|n|}nGt|ttjfstd |ntj}|||}t|trr|dtj}t|trtt|n|}td|dn|}||_|jD]} t|| x} || _||fS)aFind the correct `dtype` to use based on provided arguments. Also update the `config` based on the inferred dtype. We do the following: 1. If dtype is "auto", we try to read the config, else auto-detect dtype from the loaded state_dict, by checking its first weights entry that is of a floating type - we assume all floating dtype weights are of the same dtype 2. Else, use the dtype provided as a dict or str NautorzWill use dtype=z$ as defined in model's config objectrz.ggufrr(zTSince the `dtype` attribute can't be found in model's config object, will use dtype=z as derived from model's weightsze`dtype` provided as a `str` can only be `'auto'`, or a string representation of a valid `torch.dtype`z`dtype` can be one of: `torch.dtype`, `'auto'`, a string of a valid `torch.dtype` or a `dict` with valid `dtype` for each sub-config in composite configs, but received rzUsing different dtypes per module is deprecated and will be removed in future versions Setting different dtypes per backbone model might cause device errors downstream, therefore setting the dtype=z for all modules.)rsrrrrrrr)rrr:rMrrr update_dtyper warning_once sub_configs) rrrrrrrr main_dtypesub_config_key sub_configs r _get_dtyperBs"-J  eS ! !" 67++ 0H"LEKK ]% ] ] ]^^^^! Ag1A&A&A 0 9#/ 4Z @ @)5:J1:M:V:VW^:_:_5 % %4,Q/fS_&&& !5Z @ @KKR*/RRR&& u-- { .8s-C-CNGE5)))EEED%+#677 RJORR  '))))%00% YYr5#:#<#<== 3=j#3N3N^WUJ///T^  ?!+ ? ? ?     FL ,**!&.99 9J F)J  : rc eZdZdZedddejfdZedddejfdZdddedefdZ e d Z dddd ed e e d fdejd zdef dZddddedede fdZd S)ModuleUtilsMixinzH A few utilities for `torch.nn.Modules`, to be used as a mixin. rrrcXtd|DS)z `torch.device`: The device on which the module is (assuming that all the module parameters are on the same device). c3$K|] }|jV dSrrrparams rrz*ModuleUtilsMixin.device..s$@@UEL@@@@@@rr parametersrs rrzModuleUtilsMixin.devices+ @@doo.?.?@@@@@@rcXtd|DS)zw `torch.dtype`: The dtype of the module (assuming that all the module parameters have the same dtype). c3LK|]}||jV dSr)rrrs rrz)ModuleUtilsMixin.dtype..s5\\E%BYBYB[B[\EK\\\\\\rrrs rrzModuleUtilsMixin.dtypes+ \\T__->->\\\\\\rencoder_attention_maskc\td|dkr|dddddddf}|dkr|ddddddf}||j}d|z t j|jjz}|S)z Invert an attention mask (e.g., switches 0. and 1.). Args: encoder_attention_mask (`torch.Tensor`): An attention mask. Returns: `torch.Tensor`: The inverted attention mask. zDetected the usage of `invert_attention_mask`: This function is deprecated and will be removed in v5.12.0. Please use the new API in `transformers.masking_utils`Nrr?)rrdimr rrfinfomin)rrencoder_extended_attention_masks rinvert_attention_maskz&ModuleUtilsMixin.invert_attention_masks  E   " % % ' '1 , ,.DQQQaaaQRQRQR].S + ! % % ' '1 , ,.DQQQdTUTUTUEU.V ++J*L*LSWS]*L*^*^'+.1P+PTYT_`d`jTkTkTo*o'..rc2td|j}|\}}tj||}|ddddf||d|ddddfk}||j}|jd|jdkrP|jd|jdz }tj tj |||f||j|gd}|dddddddf|ddddddfz}|S)NzDetected the usage of `create_extended_attention_mask_for_decoder`: This function is deprecated and will be removed in v5.12.0. Please use the new API in `transformers.masking_utils`rrrrr<axis) rrrrarangerepeatr rshapecatones) input_shapeattention_maskr batch_size seq_lengthseq_ids causal_maskprefix_seq_lenextended_attention_masks r*create_extended_attention_mask_for_decoderz;ModuleUtilsMixin.create_extended_attention_mask_for_decoders\ E    &!, J,z&999dD!!!m,33J ANNRYZ^`a`a`acgZgRhh !nn^%9::  Q ."6q"9 9 9+1!4{7H7KKN)J JGPV^i^oppp K#.aaaqqq!!!m"<~aaaQUW[]^]^]^N^?_"_&&rNrr.rctd||j}|dkr|dddddddf}nv|dkrCt |jddrt ||}n,|ddddddf}ntd|d|j d| | }d |z tj |j z}|S) a Makes broadcastable attention and causal masks so that future and masked tokens are ignored. Arguments: attention_mask (`torch.Tensor`): Mask with ones indicating tokens to attend to, zeros for tokens to ignore. input_shape (`tuple[int]`): The shape of the input to the model. Returns: `torch.Tensor` The extended attention mask, with a the same dtype as `attention_mask.dtype`. zDetected the usage of `get_extended_attention_mask`: This function is deprecated and will be removed in v5.12.0. Please use the new API in `transformers.masking_utils`Nrr is_decoderz!Wrong shape for input_ids (shape z) or attention_mask (shape )rr)rrrrrMrrrrrr rrr)rrrrrs rget_extended_attention_maskz,ModuleUtilsMixin.get_extended_attention_masksD$  E   =JE     1 $ $&4QQQaaa]&C # #    ! !Q & &t{L$77 K*:*e*e++''+9D$9I*J''sKss\j\psss #:"<"<5"<"I"I#&)@#@EKPUDVDVDZ"Z&&rFonly_trainableexclude_embeddingsc|rd|D}t|dd}|rddl}d}|D]\}}|r||vr |js|s|r|t ||jjrbt|dr| } nt|dr |j j } nd} || d z| zz }|| z }|S) a Get number of (optionally, trainable or non-embeddings) parameters in the module. Args: only_trainable (`bool`, *optional*, defaults to `False`): Whether or not to return only the number of trainable parameters exclude_embeddings (`bool`, *optional*, defaults to `False`): Whether or not to return only the number of non-embeddings parameters Returns: `int`: The number of parameters. cRg|]$\}}t|tj|d%S)z.weight)rsr Embedding)rrJ module_types rrKz3ModuleUtilsMixin.num_parameters..%sJ%%%%6T;R\]hjljvRwRw%   %%%ris_loaded_in_4bitFrNr@ quant_storagerr) rLrM bitsandbytesnamed_parametersrrsr Params4bitrr@ritemsizenumel) rrrembedding_param_namesrbnb total_paramsrJr num_bytess rnum_parameterszModuleUtilsMixin.num_parameterssL  %%:>:L:L:N:N%%% !$D*=uEE  ' & & & & 0022 2 2KD%! d.C&C&C" 2. 2% 2E36;L)M)M 2un55&$)$6$6$8$8  88&$)$7$@ $%  EKKMMA$5 $AALL EKKMM1LrrFF)rrrrrrrrrr staticmethodrtuplerrrr rrrrrsfA&A5<AAAXA]%]%+]]]X] /$5/v/Z`////6''\'<%) 4'4'4'4'38_4'{T! 4'  4'4'4'4'l**.**bf*sv******rrcNeZdZdZdZdejfdZdejfdZdZ dZ d S) EmbeddingAccessMixinz Base utilities to regroup getters and setters for embeddings. Introduces the `input_layer_embed` attribute, which indicates where the input embeddings come from and where they should be set. embed_tokensrc@t|dd}t||dx}|St|dd}| t||rt||St|dd}| t||rt||St|dd}|(||ur$t|dr|St|dd}|(||ur$t|dr|Std |jjd ) z Returns the model's input embeddings. Returns: `nn.Module`: A torch module mapping vocabulary to hidden states. _input_embed_layerrN embeddingsrilanguage_modelget_input_embeddings base_modelu.`get_input_embeddings` not auto‑handled for "; please override in the subclass.)rMrrNotImplementedError __class__r)rrJdefault_embeddingrrirrs rrz)EmbeddingAccessMixin.get_input_embeddingsLsVt1>BB")tT!:!: :  G$ $T<66  !gj$&?&? !:t,, ,gt,,  !5!5 5$'' ' '7>>  &d**(>??+"6688 8T<66  !j&<&<UkAlAl&<2244 4! xT^=T x x x   rvaluectt|dd}t||rt|||dSt|ddx}#t||rt|||dSt|ddx}#t||rt|||dSt|ddx}+||ur't|dr||dSt|ddx}+||ur't|dr||dSt d |jjd ) aFallback setter that handles **~70%** of models in the code-base. Order of attempts: 1. `self.<_input_embed_layer>` (direct attribute) 2. `self.embeddings.<_input_embed_layer>` (nested embeddings for vision/audio models) 3. `self.model.<_input_embed_layer>` (encoder/decoder models) 4. delegate to the *base model* if one exists 5. otherwise raise `NotImplementedError` so subclasses still can (and should) override for exotic layouts. rrrNrirset_input_embeddingsru.`set_input_embeddings` not auto‑handled for r)rMrrr rrr)rrrJrrirrs rr z)EmbeddingAccessMixin.set_input_embeddingssst1>BB 4    D$ & & & & &#D,===j JwWacgOhOh J Je , , , , ,tWd333e @WUTXEYEY @ E4 ' ' ' ' ' 't-=tDD D^ Qd**(>??+  / / 6 6 6 6 6#4t<< e+d"e?j@fd4ZAe=d"efd5ZB dd7eCd8eDd9eDd:e$e$eDe fd;fdZEdd7eCd?ed"efd@ZFdd?ed"efdAZGd"efdBZHdd?ed"efdCZI ddDe dzd?edEed"e fdFZJdGe dzd"e fdHZKddIe dzd?ed"e fdJZLdKe dzd"e fdLZMe=d"efdMZNe=d"efdNZOddDe ezdEefdOZPd"ee e dzffdPZQdGe ezfdQZRdRZSdSZTddTe dzfdUZUddTe dzfdVZVdWZWdXZXe,jYdYZZddZefd[Z[e,jYe\j]d\Z^dd]ed"efd^Z_dd`ee dzdaefdbZ`dcZa dddeCdzdeeCdzdfed"e?jbfdgZcddhZd ddie?jbddeCdzdeeCdzdfed"e?jbf djZe ddke?jfddeCdzdledfed"e?jff dmZgdnZh ddlefdoZidpZjdqZkdreCfdsZld"e?jbe$e?jbzfdtZmduZnddvZod_epfdwedxeDfdyZqdzZre+d"efd{Zs dd}e etjuzd~ededzdedeCe zde dzde ezdzdedefdZvewexjyfdZyddZzewe,j?j@j{fdZ{ewe,j?j@j|fdZ|fdZ}fdZ~e=de,jdededEedzfdZde,jd"efdZddedzddfdZe=ddddddddd_ddd deede etjuzdzd&ee zetjuzdzde etjuzdzdededede ezdzde dedzdedee eee efzfdzdedzd"efdZe ddddedzdee dzdedee dzd"e$eeff dZededed"efdZddZe=ddZdZe+dZe+dZe+dZe+dZej9dZe+d"efdZej9ded"dfdZd"efdZdedzd"eDfdZe=dZd`ee dedzdddedzd"df dZded"dfdZded"dfdZd„Zde fdĄZ ddeded"ee$e e,j1ffdDŽZddeffdȄ ZdɄZe=d"efdʄZe=d"efd˄ZxZS)ra Base class for all models. [`PreTrainedModel`] takes care of storing the configuration of the models and handles methods for loading, downloading and saving models as well as a few methods common to all models to: - resize the input embeddings Class attributes (overridden by derived classes): - **config_class** ([`PreTrainedConfig`]) -- A subclass of [`PreTrainedConfig`] to use as configuration class for this model architecture. - **base_model_prefix** (`str`) -- A string indicating the attribute associated to the base model in derived classes of the same architecture adding modules on top of the base model. - **main_input_name** (`str`) -- The name of the principal input to the model (often `input_ids` for NLP models, `pixel_values` for vision models and `input_values` for speech models). - **can_record_outputs** (dict): N config_classgeneration_config_classrbase_model_prefixF _is_stateful model_tags input_idsmain_input_nametextinput_modalities_no_split_modules_skip_keys_device_placement_keep_in_fp32_modules_keep_in_fp32_modules_strictrE_keys_to_ignore_on_load_missing"_keys_to_ignore_on_load_unexpected_keys_to_ignore_on_save_supports_sdpa_supports_flash_attn_supports_flex_attn!_compatible_flash_implementations_tp_plan_pp_plan_ep_plan _fsdp_plansupports_gradient_checkpointing_can_compile_fullgraph_supports_attention_backend_can_record_outputsrc|jpiS)a Maps output names (e.g., "attentions", "hidden_states") to either: - A module class (e.g., `LlamaDecoderLayer`), using default index conventions: * index = 0 for a key that contains "hidden_states" (e.g. "hidden_states" or "vision_hidden_states") * index = 1 for any other key: "attentions", "cross_attentions", etc. - A class name as a string, when the class is not importable at declaration time. - An `OutputRecorder(...)` with `target_class`, optional `index`, and `layer_name`. - A list of any of the above, to record outputs from several module types under one key. Examples: These two are equivalent: ```python _can_record_outputs = { "attentions": LlamaAttention, "hidden_states": LlamaDecoderLayer } _can_record_outputs = { "attentions": OutputRecorder(LlamaAttention, index=1), "hidden_states": OutputRecorder(LlamaDecoderLayer, index=0) } ``` This means you can record outputs from the same class, by specifying a layer name. Before collecting outputs, we check that they come from this layer. If you have cross attention that come from `LlamaAttention` and self attention that also come from `LlamaAttention` but from `self_attn` you can do this: ```python class LlamaModel(PreTrainedModel): _can_record_outputs = { "attentions": OutputRecorder(LlamaAttention, index=1, layer_name="self_attn"), "cross_attentions": OutputRecorder(LlamaAttention, index=1, layer_name="cross_attn") } ``` )rDrs rcan_record_outputsz"PreTrainedModel.can_record_outputs sV'-2-rc8dtjtiS)z^ `dict[str, torch.Tensor]`: Dummy inputs to do a forward pass in the network. r.)rrrXrs r dummy_inputszPreTrainedModel.dummy_inputs6s U\,7788rc ltjdi|tj|dd}|jdd}t |dd}|j}| ||_dS| ||_dS| ||_dS| ||_dSdS)Nrr)r)super__init_subclass__inspectget_annotationsr__dict__rr))clsrchild_annotationchild_attributefull_annotationfull_attributers rrKz!PreTrainedModel.__init_subclass__=s!!++F+++ #2377;;HdKK,**>4@@)--11(DAA)  &.C     )/C     '-C     (.C   ) (rrctt|ts*t d|jjd|jjd||_|j|_d|_ | |jj dtj |j_||jj|j_|rJ |j||_n)#t,$r||_YnwxYw|jj}|t.vr[ddt.d}t3j||jj}t7|d kr |d }nd}||_|jt<t?|j<dS) NzParameter config in `zt(config)` should be an instance of class `PreTrainedConfig`. To create a model from a pretrained model use `model = z(.from_pretrained(PRETRAINED_MODEL_NAME)`T is_init_checkallow_all_kernels(|rr) rJ__init__rsr TypeErrorrrr name_or_path kernel_config%_check_and_adjust_attn_implementation_attn_implementationr,ALLOW_ALL_KERNELS_attn_implementation_internal(_check_and_adjust_experts_implementation_experts_implementation _experts_implementation_internal can_generater*from_model_configgeneration_configrrJrrpfindallr loss_typerDr|r)rrinputsrri loss_groupsrs rrZzPreTrainedModel.__init__Us &"233 ^(?^^ N3^^^   "/"594^4^ K ,); 5_5 5  18<7d7d K /8 8  4      H H)-)E)W)WX^)_)_&&& H H H)-)E)E)G)G&&& HN+ L ( (7chh|44777K ;0GHHI9~~!!%aL  "484LS00111s/D#D54D5c t|jpi|_t|jpi|_t|jpi|_t|jpi|_|j|ur|j|jjpi|j|jj pi|j|jj pi|j|jj pi| d|_ t|jpg|_t|jpg|_t|jpg|_t|jpg|_t|jpg|_t|jpg|_t|jpg|_|D]\ }t/|ddx}rJ|j fd|Dt/|ddx}rJ|j fd|Dt/|ddx}rJ|j fd |Dt/|d dx}rJ|j fd |Dt/|d dx}rJ|j  fd |Dt/|ddx}r|j|t/|ddx}r|j|t/|ddx}r|j|t/|ddx}r|j|t/|ddx}r|j|t/|ddx} r|j| t/|ddx} r&|j fd| D ||dS)a A method executed at the end of each Transformer model initialization, to execute code that needs the model's modules properly initialized (such as weight initialization). It is also used to obtain all correct static properties (parallelism plans, tied_weights_keys, _keep_in_fp32_modules, etc) correctly in the case of composite models (that is, the top level model should know about those properties from its children). F all_submodelsr?Nc&i|] \}}d||SrGrrr!r"rJs rr#z-PreTrainedModel.post_init..)%W%W%W41ammmmQ%W%W%Wrr=c&i|] \}}d||SrGrrps rr#z-PreTrainedModel.post_init..rqrr>c&i|] \}}d||SrGrrps rr#z-PreTrainedModel.post_init..rqrr@c&i|] \}}d||SrGrrps rr#z-PreTrainedModel.post_init..s)'Y'Y'YTQ4 ! q'Y'Y'Yrall_tied_weights_keysc0i|]\}}d|d|SrGrrps rr#z-PreTrainedModel.post_init..s62u2u2uTXTUWXd==Q==T--A--2u2u2urr4r5r2r3r7r6r8ch|] }d| SrGrrIs r z,PreTrainedModel.post_init..s#4X4X4Xq]]q]]4X4X4Xr)rr=r?r>r@rupdaterbase_model_pp_planbase_model_tp_planbase_model_ep_planbase_model_fsdp_planget_expanded_tied_weights_keysrurer4r5r2r3r7r6r8named_childrenrMcopyr, init_weights._backward_compatibility_gradient_checkpointing) rrCplan tied_keys keep_fp32keep_fp32_strictno_split skip_keysignore_unexpectedignore_missing ignore_saverJs @r post_initzPreTrainedModel.post_inits>T]0b11 T]0b11 T]0b11 t4"55 ?d " " M !?!E2 F F F M !?!E2 F F F M !?!E2 F F F O " "4;#C#Ir J J J%)%H%HW\%H%]%]"%()C)Ir%J%J",/0Q0WUW,X,X)!$T%;%Ar!B!B+.t/O/USU+V+V(25d6]6cac2d2d//243W3][]/^/^,'*4+G+M2'N'N$!//11 Z ZLD&vz4888t Y $$%W%W%W%W499;;CTCTCVCV%W%W%WXXXvz4888t Y $$%W%W%W%W499;;CTCTCVCV%W%W%WXXXvz4888t Y $$%W%W%W%W499;;CTCTCVCV%W%W%WXXXv|T:::t [&&'Y'Y'Y'YTYY[[EVEVEXEX'Y'Y'YZZZ#F,CTJJJy w*112u2u2u2u\e\j\j\l\l\r\r\t\t2u2u2uvvv#F,CTJJJy =*11)<<<#*63QSW#X#XX K1889IJJJ"6+>EEEx 8&--h777#F,I4PPPy C077 BBB%,F4XZ^$_$__  R7>>?PQQQ!(1RTX!Y!YY~ L4;;NKKK%f.GNNN{ Z,334X4X4X4XK4X4X4XYYY  ;;=====rct|jdr<|jjjr+|jst d|jjd|jS|jS)z: The full tp plan for the model's modules distributed_configzGExpert parallelism was requested (`enable_expert_parallel=True`), but `zs` does not define an expert-parallel plan. Add a `base_model_ep_plan` to its config, or disable expert parallelism.) rrrenable_expert_parallelr?rrrr=rs rtp_planzPreTrainedModel.tp_plansv 4; 4 5 5 !$+:X:o !=  Z/ZZZ = }rc|jSr)r@rs r fsdp_planzPreTrainedModel.fsdp_plans rc|jSr)r>rs rpp_planzPreTrainedModel.pp_plans }rrc .| i|_dSt|tstd|D]D\}}|t vr6td|d|dt t jEd|D}|D]R}| dd}d}|D]}tj ||rd }n|stj d |d S||_dS) Nz&Can only set a dictionary as `tp_plan`z#Unsupported tensor parallel style 'z ' for layer 'z'. Supported styles are cg|]\}}|Srr)rrJr\s rrKz+PreTrainedModel.tp_plan..sIIIgdATIIIr*z\d+FTzLayer pattern 'z' does not match any parameters in the model. This rule may not be applied during tensor parallelization, or may lead to dimension mismatches)r=rsrrr,rCrr-rreplacerpmatchwarningswarn)rr layer_patternparallel_stylemodel_param_names regex_patternpattern_matchedrs rrzPreTrainedModel.tp_plans <DM F$%% GEFF F.2ZZ\\   )M>%888 O.OOWdOO,01D1I1K1K,L,LOO9JI1F1F1H1HIII!YY[[  M)11#v>>M#O/   8M:66&*OE#  dmddd  rcr| i|_dSt|tstd||_dS)Nz&Can only set a dictionary as `pp_plan`)r>rsrr)rrs rrzPreTrainedModel.pp_plans@ <DM F$%% GEFF F rctt|dd}|td|||S)z Potentially dequantize the model in case it has been quantized by a quantization method that support dequantization. rNz?You need to first quantize your model in order to dequantize itr)rMr dequantize)rrrs rrzPreTrainedModel.dequantizesC t^T::  ^__ _&&t5&999rc|jrAt|jddr-|t |jddSdSdS)Ngradient_checkpointingF)rArMrgradient_checkpointing_enabledelattrrs rrz>PreTrainedModel._backward_compatibility_gradient_checkpointingsc  / ;GDKIach4i4i ;  . . 0 0 0 DK!9 : : : : : ; ; ; ;rtagsct|tr|g}|jg|_|D]%}||jvr|j|&dS)a\ Add custom tags into the model that gets pushed to the Hugging Face Hub. Will not overwrite existing tags in the model. Args: tags (`Union[list[str], str]`): The desired tags to inject in the model Examples: ```python from transformers import AutoModel model = AutoModel.from_pretrained("google-bert/bert-base-cased") model.add_model_tags(["custom", "custom-bert"]) # Push the model to your namespace with the name "my-custom-bert". model.push_to_hub("my-custom-bert") ``` N)rsrr-rU)rrtags radd_model_tagszPreTrainedModel.add_model_tags$si, dC  6D ? " DO , ,C$/))&&s+++ , ,rc |d|j}|ddx})td||jkr|n|}t |t rt t|}||_|jD]}t ||x}||_d|vr|d|_ d|vr|d|_ | dd}tg}|(| t||j|r!| t!t#ot$ ot& } | rxtd d dl} |t/j| jt7 t9gt;|5||fi|} t=| dddn #1swxYwY| r%d d lm } | | | !| S)z All context managers that the model should be initialized under go here. Args: dtype (`torch.dtype`, *optional*): Override the default `dtype` and load the model under this dtype. r torch_dtypeNz1`torch_dtype` is deprecated! Use `dtype` instead!attn_implementationexperts_implementationrWF@Detected DeepSpeed ZeRO-3: activating zero.init() for this modelrconfig_dict_or_pathr)initialize_weights_zero3)"rXrrrrsrrMrrr_rcrrPrUrrr<r-rrr deepspeedrNinitno_init_weightszeroInitr+rr]rQintegrations.deepspeedr tie_weights) rOrrrrrrrW init_contextsneeds_zero3_initrrirs r _from_configzPreTrainedModel._from_configDs 7FL11!::mT:: :K G    S T T T"fl22EE E eS ! ! *E5))E  $0 ) )N%fn=== J#(   !F * **0**5J*K*KF ' $v - --3ZZ8P-Q-QF *#JJ':EBB&)    !25#,!G!G H H H  :  !6!8!8 9 9 9577h .s-ss;T4466ssssssrzT is not available on CPU. Please make sure you are on any of the supported devices: rHz requires compute capability >= z , but found z with compute capability z.x) rr ImportErrorruziprwrxrcudaget_device_capability)rrrrrrrr info_messageprefacedevice_availability_checks device_namesmajorr\s r_flash_attn_import_errorz(PreTrainedModel._flash_attn_import_errorsSF8P   3 %|((**  L))) *)++ C'9CCCG*)++ !qq?Qqqq$q((1OPW1X1X(!W"t"t=O"t"t"tuuu.0s!N!N!N5%,!N!N!NrrrFrTz with a model not initialized on GPU. Please make sure to have access to a GPU and either initialise the model on a GPU by passing a device_map or initialising the model on CPU and then moving it to GPU, e.g. with `model.to('z')`.a with a model not initialized on GPU and with no GPU available. This is not supported yet. Please make sure to have access to a GPU and either initialise the model on a GPU by passing a device_map or initialising the model on CPU and then moving it to GPU.r)r:rrrr _name_or_pathrrKrrrrrrfloat16bfloat16rrrru)rrrVr param_devices found_devicer device_names r_flash_attn_can_dispatchz(PreTrainedModel._flash_attn_can_dispatchs( >*nnN`nnW[WbWpnnn  Y . .f:Lfffgg g &%aa(LM_(`aaa  ! !t{$788 T[=Z]^=^=^##~AS~~~  ! =   V=OVVV     50O#O#O   Z#5ZZ(,(?ZZEJZZn@ZZZ     !N!NDOObcu>v'?  :-{1022'+ ++XI[XXFQXXX $$VEWVVVtrc|jst|jjdtjjtjdkrrt j tj t j dkr>t dtj jddS)aA Check the availability of SDPA for a given model. Args: is_init_check (`bool`, *optional*): Whether this check is performed early, i.e. at __init__ time, or later when the model and its weights are fully instantiated. This is needed as we also check the devices of the weights, which are only available later after __init__. This allows to raise proper exceptions early before instantiating the full models if we know that the model does not support the requested attention. a does not support an attention implementation through torch.nn.functional.scaled_dot_product_attention yet. Please request the support for this architecture: https://github.com/huggingface/transformers/issues/28005. If you believe this error is a bug, please open an issue in Transformers GitHub repository and load your model with the argument `attn_implementation="eager"` meanwhile. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="eager")`Nrz2.4.1zUsing the `SDPA` attention implementation on multi-gpu setup with ROCM may lead to performance issues due to the FA backend. Disabling it to use alternative backends.FT)r9rrrrrhipr device_countparserrrbackendsenable_flash_sdprrVs r_sdpa_can_dispatchz"PreTrainedModel._sdpa_can_dispatchIs" >*OOO  M  ) ''))A-- e/007=3I3III   y    N  0 0 7 7 7trcf|st|jjddS)zI Check the availability of Grouped MM for a given model. z1 does not support setting experts implementation.T)_can_set_experts_implementationrrrrs r_grouped_mm_can_dispatchz(PreTrainedModel._grouped_mm_can_dispatchgs< 3355 l 7jjjkk ktrc|jst|jjdt st ddS)aK Check the availability of Flex Attention for a given model. Args: is_init_check (`bool`, *optional*): Whether this check is performed early, i.e. at __init__ time, or later when the model and its weights are fully instantiated. This is needed as we also check the devices of the weights, which are only available later after __init__. This allows to raise proper exceptions early before instantiating the full models if we know that the model does not support the requested attention. a does not support an attention implementation through torch's flex_attention. Please request the support for this architecture: https://github.com/huggingface/transformers/issues/34809. If you believe this error is a bug, please open an issue in Transformers GitHub repository and load your model with the argument `attn_implementation="eager"` meanwhile. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="eager")`z]PyTorch Flex Attention requirements in Transformers are not met. Please install torch>=2.5.0.T)r;rrrrhrrs r_flex_attn_can_dispatchz'PreTrainedModel._flex_attn_can_dispatchrsg' >*ttt ,-- o  trrrWc t|\}}|t|ddpgvrd}|bt|dd}t|rA|?||vr;|r d|dn|d}td|d|d |d |}t|\}}|}d } t|r=t jD])} |d | krt | d sd} n*|jrH| rFtr8ts*t|}tr|dkrd } |rd|}t|r |rt||nt||| rtd|dnw#t$r5} | r,t!|d} || || d} ~ wwxYw|||}t|rt||S)a Check that the `attn_implementation` exists and is supported by the models, and try to get the kernel from hub if it matches hf kernels pattern. Args: attn_implementation (`str` or `None`): The attention implementation to check for existence/validity. is_init_check (`bool`, *optional*): Whether this check is performed early, i.e. at __init__ time, or later when the model and its weights are fully instantiated. This is needed as we also check the devices of the weights, which are only available later after __init__. This allows to raise proper exceptions early before instantiating the full models if we know that the model does not support the requested attention. allow_all_kernels (`bool`, optional): Whether to load kernels from unverified hub repos, if `attn_implementation` is a custom kernel outside of the `kernels-community` hub repository. Returns: `str`: The final attention implementation to use, including potential fallbacks from sdpa to eager, or from None to sdpa (to potentially eager). r<NT"requested_attention_implementationzpaged|rzNThis model is compatible with the following flash attention implementations: `z"`. Automatically falling back to `z` instead of `z`.Fflash_attention_rflash_attention_2)rWz/You do not have `flash_attn` installed, using `z%` from the `kernels` library instead!r<rrV)rnrMrmrrrKr-r:rgrirLrjr=rNrMrrrget_correct_attn_implementation) rrrVrWis_pagedbase_implementation compatible_flash_implementationsdefault_flash_implementationapplicable_attn_implementationrequested_original_flash_attn fa_versionrs rr^z5PreTrainedModel._check_and_adjust_attn_implementationsU.)GGZ([([%% 741TVZ#[#[#a_a b b $   */6t=`bf/g/g ,,Pcddd C4@'/OOOGOwB=a@BBBTtuvTw-##zfFzz6Rzzbuzzz'C#(FGZ([([%%)<&(-% 'K^ _ _ _ BGII   (+Jj+J+JJJ@LMijllK591E  % [- [%&& [+,,  [ .HH[-\ *%'' 6,?CV,V,V16- [1Z:X1Z1Z. 3 4 4 L u56J[00Nbstttt1''>Jh>>>   0n!$%8% Check that the `experts_implementation` exists and is supported by the models. Args: experts_implementation (`str` or `None`): The experts implementation to check for existence/validity. Returns: `str`: The final experts implementation to use. )"get_correct_experts_implementation)rr!applicable_experts_implementations rrbz8PreTrainedModel._check_and_adjust_experts_implementations-1,S,STj,k,k)00rrequested_attentionc|dn|}|dgtzvryd|d}|jst|ddr1|dz }t jD]}|d|d |d z }|dd }|jr|d z }|jr|d z }t|dzt|rQtj d|x}r:t| d}|||n]d|vr||nCd|vr? ||n(#tt"f$r}|d|vr|d}Yd}~nd}~wwxYw|S)Nsdpaeager Specified `attn_implementation="zc"` is not supported. The only possible arguments are `attn_implementation="eager"`, `"paged|eager"`_supports_flash_attn_2F, z&`"attn_implementation=flash_attention_z0"`, `"attn_implementation=paged|flash_attention_z"`, zB, `"attn_implementation=sdpa"`, `"attn_implementation=paged|sdpa"`z(, `"attn_implementation=flex_attention"`rHrz^flash_attention_(\d)$rrflex_attention)ALL_ATTENTION_FUNCTIONS valid_keysr:rMrKr-r9r;rrmrprqrgrouprrrr)rrrVapplicable_attentionmessager fa_matchedrs rrz/PreTrainedModel.get_correct_attn_implementation s*)<)DvvJ] y3J3U3U3W3W'W W WA3GAAA  ( 'GD:RTY,Z,Z '4"F"K"M"MUUJ U U UEO U U UUGG!#2#," `__' FEEWs]++ + (K_ ` ` ` /)$=?STT TJ /Z--a0011J  ) )ZWd ) e e e e !5 5 5  ( ( 7 7 7 7 + + + /'' 6666 , / / /&2vAT7T7TG'.$$$$$$ / $#sD55E EErequested_expertsc|dn|}dgtttjtt jzz}d|D}d|dz|d<d|}||vrd|d|d }t ||dkr> |n(#t tf$r}|dkr|d}Yd}~nd}~wwxYw|S) N grouped_mmrcg|]}d|d S)z`experts_implementation="z"`r)rfns rrKzFPreTrainedModel.get_correct_experts_implementation..6s$!`!`!`"Db"D"D"D!`!`!`rzand r<rz#Specified `experts_implementation="z5"` is not supported. The only possible arguments are rH) rrer?r-r8rrrr)rrapplicable_expertsbase_experts_fnsvalid_experts_str_listvalid_experts_strrrs rrz2PreTrainedModel.get_correct_experts_implementation3sL->-F\\L]#9tC0E0J0L0L,M,MPSTmTrTtTtPuPu,u'v'vv!`!`O_!`!`!`%+.DR.H%Hr" II&<== %5 5 5(6H(($((( W%% %  - - ---//// , - - -$ 44G%,"""""" - "!s+CC% C  C%cht|dd}t|tr|Stj|j}|dS tj|}n#ttf$rYdSwxYwtj d|tj rd|v}nd}||_|jS)a Detect whether the class supports setting its attention implementation dynamically. Inspects the module source as a heuristic, which avoids maintaining yet another property flag. Instead, the flag is set dynamically on the first succesful call. )_can_set_attn_implementation_cached_valueNFz%^class \w*Attention\w*\(nn\.Module\):z&ALL_ATTENTION_FUNCTIONS.get_interface(T)rMrsrr modulesrrrL getsourcerr[rprq MULTILINErrO cached_value class_modulecodecan_sets r_can_set_attn_implementationz,PreTrainedModel._can_set_attn_implementationKss$OQUVV lD ) )  {s~66  5 $\22DD#   55  9=tR\ R R >$FGGG8?5<<A''A<;A<ct|dd}t|tr|Stj|j}|dS tj|}n#ttf$rYdSwxYwd|v}||_ |S)a Detect whether the class supports setting its experts implementation dynamically. Inspects the module source as a heuristic, which avoids maintaining yet another property flag. Instead, the flag is set dynamically on the first succesful call. ,_can_set_experts_implementation_cached_valueNFz@use_experts_implementation) rMrsrr rrrrLrrr[r'rs rrz/PreTrainedModel._can_set_experts_implementationhss$RTXYY lD ) )  {s~66  5 $\22DD#   55 047;B8r%c t|ts|n|d|jj}||jjkr`|s(t |jj dn$| |d|}||j_ | D]}||urt|tr|jj|jjkrt|jds|s(t |jj dn|}t|trM|jjD]@}t!|j||jur"|||jj}nA||}||j_ d|j_|jjD]}t!|j|x}t|ts|n|||j}t|ds||jkr|dgt&zvr??8+i.LLL)'=T=_=_=a=a1aaa(e?Qeekxee8<=T=_=_=a=a8b8bee ?QI;NN@S`@@hz@@@y*=>>8%79 8 8rcd|jji}|jjD]$}t|j|d}| |j||<%|S)aU Return the experts implementation of this model and its submodels, as a `dict` in the form accepted by `set_experts_implementation` (`""` for this model, and one entry per sub_config). This is the counterpart of `set_experts_implementation`, e.g. to snapshot the current implementation and later restore it. rN)rrcrrM)rrr,r-s rget_experts_implementationz*PreTrainedModel.get_experts_implementationsX #%dk&I!J![4 Z ZM ]DAAI$8A8Y&}5%%rc t|ts|n|d|jj}|jj}d||fvr||krt d|d|d||jjkr5||}|r ||j_| D]}||urt|tr|jj |jj kr|r|}t|trM|jj D]@}t|j||jur"|||jj}nA||}||j_dS)a Set the requested `experts_implementation` for this model. Args: experts_implementation (`str` or `dict`): The experts implementation to set for this model. It can be either a `str`, in which case it will be dispatched to all submodels if relevant, or a `dict` where keys are the sub_configs name, in which case each submodel will dispatch the corresponding value. rdeepgemm_megamoez*Cannot switch experts implementation from  to z at runtime: `deepgemm_megamoe` is a load-time choice. Reload via `from_pretrained(..., experts_implementation=...)` to switch.N)rsrrrrcr{rbrrdrrrrrMr)rrr*currentrPr+r,s rset_experts_implementationz*PreTrainedModel.set_experts_implementations4d;; U " "'++B 0STT !+5 '+C!D D DTlIlIlPWPPLdPPP  $t{'J J J'+'T'TUm'n'n $3355 X?W < W WI%%y/::&$.$+2GGG==??H&>"4d;;")-)@"" "4; >>)BRRR1G1K1K -y/?/W22."E S &/%Q%QRd%e%e"DV  A+ W WrcDd}g}t}d}|D]}t|trt |ds( |}n#t $rYJwxYw|t |dsat|}||vru||| | |d}||_ |r |d|_ |s)t|jjddSdS) z Enables the gradients for the input embeddings. This is useful for fine-tuning adapter weights while keeping the model weights fixed. c0|ddSNT)requires_grad_)rCinputoutputs rmake_inputs_require_gradszMPreTrainedModel.enable_input_require_grads..make_inputs_require_grads5 s  ! !$ ' ' ' ' 'rFrNregister_forward_hookTra does not expose input embeddings. Gradients cannot flow back to the token embeddings when using adapters or gradient checkpointing. Override `get_input_embeddings` to fully support those features, or set `_input_embed_layer` to the attribute name that holds the embeddings.)rerrsrrrrrtrWrUr=_require_grads_hooks_require_grads_hookrrrr)rr<hooks seen_modulesfound_embeddingsrCinput_embeddings embedding_ids renable_input_require_gradsz*PreTrainedModel.enable_input_require_grads/ s  ( ( (uu  llnn $ $Fv77 GFLb#>#@#@  &     'w7GI`/a/a'.//L|++   \ * * * LL)??@YZZ [ [ [#  $)!  0',QxD $    >*www       sA(( A54A5ct|dd}|sdS|D]}|g|_t|dr|`dSdS)z4 Removes the `_require_grads_hook`. r>Nr?)rMremover>rr?)rr@hooks rdisable_input_require_gradsz+PreTrainedModel.disable_input_require_grads[ sr4d;;  F  D KKMMMM$&! 4. / / )((( ) )rmodalityc:|dvrgd}n$|dkrgd}n|ddg}ntd||D]$}t||rt||cS%|j|ur=t|jd r(|j| }||jkr|S|S) ai Best-effort lookup of the *encoder* module. If provided with `modality` argument, it looks for a modality-specific encoder in multimodal models (e.g. "image_encoder") By default the function returns model's text encoder if any, and otherwise returns `self`. Possible `modality` values are "image", "video" and "audio". imagevideo vision_towervisual vision_modelvision_encoder image_toweraudio) audio_tower audio_encoderspeech_encoderN text_encoderencoderHUnnrecognized modality, has to be "image", "video" or "audio" but found get_encoderrJ)rrrMrr\)rrJpossible_module_namesrJ base_encoders rr\zPreTrainedModel.get_encoderj s ) ) )$o$o$o ! !  $V$V$V ! !  %3Y$? ! !rhprrss s) + +DtT"" +tT***** + ?$ & &74?M+R+R &?666IILt..## rc<|dvrgd}n$|dkrddg}n|ddg}ntd ||D]&}t||rt|||dS'|j|ur>>>{& FK((((('&  ( (( ?%6688 / / et##(////t^^N5#...  / /  - -" ? LSc : : : :!-gfmMach6i6i- FM&*<=>>>>>.---  5 6 6 ?  $ $ & & & & & v bnbnbn] ^ ^ ?f.777F,555vx..: 6=)))vvt,,8 FK(((v~t44@ F/000 6-... F677777A@ &"2"; ; ;Pc@d@d ;#y00$F$455;  &gfm44OL! Jv 5 5 5 Jv/ > > > > >< ; ; ;ris_custom_codec.t|ddrdS|rctd|dDr6td|dDr d|_dS||d|_dS)zM Initialize the weights if they are not already initialized. rrFNc38K|]}t|ddVdS)rrFNrMrs rrz6PreTrainedModel._initialize_weights.. s/nnEGE#7??nnnnnnr)recursec3<K|]}|t|ddVdS)NrrFr)rbuffers rrz6PreTrainedModel._initialize_weights.. sA% 4e<<%%%%rT)rMallrbuffersrrr)rrCrs r_initialize_weightsz#PreTrainedModel._initialize_weights s 6/ 7 7  F  nnVM^M^glM^MmMmnnnnn $nnUn;; )-F % F 6"""$(!!!rc^ttjjdsYdtjdttjt gdfdt ffd t tjjdt|d}||j| dS)a This is equivalent to calling `self.apply(self._initialize_weights)`, but correctly handles composite models. This function dynamically dispatches the correct `init_weights` function to the modules as we advance in the module graph along the recursion. It can handle an arbitrary number of sub-models. Without it, every composite model would have to recurse a second time on all sub-models explicitly in the outer-most `_init_weights`, which is extremely error prone and inefficient. smart_applyrCrNrc|D]7}t|tr||j|*|||8||||Sr)childrenrsrr)rCrrchildrs rrz7PreTrainedModel.initialize_weights..smart_apply2 sv#__..??E!%99?# E5+DnUUUU# E2~>>>>6>*** r) rrrr'rrrrMrr)rsmart_apply_fnrs @rinitialize_weightsz"PreTrainedModel.initialize_weights% sux 66 A BI 8RY.| s@111AEAvOOOO____111rtie_word_embeddingsNz ^[A-Za-z0-9_\.]+(weight)|(bias)$c3BK|]}|VdSr)r)rr!common_case_regexs rrzAPreTrainedModel.get_expanded_tied_weights_keys.. s2__a &&q))______rch|]\}}|Srrrr!r\s rrxzAPreTrainedModel.get_expanded_tied_weights_keys.. sWWWA1WWWrch|]\}}|Srrrs rrxzAPreTrainedModel.get_expanded_tied_weights_keys.. s/[ [ [ !QA[ [ [ r^c.tj|Srro)x source_names rrz@PreTrainedModel.get_expanded_tied_weights_keys.. BIk14M4Mrc.tj|Srro)r target_names rrz@PreTrainedModel.get_expanded_tied_weights_keys.. rrrzAThere is an issue with your definition of `tie_weights_keys` for :z . We found z to tie into )rLrsrr~r,ryrErMrrpcompilerr-rrr named_buffersrfilterrrrr )rrnexpanded_tied_weightsrPsubmodel_tied_weights tied_mappingrall_param_names source_params target_paramstarget_nsource_nrrrrs @@@@rr~z.PreTrainedModel.get_expanded_tied_weights_keysC sKd  )$& !%)%7%7%7%O%O H H! i99H,5,T,Tch,T,i,i)||1111I^IdIdIfIf111-*001FGGG( (. &dk3H%PP" I  !IJ'JKK ____<3D3D3F3FI\I\I^I^3^___ _ _ '$$&& &!#WW)>)>PU)>)V)VWWW[ [ ,,e,DD[ [ [  )5(:(:(<(< ? ? $K +K +K"6*M*M*M*M#_#_``M"6*M*M*M*M#_#_``M &&**=))A--}%%M(:(::a?? LXcLLfqLL -LL source" but `source` is missing in the checkpoint while `target` exists, we *swap* source and target so we can still tie everything to the parameter that actually exists. TrmNrzDThe tied weights mapping and config for this model specifies to tie r3z, but both are present in the checkpoints with different values, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning.rFzYThis checkpoint seem corrupted. The tied weights mapping for this model specifies to tie zk, but both are absent from the checkpoint, and we could not find another related tied weight for those keysrH)rur~rr, enumerate get_parameterrrurequalrrrXrvr get_submoduler _adjust_biasdiscard)rrrritarget_param_namesource_param_nameremove_from_missingsource_is_theretarget_is_there source_param target_param target_backup source_backuptarget_backup_is_there parent_namerJrs rrzPreTrainedModel.tie_weights s! P2II;;$;OOI**++ 9B99M9MF 8F 8 5A5!#4'&*#"3<"G"3<"G#//#'#5#56G#H#HL#'#5#56G#H#HL$*/699l>Q>VZ`>`>` !;|\BB !ct0 2667HIII  !)_;LN_'8%%(8A!a%''8J4 }),===5B,5V2 6&4A 1 % /4+_0__6G___ 778IJJL'''$5$<$= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128. For more details about this, or help on choosing the correct value for resizing, refer to this guide: https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html#requirements-tc mean_resizing (`bool`): Whether to initialize the added embeddings from a multivariate normal distribution that has old embeddings' mean and covariance or to initialize them with a normal distribution that has a mean of zero and std equals `config.initializer_range`. Setting `mean_resizing` to `True` is useful when increasing the size of the embeddings of causal language models, where the generated tokens' probabilities won't be affected by the added embeddings because initializing the new embeddings with the old embeddings' mean will reduce the kl-divergence between the next token probability before and after adding the new embeddings. Refer to this article for more information: https://nlp.stanford.edu/~johnhew/vocab-expansion.html Return: `torch.nn.Embedding`: Pointer to the input tokens Embeddings Module of the model. Nrr modifier_rank) _resize_token_embeddingsrrr-rrGatheredParametersrmrrry vocab_sizer)rrrr model_embedsrrrs rresize_token_embeddingsz'PreTrainedModel.resize_token_embeddings s@H44^EWYfgg  !&8&@ t^44V9JRV9V % ' ' 6  6    22<3FVZ2[[ : :)06q9  : : : : : : : : : : : : : : :&,215J4> ##%%0$ s,B  BBc|}|||||}t|dr|j}t |||jj}||||t|do|j du}|rtrR|sPddl } | j |jd5|jjd}dddn #1swxYwYn|jjd}||} t!| t"jjr|| ||} n|| ||} t| dr| j}t | || jj} | | || |S)N_hf_hookrrr)r)r_get_resized_embeddingsrrrrmrr9r rr-rrrrr#rsrrr_get_resized_lm_headr&) rrrrold_embeddingsr%rHold_embeddings_requires_gradrr old_lm_head new_lm_headold_lm_head_requires_grads rrz(PreTrainedModel._resize_token_embeddingsW si224455 N,>    >: . . 5!*D ~t 4 4 4'5'<'J$%%&BCCC !!.111t^44V9JRV9V   ))++ @L @    ^66~7L\`6aaDD%3%:%@%CNDDDDDDDDDDDDDDD"0!6!= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128. For more details about this, or help on choosing the correct value for resizing, refer to this guide: https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html#requirements-tc mean_resizing (`bool`): Whether to initialize the added embeddings from a multivariate normal distribution that has old embeddings' mean and covariance or to initialize them with a normal distribution that has a mean of zero and std equals `config.initializer_range`. Setting `mean_resizing` to `True` is useful when increasing the size of the embeddings of causal language models, where the generated tokens' probabilities will not be affected by the added embeddings because initializing the new embeddings with the old embeddings' mean will reduce the kl-divergence between the next token probability before and after adding the new embeddings. Refer to this article for more information: https://nlp.stanford.edu/~johnhew/vocab-expansion.html Return: `torch.nn.Embedding`: Pointer to the resized Embedding Module or the old Embedding Module if `new_num_tokens` is `None` Nz5Asking to pad the embedding matrix to a multiple of `z@`, which is not and integer. Please make sure to pass an integerrrzYou are resizing the embedding layer without providing a `pad_to_multiple_of` parameter. This means that the new embedding dimension will be a.. This might induce some performance reduction as *Tensor Cores* will not be available. For more details about this, or help on choosing the correct value for resizing, refer to this guide: https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html#requirements-tcrrzOld embeddings are of type , which is not an instance of zj. You should either use a different resize function or make sure that `old_embeddings` are an instance of rHrzThe new embeddings will be initialized from a multivariate normal distribution that has old embeddings' mean and covariance. As described in this article: https://nlp.stanford.edu/~johnhew/vocab-expansion.html. To disable this, use `mean_resizing=False`)rsrrrmrrrrrr-rrrr&rrrr[rurrrr(_init_added_embeddings_weights_with_meanrrr) rrrrrrrold_num_tokensold_embedding_dimr%added_num_tokensnparamss rrz'PreTrainedModel._get_resized_embeddings~ s*V  )0#66  aL^aaa%!/!6!3HX\2]] Q Q4B4I4N4N4P4P1 1 Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q1?0E0J0J0L0L -N- ^ + +4N4P4P +,:N )! !.",77 $d>.B.B$$bdbn$$L$$$   !(/ '-     N * *= *   ~ . . . . n , , ,   =    .> )++ L     ^668M7N^b6ccAA&HX =="NNDT  / / % ' ' R  R    $+^-BCF2262KK V V4B4I4NrPQrSTSTSTu4U%*2A2qqq51 V V V V V V V V V V V V V V V1?0E0J2A2qqq50QN ! &rr111u - & ' ' 2  2    $+^-BCF2262KK 6 6(6(=%0>0E0J0PQR0S-"-9~PQ?QUcUo>o>o15N. 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6*8)>)CN ! &,:,A,F,LQ,ON ))5>A;MQ_Qk:k:k-1*sI>C''C+.C+'I  II>,K66K:=K:+AN<<OOr transposedc ||St|do|jdu}tr|sddl}|j|jd5|s|jn*|j\}}dddn #1swxYwYnI|s|jn*|j\}}||krts ||_ |St|tj s:tdt|dtj dtj d|s||fn||f} |jdu} tj | | |jj|jjd } ||kr|s|| n||kr|rt&d ||z } tr|sddl}|jg} | r | |jgz } |j| d5||| ||| || r||| | dddn #1swxYwYn3||| ||| || r||| | t/||}trn|slddl}|j|j| j| jg} |j| d5|| |||| dddn #1swxYwYn|| |||| t3| d d | S) a Build a resized Linear Module from a provided old Linear Module. Increasing the size will add newly initialized vectors at the end. Reducing the size will remove vectors from the end Args: old_lm_head (`torch.nn.Linear`): Old lm head liner layer to be resized. new_num_tokens (`int`, *optional*): New number of tokens in the linear matrix. Increasing the size will add newly initialized vectors at the end. Reducing the size will remove vectors from the end. If not provided or `None`, just returns a pointer to the input tokens `torch.nn.Linear` module of the model without doing anything. transposed (`bool`, *optional*, defaults to `False`): Whether `old_lm_head` is transposed or not. If True `old_lm_head.size()` is `lm_head_dim, vocab_size` else `vocab_size, lm_head_dim`. mean_resizing (`bool`): Whether to initialize the added embeddings from a multivariate normal distribution that has old embeddings' mean and covariance or to initialize them with a normal distribution that has a mean of zero and std equals `config.initializer_range`. Setting `mean_resizing` to `True` is useful when increasing the size of the embeddings of causal language models, where the generated tokens' probabilities will not be affected by the added embeddings because initializing the new embeddings with the old embeddings' mean will reduce the kl-divergence between the next token probability before and after adding the new embeddings. Refer to this article for more information: https://nlp.stanford.edu/~johnhew/vocab-expansion.html Return: `torch.nn.Linear`: Pointer to the resized Linear Module or the old Linear Module if `new_num_tokens` is `None` Nrrrz#Old language model head is of type rzg. You should either use a different resize function or make sure that `old_lm_head` are an instance of rH)rqrraThe new lm_head weights will be initialized from a multivariate normal distribution that has old embeddings' mean and covariance. As described in this article: https://nlp.stanford.edu/~johnhew/vocab-expansion.html. To disable this, use `mean_resizing=False`rrT)rrr-rrrrmr&rrrsrrzr[rurqrrrrr%_init_added_lm_head_weights_with_mean"_init_added_lm_head_bias_with_meanr!_copy_lm_head_original_to_resizedr)rrrrrrrrold_lm_head_dimnew_lm_head_shapehas_new_lm_head_biasrrrnum_tokens_to_copys rrz$PreTrainedModel._get_resized_lm_head sH  ! t^44V9JRV9V % ' '      22;3EUY2ZZ  5?bK&++---[EWEYEYE[E[E`E`EbEb0                2<^ "'')))ASAUAUAWAWA\A\A^A^ ,NO ^ + +4N4P4P +'5K $ +ry11 !d;6G6G!!gigp!!I!!! FPv_n==VdfuUv*/t; i %%,$*     N * *= *   { + + + + n , , ,   =    .> )++ hL h    %,-'1{/00F^66vT6RRll>>#[/>Sceo,l?? [Zjkkk lllllllllllllll::o~O_ak(h;;KVfggg @@ % ' '      !(+*:K)covariance_matrix) sample_shaper<r)rmrr rrroTrpositive_definitecheckr distributionsmultivariate_normalMultivariateNormalsamplerr) rrr%rrold_embeddings_weightmean_embeddingsold_centered_embeddings covarianceepsilonis_covariance_psd distributions rrz8PreTrainedModel._init_added_embeddings_weights_with_mean s{!/ 5 : = =em L L*%:CCC"7/"I,.1HH>Y '9??*@TUUYY[[   .BUU7Z3GVLFREXEX.0FYFFb&,--  ! &r,<'<'>'>'A B B B  aaa(//0@!DDGGH]Hcdd  ! &r,<'<'>'>'A B B Brc|r6|jjj|j_|jjj|j_||||||r8|jjj|j_|jjj|j_dSdSr)rmrr r)rrrrrrrs rrz5PreTrainedModel._init_added_lm_head_weights_with_mean s  @&1&8&=&?K  #&1&8&=&?K  # 55k;P^`pqqq  @&1&8&=&?K  #&1&8&=&?K  # # # @ @rc4tj|jjdtj}tj|jjdtj}|jjd|zd|d|zdS)Nr)rrrr<rrn)rrorqrrrpr r)rrrr bias_meanbias_stds rrz2PreTrainedModel._init_added_lm_head_bias_with_mean sJ{/41EMRRR 9[-2;;;>>u}MMb#33556>>ISWZbSb>cccccrc|s,|jjd|ddf|jjd|ddf<n+|jjddd|f|jjddd|f<|r%|jjd||jjd|<dSdSr)rmrrq)rrrrrrs rrz1PreTrainedModel._copy_lm_head_original_to_resized s n>I>P>UViWiViklklklVl>mK  #$7%7$7$: ; ;>I>P>UVWVWVWYlZlYlVl>mK  #AAA':(:':$: ;  d9D9I9NObPbOb9cK  !"5#5"5 6 6 6 d drnew_num_position_embeddingsc \td|jd|jd|jjd)Nz4`resize_position_embeddings` is not implemented for B`. To implement it, you should overwrite this method in the class  in `modeling_.py`rrr)rrs rresize_position_embeddingsz*PreTrainedModel.resize_position_embeddings sV! p4> p p26. p pPTP^Pi p p p   rc \td|jd|jd|jjd)Nz1`get_position_embeddings` is not implemented for r r!r"r#rs rget_position_embeddingsz'PreTrainedModel.get_position_embeddings sV! p p p26. p pPTP^Pi p p p   rcttjdkr||ddS)z Initialize and tie the weights if needed. If using a custom `PreTrainedModel`, you need to implement any initialization logic in `_init_weights`. rF)rN)rrrrrrs rrzPreTrainedModel.init_weights sN 6 7 75<;O;O O O  # # % % % 511111rc|jst|jjd|ddi}t jt fi|}dtj|j j v}|s| d|nC| t |j dt d |jd k}|pt|d d}|r|dSdS) a Activates gradient checkpointing for the current model. We pass the `__call__` method of the modules instead of `forward` because `__call__` attaches all the hooks of the module. https://discuss.pytorch.org/t/any-different-between-model-input-and-model-forward-input/3690/2 Args: gradient_checkpointing_kwargs (dict, *optional*): Additional keyword arguments passed along to the `torch.utils.checkpoint.checkpoint` function. z) does not support gradient checkpointing.N use_reentrantFrT)enablegradient_checkpointing_funcrVYou are using an old version of the checkpointing format that is deprecated (We will also silently ignore `gradient_checkpointing_kwargs` in case you passed it).Please update to the new format on your modeling file. To use the new format, you need to completely remove the definition of the method `_set_gradient_checkpointing` in your model.r._hf_peft_config_loaded)rArrr functoolsr rrL signature_set_gradient_checkpointingrapplyrrr/rMrE)rgradient_checkpointing_kwargsr+_is_using_old_formatneeds_embedding_gradsenable_input_gradss rrz-PreTrainedModel.gradient_checkpointing_enable s-3 d 7bbbcc c ( 0-@[\\\ 8&AAA04-, >*]]]   rcZ|jr|dtj|jjv}|s|dnCt d|t|jdt|ddr| dSdS)zK Deactivates gradient checkpointing for the current model. rF)r*r-r,r.N) rArLr0r1rrrr2r rMrI)rr4s rgradient_checkpointing_disablez.PreTrainedModel.gradient_checkpointing_disable9 s  / S$+g.?@`.a.a.l#l ' S000>>>>L 74#C5QQQRRR 415 9 9 /  , , . . . . . / /rcXtd|DS)zT Whether gradient checkpointing is activated for this model or not. c3DK|]}t|do|jVdS)rN)rr)rms rrz.R s6mmYZ71677TA2||| d<t|dd} t|dd} | duo(t| to| } | !| s| st d| jjd|jtdstd tj |r td |d dStj|d tj|}|r| dd}| d|tj jd}| dd}t)j|fdd i| j}||}i}| | |\}}d|d<t3|}|j}t7|dd|j_|jjdg|j_ |!rtE|||j|r| s|j#||$r|j%#|| rt&d|'|}|r@t&di}|(D] \}}||d|<|}|)}tU|dkrt d|d}|j+|}|#|||,}d}t[|drtUt]|j/0dkrLd |j/0vsd!|j/0vrd }tcj2d"tfr'thj5j6j7D]\}}||}|j8)tU|j8dkr|j8D] }||vr||= |j!ts||j:|j;|j}ty||}| r|s| st{||}| st|}t||}nt}|Ad#d$Ad%d&} t|| |'}!tjC|D]}"tj D||"}#|Ad#d(Ad%d(}$|"Ad#d(Ad%d(}%tjFd)}&|"G|$rStj |#r4|"|!jHvr+|r)|&I|%tjJ|#d}'|!jKr|r| r| r|!jLni}'tjN|!jH(d*+D]\2}(tj D|2}"i})|(D]P}*||*}+|r |+jOjPd,krt||*}+|+R|)|*<Q|rm| rk| si t{||)})|!jKr3|'S2fd-|)TDn#t$rtd.wxYwt|)|"|/~)d},|!jKr d0|Xi|!jY|'d1},|,>tj D||}-t&d2|-nt}.tj D|t|.|}.t|.d3d455}/tj]|,d6d 7d8z}0|/^|0dddn #1swxYwYt&d9|d:tU|!jHd;|.d|rgt||j`|<}1|1atj D|d=|b||||||>dSdS)?a Save a model and its configuration file to a directory, so that it can be re-loaded using the [`~PreTrainedModel.from_pretrained`] class method. Arguments: save_directory (`str` or `os.PathLike`): Directory to which to save. Will be created if it doesn't exist. is_main_process (`bool`, *optional*, defaults to `True`): Whether the process calling this is the main process or not. Useful when in distributed training like TPUs and need to call this function on all processes. In this case, set `is_main_process=True` only on the main process to avoid race conditions. state_dict (nested dictionary of `torch.Tensor`): The state dictionary of the model to save. Will default to `self.state_dict()`, but can be used to only save parts of the model or if special precautions need to be taken when recovering the state dictionary of a model (like when using model parallelism). push_to_hub (`bool`, *optional*, defaults to `False`): Whether or not to push your model to the Hugging Face model hub after saving it. You can specify the repository you want to push to with `repo_id` (will default to the name of `save_directory` in your namespace). max_shard_size (`int` or `str`, *optional*, defaults to `"50GB"`): The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size lower than this size. If expressed as a string, needs to be digits followed by a unit (like `"5MB"`). If a single weight of the model is bigger than `max_shard_size`, it will be in its own checkpoint shard which will be bigger than `max_shard_size`. variant (`str`, *optional*): If specified, weights are saved in the format model..safetensors. token (`str` or `bool`, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use the token generated when running `hf auth login` (stored in `~/.huggingface`). save_peft_format (`bool`, *optional*, defaults to `True`): For backward compatibility with PEFT library, in case adapter weights are attached to the model, all keys of the state dict of adapters needs to be prepended with `base_model.model`. Advanced users can disable this behaviours by setting `save_peft_format` to `False`. save_original_format (`bool`, *optional*, defaults to `True`): For backward compatibility with the previous versions of `transformers` you can save the checkpoint with its reverse mapping. The reverse mapping needs to exists even if the model was loaded from a None legacy checkpoint. kwargs (`dict[str, Any]`, *optional*): Additional key word arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method. Nrr.FrzThe model is quantized with z and is not serializable - check out the warnings from the logger on the traceback to understand the reason why the quantized model is not serializable.z0.31.4z[Saving a model with tensor parallelism requires `huggingface_hub` version 0.31.4 or higher.zProvided path (z#) should be a directory, not a fileT)exist_okcommit_messagerepo_idr< create_prrHr$formatrHrFSDP)rzhDetected adapters on the model, saving the model in the PEFT format, only adapter weights will be saved.)rzTo match the expected format of the PEFT library, all keys of the state dict of adapters will be prepended with `base_model.model`.zbase_model.model.zMultiple active adapters detected, saving multiple active adapters is not supported yet. You can save adapters separately one by one by iteratively calling `model.set_adapter(adapter_name)` then `model.save_pretrained(...)`r hf_device_maprdiskz}Attempting to save a model with offloaded modules. Ensure that unallocated cpu memory exceeds the `shard_size` (50GB default)z.binz {suffix}.binrz{suffix}.safetensors)filename_patternrDrz(.*?)-\d{5}-of-\d{5}zWriting model shards)descrc3ZK|]%}|tjiV&dSr)rrbasename)rr! shard_files rrz2PreTrainedModel.save_pretrained..[s9)m)mPQ1bg.>.>z.J.J*K)m)m)m)m)m)mrzWe could not revert some weight conversions because of offlading, and several weights needed for a single conversion operation living in different shard files. Try reducing `max_shard_size` a bit, or worst case set `save_original_format=False`.)metadatatotal_parameters)rU weight_mapzModel weights saved in wrrr)indent sort_keys z:The model is bigger than the maximum size per checkpoint (z) and is going to be split in z^ checkpoint shards. You can find where each parameters has been saved in the index located at )rz README.md)rIrrK)crMrsrSis_serializablerquantization_config quant_method_tp_sizervrrrrrerrormakedirsrrXrseprr create_reporJ_get_files_timestampsget_state_dict_and_metadata unwrap_modelrrrrr removeprefix architecturesrr'save_pretrainedrergrget_adapter_state_dictr,active_adaptersr peft_configrrrerNrrrIS_SAGEMAKER_MP_POST_1_10smpstatemodule_managertranslate_functionsr8rFr= _device_meshrr%rZrrWrrlistdirrrprr filename_to_tensors fullmatchrGrtensor_to_filenamerktqdmrrur5 contiguousryr-rr{safe_save_filer rUrYrjsondumpswriterpr-save_upload_modified_files)3rrArBrrCrDrrrErFrr.rquantization_serializablesave_directory_pathrIrJrKfiles_timestampsrU model_to_saverpeft_state_dictrractive_adaptercurrent_peft_config is_offloaded smp_to_hfr\ ignore_keyrrPstate_dict_splitr full_filenameweights_no_suffixfilename_no_suffixregrW tensor_namesshard_state_dict tensor_namerindexpath_to_weightssave_index_filer6content model_cardrTs3 @rrizPreTrainedModel.save_pretrainedT s v  #F7O!(/G!O!Ot^T::  $ qL+)N)N qS_SoSoSqSq "  #,B #Kd #u|/O/\uuu  = $-PQY-Z-Z $m  7>>. ) )  LL^>^^^ _ _ _ F NT2222 i77  J#ZZ(8$??Njj,?,E,Ebgk,R,RSU,VWWG ;66I*fhh*7LLTLVLLTG#99.II   ##/#K#KD#Q#Q J!%T** #%(ZZ%5%5c%:%:1%= "/<.E.N.[.[\b.c.c-d *     I t^DK H H H H  D) E$44^DDD  "" P/??OOO% D ~+AAZAXX #1KK^')O&0&6&6&8&8KK UEJ(AC(A(ABB!0J!%!5!5!7!7~&&**$u"0!2&*&6~&F##33NCCC  &1133J D/ * * C*1133445599$,3355554CUC\C\C^C^9^9^L M:    % 3 # 8 L 3 3 1&Yz22   ' 3D>LL4L'//GGOOP^`vww= )9.     >22 ) )HGLLBBM!- 4 4VR @ @ H HY[ \ \ "*!1!1&"!=!=!E!EnVX!Y!Y *455C##$566 )GNN=11 )$4$HHH#IMM"455A -(((  &  %)=F\ 33 )0  0 6 6 8 8?U) ) ) ( !( ! $J w||NJ??H! + D D # 44  RFM$6&$@$@5m[QQF170A0A0C0C --   4 =S  '? O_'`'`$'2n")))m)m)m)mUeUjUjUlUl)m)m)mmmm &< +X I I I I    & /1D1D1F1FdJZJcd(E = gll><HHO KKC/CC D D D D5O gll><Y`;a;abbOosW=== !*U1EEEL    ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! KK7^77 0 DEE77$3777     27DOSXYYYJ OOBGLLEE F F F  ' ' -# (       s A ]]070a33a7:a7c|j|jng}|dg}t|tr|g}|D]}||vr|||r||d<t j|i|S)Nr)r-rrsrrUrJrC)rrrr tags_kwargsrrs rrCzPreTrainedModel.push_to_hubs"&/"=t2jj,, k3 ' ' (&-K ! !C$ C    "!F6N"uww"D3F333rctd|D}|r0td|D}||z}|S)a Get the memory footprint of a model. This will return the memory footprint of the current model in bytes. Useful to benchmark the memory footprint of the current model and design some tests. Solution inspired from the PyTorch discussions: https://discuss.pytorch.org/t/gpu-memory-that-model-uses/56822/2 Arguments: return_buffers (`bool`, *optional*, defaults to `True`): Whether to return the size of the buffer tensors in the computation of the memory footprint. Buffers are tensors that do not require gradients and not registered as parameters. E.g. mean and std in batch norm layers. Please see: https://discuss.pytorch.org/t/what-pytorch-means-by-buffers/120266/2 c3hK|]-}||zV.dSrr=r@rs rrz7PreTrainedModel.get_memory_footprint..s=YYe%..""U%7%7%9%99YYYYYYrc3hK|]-}||zV.dSrr)rbufs rrz7PreTrainedModel.get_memory_footprint..s;YY33<<>>C,<,<,>,>>YYYYYYr)sumrr)rreturn_buffersmemmem_bufss rget_memory_footprintz$PreTrainedModel.get_memory_footprintsdYYtGXGXYYYYY  !YY$,,..YYYYYH.C rct|ddtjkrddlm}t j|i||D]Y}t||rGt|dkr |d}n| dd}||Z|St|ddtj kr t|ddrtdt j|i|S) Nquantization_methodr HQQLinearrris_loaded_in_8bitFzCalling `cuda()` is not supported for `8-bit` quantized models. Please use the model as it is, since the model has already been set to the correct devices.) rMr~HQQhqq.core.quantizerrJrrrsrrBITS_AND_BYTESr)rrrrrCrrs rrzPreTrainedModel.cudas$ 4. 5 59K9O O O 3 3 3 3 3 3 EGGL$ )& ) ) ),,.. ( (fi00(4yy1}}!%a!'Hf!=!=KK'''K 4. 5 59K9Z Z Zt0%88  suww|T,V,,,rcdd|v}|s#|D] }t|tjrd}n!t|ddtjkrddlm}tj |i|| D]Y}t||rGd|vr |d}n|d}d|vr |d}n|r|}nd}|||_ | |Z|S|r.t|ddtj krtdt|ddtjkrA|rtdt|d d rt!d std n0t|ddtjkr|rtd tj |i|S)NrTrrrrzBCasting a Quark quantized model to a new `dtype` is not supported.zYou cannot cast a bitsandbytes model in a new `dtype`. Make sure to load the model using `from_pretrained` using the desired `dtype` by passing the correct `dtype` argument.rFz0.48zsYou need to install `pip install bitsandbytes>=0.48.0` if you want to move a 8-bit model across devices using to().zYou cannot cast a GPTQ model in a new `dtype`. Make sure to load the model using `from_pretrained` using the desired `dtype` by passing the correct `dtype` argument.)rsrrrMr~rrrrJr r compute_dtyperQUARKrrreGPTQ) rrrdtype_present_in_argsargrrCrrrs rr zPreTrainedModel.tos@!(6 1$   c5;//,0)E 4. 5 59K9O O O 3 3 3 3 3 3 EGGJ ' ' ' ',,.. ( (fi00(6))!'!1!%a&(( &w.% # $(/4,KK'''K  cWT3H$%O%OSeSk%k%kabb b 4. 5 59K9Z Z Z$  P t0%88 AZ[aAbAb  JT0$ 7 7;M;R R R$  Huwwz4*6***rcpt|ddrtdtj|S)NrFz`.half()` is not supported for quantized model. Please use the model as it is, since the model has already been casted to the correct `dtype`.)rMrrJhalfrrrs rrzPreTrainedModel.half sD 4 / / 'I   577<& &rcpt|ddrtdtj|S)NrFz`.float()` is not supported for quantized model. Please use the model as it is, since the model has already been casted to the correct `dtype`.)rMrrJfloatrs rrzPreTrainedModel.floatsD 4 / / (I  !577=$' 'rrrrct||jtjt g}|r!|t trddl}|sw|sut d| tj |j tt!gnr|r5| t#jdt'gn:| t#jdtjg|S)Nrrrr)rrrno_tie_weightsrPrUr<r-rrrrNrrrr+rrrrmeta_device_safe_creation_ops)rOrrrrWrrs rget_init_contextz PreTrainedModel.get_init_contextsO +5#,??ATAVAVXeXgXgh  :  !6!8!8 9 9 9 % ' ' _     T(: T ^___$$,..!++@P@R@R+SS')) T$$el6&:&: _use_kernels)r use_kernelsr]rrs rset_use_kernelszPreTrainedModel.set_use_kernelsOs%  &'))  I>$mX\]jXkXkmm &3"55d;;;77=== d & & & & & & %D   rr) rrrrrrrrr fusion_configrrOrrrrrrrrrrc |dd}|dd}|dd}|dd}|dd}|dd}|d d}|d d}|d d}|d d}|d d}|dd}|dd}|dd}|dd}|dd}|dipi}|dd} |dd}!|dd}"|dd}#|dd}$|dd}%|dd}&|dd}'|dd}(|dd})|d d}*|d!d}+|%|#d"}#d#D]},||,d}-|||n|}|d"}tr|sd$}|||||||d%}.i|.d&|i}/|||"td'|d"krGt t jd(d)rt d*|#|$t|#|$|&|+\}}&}$|"tstd,|i}t||/fi|\}0}}t|}d-d.|d/}1|||1d0<t|tsn||n|}2|j}3|3t|jd1|3j|2fd$|"||d2|.|\}}4d|4vr|4d|4d|}n'tj|}|}4t)|d|}||/d&<d3|vr|d3|_d4|vr|d4|_t/|||| |1\}5}}|"rQ|5td5|>t|t0rd6|vsd6|vrt5d7|*|)std8d$})t9|||"| |/|1|t)|d9d|: \}6}7|5du}8t=||6||7|| |5\}}|"rXd;dd$|:|?d@}||_#| tj| |_$t)|dAd} | d;dBl%m&};|;||| |*O|)rMd;dCl'm(}<|(rtSgng}=tU|=5|<|||*dddn #1swxYwY|+||8tX|(}>tj|}tU|>5||g| Ri|4}?t[|?|5|5.|?|||6|)Ddddn #1swxYwY|?/|}@ta|?|+|5}Atbr|&te|?|#|%|&|$}?|tg|?|||5}ti|||7|||||@|5|&| |A| |.| E}B|5|?||6|B\}C}D|6|?|B|C}C|?7|?8|)|*|?9r)tu|?dFr|"s|?j;|!|||fi|.d|'i||ityt{|d;ks#d6t{|vrt}|?|5|||D||5|5|?_?|5@|?|0 |||dG<|?A|0| |B|H}C|r|?|CBfS|?S)Ia@ Instantiate a pretrained pytorch model from a pre-trained model configuration. The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated). To train the model, you should first set it back in training mode with `model.train()`. The warning *Weights from XXX not initialized from pretrained model* means that the weights of XXX do not come pretrained with the rest of the model. It is up to you to train those weights with a downstream fine-tuning task. The warning *Weights from XXX not used in YYY* means that the layer XXX is not used by YYY, therefore those weights are discarded. Parameters: pretrained_model_name_or_path (`str` or `os.PathLike`, *optional*): Can be either: - A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co. - A path to a *directory* containing model weights saved using [`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`. - `None` if you are both providing the configuration and state dictionary (resp. with keyword arguments `config` and `state_dict`). model_args (sequence of positional arguments, *optional*): All remaining positional arguments will be passed to the underlying model's `__init__` method. config (`Union[PreTrainedConfig, str, os.PathLike]`, *optional*): Can be either: - an instance of a class derived from [`PreTrainedConfig`], - a string or path valid as input to [`~PreTrainedConfig.from_pretrained`]. Configuration for the model to use instead of an automatically loaded configuration. Configuration can be automatically loaded when: - The model is a model provided by the library (loaded with the *model id* string of a pretrained model). - The model was saved using [`~PreTrainedModel.save_pretrained`] and is reloaded by supplying the save directory. - The model is loaded by supplying a local directory as `pretrained_model_name_or_path` and a configuration JSON file named *config.json* is found in the directory. state_dict (`dict[str, torch.Tensor]`, *optional*): A state dictionary to use instead of a state dictionary loaded from saved weights file. This option can be used if you want to create a model from a pretrained configuration but load your own weights. In this case though, you should check if using [`~PreTrainedModel.save_pretrained`] and [`~PreTrainedModel.from_pretrained`] is not a simpler option. cache_dir (`Union[str, os.PathLike]`, *optional*): Path to a directory in which a downloaded pretrained model configuration should be cached if the standard cache should not be used. ignore_mismatched_sizes (`bool`, *optional*, defaults to `False`): Whether or not to raise an error if some of the weights from the checkpoint do not have the same size as the weights of the model (if for instance, you are instantiating a model with 10 labels from a checkpoint with 3 labels). force_download (`bool`, *optional*, defaults to `False`): Whether or not to force the (re-)download of the model weights and configuration files, overriding the cached versions if they exist. proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. output_loading_info(`bool`, *optional*, defaults to `False`): Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages. local_files_only(`bool`, *optional*, defaults to `False`): Whether or not to only look at local files (i.e., do not try to download the model). token (`str` or `bool`, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use the token generated when running `hf auth login` (stored in `~/.huggingface`). revision (`str`, *optional*, defaults to `"main"`): The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any identifier allowed by git. To test a pull request you made on the Hub, you can pass `revision="refs/pr/"`. attn_implementation (`str`, *optional*): The attention implementation to use in the model (if relevant). Can be any of - `"eager"` (manual implementation of the attention) - `"sdpa"` (using [`F.scaled_dot_product_attention`](https://pytorch.org/docs/master/generated/torch.nn.functional.scaled_dot_product_attention.html)) - `"flash_attention_2"` (using [Dao-AILab/flash-attention](https://github.com/Dao-AILab/flash-attention)) - `"flash_attention_3"` (using [Dao-AILab/flash-attention/hopper](https://github.com/Dao-AILab/flash-attention/tree/main/hopper)) - `"flash_attention_4"` (using [Dao-AILab/flash-attention/flash_attn/cute](https://github.com/Dao-AILab/flash-attention/tree/main/flash_attn/cute)). By default, if available, SDPA will be used. The default is otherwise the manual `"eager"` implementation. Accept HF kernel references in the form: /[@][:] - and are any non-"/" and non-":" sequences. - "@" is optional (branch, tag, or commit-ish), e.g. "@main", "@v1.2.0", "@abc123". - ":" is optional and selects a function inside the kernel repo. - Both options can appear together and in this order only: @revision first, then :kernel_name. - We intentionally allow a leading "|" prefix (e.g., "flash|...") because the code strips it before loading; '|' is not excluded in the character classes here. Examples that match: "org/model" "org/model@main" "org/model:custom_kernel" "org/model@v1.2.3:custom_kernel" experts_implementation (`str`, *optional*): The experts implementation to use in the model (if relevant). Can be any of: - `"eager"` (sequential implementation of the experts matrix multiplications). - `"batched_mm"` (using [`torch.bmm`](https://pytorch.org/docs/stable/generated/torch.bmm.html)). - `"grouped_mm"` (using [`torch.nn.functional.grouped_mm`](https://docs.pytorch.org/docs/main/generated/torch.nn.functional.grouped_mm.html)). By default, if the model supports it, `"grouped_mm"` will be used. The default is otherwise the manual `"eager"` implementation. > Parameters for big model inference dtype (`str` or `torch.dtype`, *optional*, defaults to `"auto"`): Override the default `torch_dtype` and load the model under a specific `dtype`. The different options are: 1. `torch.float16` or `torch.bfloat16` or `torch.float`: load in a specified `dtype`, ignoring the model's `config.dtype` if one exists. If not specified - the model will get loaded in `torch.float` (fp32). 2. `"auto"` - A `dtype` or `torch_dtype` entry in the `config.json` file of the model will be attempted to be used. If this entry isn't found then next check the `dtype` of the first weight in the checkpoint that's of a floating point type and use that as `dtype`. This will load the model using the `dtype` it was saved in at the end of the training. It can't be used as an indicator of how the model was trained. Since it could be trained in one of half precision dtypes, but saved in fp32. 3. A string that is a valid `torch.dtype`. E.g. "float32" loads the model in `torch.float32`, "float16" loads in `torch.float16` etc. For some models the `dtype` they were trained in is unknown - you may try to check the model's paper or reach out to the authors and ask them to add this information to the model's card and to insert the `dtype` or `torch_dtype` entry in `config.json` on the hub. device_map (`str` or `dict[str, Union[int, str, torch.device]]` or `int` or `torch.device`, *optional*): A map that specifies where each submodule should go. It doesn't need to be refined to each parameter/buffer name, once a given module name is inside, every submodule of it will be sent to the same device. If we only pass the device (*e.g.*, `"cpu"`, `"cuda:1"`, `"mps"`, or a GPU ordinal rank like `1`) on which the model will be allocated, the device map will map the entire model to this device. Passing `device_map = 0` means put the whole model on GPU 0. To have Accelerate compute the most optimized `device_map` automatically, set `device_map="auto"`. For more information about each option see [designing a device map](https://hf.co/docs/accelerate/main/en/usage_guides/big_modeling#designing-a-device-map). max_memory (`Dict`, *optional*): A dictionary device identifier to maximum memory if using `device_map`. Will default to the maximum memory available for each GPU and the available CPU RAM if unset. tp_plan (`Optional[Union[dict, str]]`, *optional*): A torch tensor parallel plan, see [here](https://pytorch.org/tutorials/intermediate/TP_tutorial.html). Use `tp_plan="auto"` to use the predefined plan based on the model. If it's a dict, then it should match between module names and desired layout. Note that if you use it, you should launch your script accordingly with `torchrun [args] script.py`. This will be much faster than using a `device_map`, but has limitations. tp_size (`str`, *optional*): A torch tensor parallel degree. If not provided would default to world size. device_mesh (`torch.distributed.DeviceMesh`, *optional*): A torch device mesh. If not provided would default to world size. Used only for tensor parallel for now. If provided, it has to contain dimension named `"tp"` in case it's > 1 dimensional, this dimension will be used for tensor parallelism offload_folder (`str` or `os.PathLike`, *optional*): If the `device_map` contains any value `"disk"`, the folder where we will offload weights. offload_buffers (`bool`, *optional*): Whether or not to offload the buffers with the model parameters. quantization_config (`Union[QuantizationConfigMixin,Dict]`, *optional*): A dictionary of configuration parameters or a QuantizationConfigMixin object for quantization (e.g bitsandbytes, gptq). subfolder (`str`, *optional*, defaults to `""`): In case the relevant files are located inside a subfolder of the model repo on huggingface.co, you can specify the folder name here. variant (`str`, *optional*): If specified load weights from `variant` filename, *e.g.* pytorch_model..bin. use_safetensors (`bool`, *optional*, defaults to `None`): Whether or not to use `safetensors` checkpoints. Defaults to `None`. If not specified and `safetensors` is not installed, it will be set to `False`. weights_only (`bool`, *optional*, defaults to `True`): Indicates whether unpickler should be restricted to loading only tensors, primitive types, dictionaries and any types added via torch.serialization.add_safe_globals(). When set to False, we can load wrapper tensor subclass weights. disable_mmap (`bool`, *optional*): Whether to disable memory mapping when loading safetensors checkpoints. When `None` (default), it is auto-detected to `True` when the checkpoint lives on an `hf-mount` FUSE filesystem (used by HF Spaces/Endpoints), where mmap + parallel page-faults can deadlock. When `True`, files are read fully into memory and parsed with `safetensors.torch.load`. When `False`, the default memory-mapped loader is always used. fusion_config (`dict[str, bool | dict[str, Any]]`, *optional*): Optional fusion configuration applied before model instantiation. Each key enables a fusion family and its value can either be `True` to enable that fusion with default options or a dictionary of family-specific options. For example, `{"patch_embeddings": True}` enables patch embedding fusion. This should only be used as an inference optimization, as it can slightly change outputs. If omitted, `from_pretrained()` falls back to `config.fusion_config` when available. Refer to the fusion mapping guide in `docs/source/en/fusion_mapping.md` for more details. key_mapping (`dict[str, str], *optional*): A potential mapping of the weight names if using a model on the Hub which is compatible to a Transformers architecture, but was not converted accordingly. kwargs (remaining dictionary of keyword arguments, *optional*): Can be used to update the configuration object (after it being loaded) and initiate the model (e.g., `output_attentions=True`). Behaves differently depending on whether a `config` is provided or automatically loaded: - If a configuration is provided with `config`, `**kwargs` will be directly passed to the underlying model's `__init__` method (we assume all relevant updates to the configuration have already been done) - If a configuration is not provided, `kwargs` will be first passed to the configuration class initialization function ([`~PreTrainedConfig.from_pretrained`]). Each key of `kwargs` that corresponds to a configuration attribute will be used to override said attribute with the supplied `kwargs` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's `__init__` function. Activate the special ["offline-mode"](https://huggingface.co/transformers/installation.html#offline-mode) to use this method in a firewalled environment. Examples: ```python >>> from transformers import BertConfig, BertModel >>> # Download model and configuration from huggingface.co and cache. >>> model = BertModel.from_pretrained("google-bert/bert-base-uncased") >>> # Model was saved using *save_pretrained('./test/saved_model/')* (for example purposes, not runnable). >>> model = BertModel.from_pretrained("./test/saved_model/") >>> # Update configuration during loading. >>> model = BertModel.from_pretrained("google-bert/bert-base-uncased", output_attentions=True) >>> assert model.config.output_attentions == True ``` rNrroutput_loading_infoF_from_pipeline _from_autorrr max_memoryoffload_folderrr]rrrradapter_kwargs adapter_namerxrgrrtp_sizerrtrust_remote_coderWrr] key_mappingr)mirror _fast_initlow_cpu_mem_usagefrom_tf from_flaxoffload_state_dictT)rrrrrrrrzq`state_dict` cannot be passed together with a model name or a `gguf_file`. Use one of the two loading strategies. WORLD_SIZEraYou've set device_map=`auto` while triggering a distributed run with torchrun. This might lead to unexpected behavior. If your plan is to load the model on each device, you should set device_map={: PartialState().process_index} where PartialState comes from accelerate library)rrrzIaccelerate is required when loading a GGUF file `pip install accelerate`.ripytorch) file_typer%from_auto_classusing_pipelinezN does not define `config_class`; pass an explicit config to `from_pretrained`.)return_unused_kwargsrrrrrzYou cannot combine Quantization and loading a model from a GGUF file, try again by making sure you did not passed a `quantization_config` or that you did not load a quantized model from the Hub.rOzxOne or more modules is configured to be mapped to disk. Disk offload is not supported for models loaded from GGUF files.zA kernel_config was provided but use_kernels is False; setting use_kernels=True automatically. To suppress this warning, explicitly set use_kernels to True.transformers_weights) rrrrrrrrrr)load_gguf_checkpointrr)return_tensors model_to_loadrrSr)register_fusion_patches)(register_kernel_replacements_and_fusions)rirrrr)rrrrrrrrrrrrrrradjust_generation_fnr)r load_configr)CrXrrrrrrrrrrGrdr@r2rsr r)rfrom_pretraineddeepcopyrMr_rcrTrrr{rrrrmodeling_gguf_pytorch_utilsrrrr\rfusion_mappingrrrr<r]rrrQpreprocess_modelrr!rrEr/r_load_pretrained_model_finalize_model_loadingevalrrerrrrer1rpostprocess_model load_adapterto_dict)ErOrrrrrrrrrrrr model_argsrrrrr from_pipelinerrrrrrrr]rrrrrrgrrrrrrrWrr]rrJr\rdownload_kwargs_with_commit_adapter_model_pathr config_pathr) model_kwargsrrrrr dummy_modelrrallow_all_kernels_contextmodel_init_contextrirweight_conversionsr loading_infodisk_offload_indexsE rrzPreTrainedModel.from_pretrainedys jZZ d33 **Y--ZZ d33 $jj)>FF #3T::  **\599 7D))jj55 ZZ d33 ZZ d33 $4d;; **%6>>$jj)>EEJJ{B// jj66 **Y-- **%5r::@bFFHHzz.)<< "JJ':DAAJJ{D11 **Y--**Y--06 ;OQU0V0Vjj55 "JJ':DAA"JJ':EBBjj66  ?D99 jj55  )goGp ' 'D 4&&AA  "".EEKE =E    $%5 $# #, 0 "  'V&U-&U&U#  !'D'PT]TiD    C |S(I(I$J$J  KKc     '"5/Lkj000 ,J W  )@)B)B hii i  !NM` ) 'N N N N J:N .j99 #*Wfgg  $+8J' (&"233 G$*$6&&;GGKK]6**F!L!&.+FFK5@#M2 !F * **0**5J*K*KF ' #v - --3ZZ8P-Q-QF *+; '\:, , ( fj  ' Y%J--&28J>  $ ? ? ? ? ? ? # #C ? ? ?  $ $ Z Z Z Z Z ZFW(^)>)@)@(A(A\^ % !:;; U U88fmTTT U U U U U U U U U U U U U U U!11%GY[lmmv&& / 0 0  C<<<<|<.sZHH$.a#>#>EA%OHHHHHHr)preadr)r'rr$)r%rbackend)rz5Neither a state dict nor checkpoint files were found.)rirrrr)-rrr]r^r~rrrrr-rlevelrkWARNINGrIrMrrr0rrrr3caching_allocator_warmupr-ryr:rrr6rzrer)rrr*r+rwrrWr.rr$r__exit__)rirrrrrris_hqq_or_quarkrexpanded_device_maprmerged_state_dict ckpt_filerr all_pointerfiler5is_mpsrr file_pointerr!s rrz&PreTrainedModel._load_pretrained_models#/ "/ &d2 |7W7d  "  $i 8  !>!@!@HHHEEF;A&U&6&6oOGV#,TT&Za#b#b#bLOOL111)..00II/;/E/Ea/H/H)!,,I I"-$&!!1ppI%,,_YU`Um-n-n-noooop!!XYYY/S,' #5 000 ,L,! - - 4t,,,,///s5II !I rc  |||||j|j|j||j||j d| |t||j |j |tn(#t||j |j |twxYw|S)a$Perform all post processing operations after having loaded some checkpoints into a model, such as moving missing keys from meta device to their expected device, reinitializing missing weights according to proper distributions, tying the weights and logging the loading report.F)rr)rirrrr) mark_tied_weights_as_initialized&_move_missing_keys_from_meta_to_devicemissing_and_mismatchedrrr_initialize_missing_keysrrr#_adjust_missing_and_unexpected_keysr{rrr)rirrs rrz'PreTrainedModel._finalize_model_loadings   2 2< @ @ @  8 83355&'(      * *;+C D D D   <+DX]  ^ ^ ^  5 5l C C C !.9.W(3(K)       !.9.W(3(K)      s BB??%C$czd|D}|d|D}g}|D]x\}}|r |jd}||}n8|r6t |dkrd|j|gn|j}||vr||y|S)Ncnh|]2}d|ddd3S)rHNr<)rrrrs rrxz>PreTrainedModel.retrieve_modules_from_names..$s7FFFsxx #ss 344FFFrch|]_}t|dk|d/d|ddd`S)rr<rHNr )risdigitrrrs rrxz>PreTrainedModel.retrieve_modules_from_names..)sY b b bss3xx!||PSTVPWP_P_PaPa|SXXciinnSbS) * *|||rrHr)unionrLr+rgrrrU) rrm add_prefix remove_prefix module_keysretrieved_modulesrJrC_prefixs rretrieve_modules_from_namesz+PreTrainedModel.retrieve_modules_from_names#sFFFFF "'' b be b b b   ..00 1 1LD& m!3666((11 mCFt99q==sxx!7 >???VZVl{""!((000  r AutoModelct|ts|j}ddlmcm}t ||st|d||_dS)aU Register this class with a given auto class. This should only be used for custom models as the ones in the library are already mapped with an auto class. Args: auto_class (`str` or `type`, *optional*, defaults to `"AutoModel"`): The auto class to register this new model with. rNz is not a valid auto class.) rsrrtransformers.models.automodelsrrr _auto_class)rO auto_class auto_modules rregister_for_auto_classz'PreTrainedModel.register_for_auto_class:sn*c** -#,J666666666{J// I GGGHH H$rc t|rdS| |jjdS|jj|ddddgfvrd}t|jdd}|jj|jj|jjks8|jj|jj|jjks|@||jjkr0|d|jjd|jjd|jjd |d z }t |dSdS) zv Shows a one-time warning if the input_ids appear to contain padding and no attention mask was given. Nr<rzWe strongly recommend passing in an `attention_mask` since your input_ids may be padded. See https://huggingface.co/docs/transformers/troubleshooting#incorrect-output-when-padding-tokens-arent-masked. sep_token_idz5 You may ignore this warning if your `pad_token_id` (z&) is identical to the `bos_token_id` (z), `eos_token_id` (z), or the `sep_token_id` (z ), and your input is not padded.)ryr pad_token_idrM bos_token_id eos_token_idrr)rr.r warn_stringr/s r%warn_if_padding_and_no_attention_maskz5PreTrainedModel.warn_if_padding_and_no_attention_maskPsM i   F  &DK,D,L F ; #yRG'< < <F #4;EEL)5$+:RVZVaVn:n:nK,8T[=UY]YdYq=q=q ,AY1Y1Y]T[Me]].2k.F]][_[f[s]].:]]]    , , , , ,- = rrrzrs rsupports_pp_planz PreTrainedModel.supports_pp_plans< = 4 ? # 4 ; ) 4urct|dr|jSt|dd}| |tvr td|dd}t|S)N_loss_functionriz `loss_type=zZ` was set in the config but it is unrecognized. Using the default loss: `ForCausalLMLoss`. ForCausalLM)rr;rMrJrr)rris r loss_functionzPreTrainedModel.loss_functions 4) * * '& &D+t44    = =   >i>>>   &II&&rc||_dSr)r;rrs rr=zPreTrainedModel.loss_functions#rc$t|ddS)NrFrrs rrzPreTrainedModel.use_kernelsst^U333rrct|rt|ddrdS|r|ddSt|ddrtdd|_dS)NrFTzmDisabling kernels at runtime is a no-op as there is no 'unkernelize' routine; keeping current kernels active.)rrMrrrrr?s rrzPreTrainedModel.use_kernelss ;; 74??  F  &   & & & & &t^U33 ##D!&D   rcb|jjdkrtdddStS)aUBuild the default `CompileConfig` for `get_compiled_call`. Inductor + `reduce-overhead` (the `CompileConfig` defaults) target CUDA. torch_tpu registers its own TorchDynamo backend named `"tpu"`; route `device.type == "tpu"` through it with static shapes to match the common StaticCache + fixed-prefill usage.tpuFrx)rdynamicr)rrur(rs r_default_compile_configz'PreTrainedModel._default_compile_configs3 ; u $ $ INNN Nrcompile_configcnd|jjvr|jS|p|}t |jddp|}t |drt |d||kr5||_tj |jfi| |_ |j S)aReturn a `torch.compile`'d version of `self.__call__`. This is useful to dynamically choose between non-compiled/compiled `forward` during inference, especially to switch between prefill (where we don't want to use compiled version to avoid recomputing the graph with new shapes) and iterative decoding (where we want the speed-ups of compiled version with static shapes).llama4rFN_compiled_call_last_compile_config) r model_type__call__rErMrgrrJrrrrI)rrFdefault_configs rget_compiled_callz!PreTrainedModel.get_compiled_calls t{- - -= 'I4+G+G+I+I !79I4PPrTXTpTpTrTr.// [t3^DDVV(6D %"'- "Z"ZAWAWAYAY"Z"ZD ""rc|jSr)rCrOs ris_backend_compatiblez%PreTrainedModel.is_backend_compatibles ..rrrrrc L|du}tr|sdStrts|s|D],\}}t j|d}t |||-|D],\}} t j| d}t |||-dS||j z D]{}| |}t||d} t j || }|)t||||dd||jt |||||D]>\}} t||d} t j | | }t |||?dS)aMove the missing keys (keys that are part of the model parameters, but were NOT found in the loaded state dicts) back from meta device to their device according to the `device_map` if any, else cpu. Takes care of sharding those missing parameters if `device_mesh` is provided, i.e. we are using TP. All non-persistent buffers are also moved back to the correct device (they are not part of the state_dict, but are not missing either). NrrT)valid_torch_deviceF)r-r.rrr zeros_likerrrur-rvr4 empty_likerHget_local_ranknamed_non_persistent_buffers) rrrrrrrrrr param_device buffer_devices rrz6PreTrainedModel._move_missing_keys_from_meta_to_devices$4/ % ' '    F    %9%;%; L "3355 = = U(u===*4e<<<<#1133 = = V(>>>*4e<<<< F  $"<"A"A"C"CC = =C0055E%j#$OOOL$U<@@@E&+%T5+:T:T:V:VXc +4e<<<<<<>> 9 9KC&z34PPPM$VMBBBE &tS% 8 8 8 8 9 9rc8trYtsK|D]/} ||}d|_ #t $rY,wxYwd|_t r|sddl}td|d D}|j |d5| ddddS#1swxYwYdS| dS)a1 Initialize the missing keys (keys that are part of the model parameters, but were NOT found in the loaded state dicts), according to `_initialize_weights`. Indeed, since the corresponding weights are missing from the state dict, they will not be replaced and need to be initialized correctly (i.e. weight initialization distribution). Also marks non-missing params/buffers with `_is_hf_initialized` and propagates this flag to modules, so that `_initialize_weights` can skip fully-initialized modules entirely. TrNc4h|]}t|dd|SrrFr)rr"s rrxz;PreTrainedModel._initialize_missing_keys../s,tttqGTUWkmrLsLsttttr) keep_varsr) r.rrrvrrAttributeErrorr-rrrrrr)rrrparam_or_bufferrnot_initialized_parameterss rrz(PreTrainedModel._initialize_missing_keyss    +%9%;%; + ((  &*&B&B3&G&GO9=O66%D&*D # & ' ' & &    *.ttDOOdO;;BBDDttt** &223M]^2__ * *''))) * * * * * * * * * * * * * * * * * *  # # % % % % %s#A AAC99C=C=ctd|D}|rdhn t}td|D}|r|d|jp t}|jp t|z}d\t |dkr1tjd d|Dt |dkr1tjd d |Dfd |j D|_ fd |j D|_ d Sd S) zAdjust the `missing_keys` and `unexpected_keys` based on current model's exception rules, to avoid raising unneeded warnings/errors. This is performed in-place. c3FK|]\}}|dVdS)zrotary_emb.inv_freqNr)rrr\s rrzFPreTrainedModel._adjust_missing_and_unexpected_keys..=s4"p"pifVW6??3H#I#I"p"p"p"p"p"przrotary_emb\.inv_freqc3FK|]\}}|dVdS) position_idsNrcrds rrzFPreTrainedModel._adjust_missing_and_unexpected_keys..@s3&m&m96STv~'F'F&m&m&m&m&m&mrz(^|\.)position_ids$NNrrYc3"K|] }d|dV dSrXrNrrpatterns rrzFPreTrainedModel._adjust_missing_and_unexpected_keys..Hs*6g6g7G6g6g6g6g6g6grc3"K|] }d|dV dSrirrjs rrzFPreTrainedModel._adjust_missing_and_unexpected_keys..Js*9m9mg/w///9m9m9m9m9m9mrNc>h|]}||Srrq)rrignore_missing_regexs rrxzFPreTrainedModel._adjust_missing_and_unexpected_keys..Ns5)))h|]}||Srrn)rrignore_unexpected_regexs rrxzFPreTrainedModel._adjust_missing_and_unexpected_keys..Ts5,,,?V?]?]^a?b?b?j?j?j?jr) rwrrerWr6r7rrprrrr) rrhas_inv_freq_buffersadditional_unexpected_patternshas_position_ids_buffersmissing_patternsunexpected_patternsrorqs @@rrz3PreTrainedModel._adjust_missing_and_unexpected_keys6s #"p"p[_[m[m[o[o"p"p"pppFZ)e*A)B)B`c`e`e&#&&m&mX\XjXjXlXl&m&m&m#m#m # G * . ./E F F F?H355#FO#%%Sqq8B55  1 $ $#%:chh6g6gVf6g6g6g.g.g#h#h " # #a ' '&(j9m9mYl9m9m9m1m1m&n&n # +))))+8)))L % # .,,,,+;,,,L ( ( ( / .rctdiD](}|}t|dd)rfd|jD|_dSdS)aEAdds the `_is_hf_initialized` flag on parameters that will be tied, in order to avoid initializing them later as they will be tied (overwritten) anyway. This is very important as most embeddings are tied, and they are huge params (vocabularies are often 256k), so running inits on them is very costly.rurrTcnh|]1}|jvs$t|dd/|2Sr\)rurMrv)rrrs rrxzCPreTrainedModel.mark_tied_weights_as_initialized..msU)))$444t;;C@@BVX]^^5444rN)rMr-rrrr)rr tied_paramrs` rrz0PreTrainedModel.mark_tied_weights_as_initializedXs "$(?DDIIKK 7 7J&&z22E E/ 6 6 6 6     ))))'4)))L % % %  rrc ||S#t$rYnwxYw ||S#t$rYnwxYwt||\}}|dkrTt |jdt jjj t jjj ur| Std|d)ai Return the parameter or buffer given by `target` if it exists, otherwise throw an error. This combines `get_parameter()` and `get_buffer()` in a single handy function. If the target is an `_extra_state` attribute, it will return the extra state provided by the module. Note that it only work if `target` is a leaf of the model. _extra_stateget_extra_staterz2` is neither a parameter, buffer, nor extra state.) rr^ get_bufferrUrMrrrr'r|)rrrCrs rrvz'PreTrainedModel.get_parameter_or_bufferts  %%f-- -    D  ??6** *    D 1$?? . ( ((*;UX_=\]]8?233))++ +[[[[\\\s $$= A  A rrc#K|||D]J\}}d|vr|ddnd|f\}}||}||jvr||fVKdS)zSimilar to `named_buffers`, but only yield non-persistent ones. It is handy as it's not perfectly straightforward to know if they are persistent or not)rrrHrrN)rrr_non_persistent_buffers_set)rrrrJrrbuf_names rrWz,PreTrainedModel.named_non_persistent_bufferss !..wQa.bb # #LD&7:Tkkt{{3222Dz FH''//F6===Fl"""  # #rc|j|k}t|}|jr|r|d|Sr8)trainingrJtrainrr)rr changed_modeoutrs rrzPreTrainedModel.trainsO}, ggmmD!!   '  '   & & & rc,|dS)NF)rrs rrzPreTrainedModel.evalszz%   rc|jduS)zReturn whether the current model is custom code, i.e. code loaded from the hub, or class that we just registered via `register_for_auto_class`.N)r*rPs rrzPreTrainedModel.is_remote_codesd**rc`|p|jd S)zReturn whether the current model is custom code, i.e. either code loaded from the hub, or defined in any user-specific module/session.z transformers.)rrr rPs rrzPreTrainedModel.is_custom_codes.!!##U3>+D+D_+U+U'UUrr)rNFrr8)NNT)NFT)TNFr@NNTT)Trg)r&)TT)rrrrr)rur rr)r*r*r+rr,rr-rr/r1r2rer3r4r5rErr6r7r8r9r:r;r<r=r_r>rr?r@rArBrCrDrrcompilerallow_in_graphr}rFrrHrKrZrrrrsetterrrr classmethodrrr'rrerrrrrrrr^rbrrr$rr.r0r5rErIr\rarfrhno_gradrrrguard_torch_init_functionsrr~rrrrrrrzrrrrrr$r&rrrr1r;r?rPathLikerir r_rCrrr rrrrrr^rrrrrrrzrrr%r-r4r6rr9r=rr(rErNrQrSrrrrrvrrWrrrr __classcell__rs@rrrs(37L$'(4/6666FT"23FFFKsL$#'JS D ''''OS&&&)/cDIo...6:s3x$s)+d2999?CSDI!5!<CCC :>3s8d3i/$6===@D #c(T#Y"6"=DDD*.S#X---CG#SXS %9D%@GGGFJ&C49(%tCy4'7>>> $Hd38n###H,0Hd3c3h'(/// $Hd38n###"&JS#X%%%-2#T111#(D((().---'++++  ^").Dn)<$=).).).#"X).V9d3 #45999X9 /////0-M/-M-M-M-M-M-M^G>G>G>R c3h   X 4S>Xc5c?23X ^!DcNT1!!!^!F ^DeCHo!56=^ : : : :;;; ,49s?,t,,,,@GG[GR;BI;;;X; $T$$$[$XFH-1@@@%-@!) @ !x}!5s!:; @ #(hm(Bq._cq. q.q.q.q.f 1sUYz 1^a 1 1 1 1$$$$3:$$^b$$or$$$$$$$$L"C$J"SV""""0=T===[=8[.d8d83:d8Z^d8d8d8d8L &DcDj,A & & & &:Wt:W:W:W:Wx***X ) ) )C$J@%%S4Z%%%%4.%%%"U]__=?=?_=?~))$))))2U]__$T$&&HH'&_H8p%p%Dp%Tp%p%p%p%dY8Y8C4Y8SWY8Y8Y8Y8v M M M&*)-" 99d 9 $J9 9  9999v%+%+%+%+T&*)-" ]] ]d ] $J ]  ]  ]]]]D&* " AAYAd A A  A  AAAAF@!@@@@@@,ddd d d d c     bl8K)K    2 2 2(.(.(.(.T:>gq$\d,///(n4nnnXn!%"&!$*"#'!%%)||bk)||4K |  | c |t|TzD ||#|||||  U> %&& 4 4 4 4'& 4$ U58?   ----! -2 U58? :+:+:+:+:+x'''''(((((K/3IMbfimbm[>U[T (&(&,:M(&\i(&(&(&(&T ?C.2(-$!&#''+!AE$(VVV - .V'*R['84'?V!3&4t; V $t+ V "& VVVTzD VVVVCS#X!667$>VTkV %!VVV[Vp +/ g0g0 g04Kg0s)d*g0) g0 Cy4' g0  $& ' g0g0g0\g0R$/$?P$ $$$\$L!!!!.%%%[%*#-#-#-J  X X  X  ' 'X '$$$4T444X4 & &$ & & & &     # 0D#####$//[//93i/94K/9- /9 "D( /9  /9/9/9/9b"&T"&d"&"&"&"&H @Q VZ    D8]c]]]]2>B # # #6: # %U\)* + # # # #$!!!+t+++[+ VtVVV[VVVVVrr&z model file)object object_class object_files recursivecdSrrrirs rrfrfsVYVYrcdSrrrs rrfrfsJM#rctri}|r||d<t|fi|St|drt|jS|S)a Recursively unwraps a model from potential containers (as used in distributed training). Args: model (`torch.nn.Module`): The model to unwrap. recursive (`bool`, *optional*, defaults to `False`): Whether to recursively extract all cases of `module.module` from `model` as well as unwrap child sublayers recursively, not just the top-level distributed containers. rrC)rdrrrfrC)rirrs rrfrfsh    ,"+F; *5;;F;;; 5( # #  -- -LrrcH|dkrdStj|jdvS)zCheck if the device is an accelerator. We need to function, as device_map can be "disk" as well, which is not a proper `torch.device`. rOF)rr)rrrurs ris_accelerator_devicers,u|F##(??raccelerator_device_mapctd}|j}tr|jng}|D]\}}||vr ||}|||||} n|} | | z} t|dkr)t||ddu} | | rtndz} ||xx| z cc<|S)z This utility function calculates the total bytes count needed to load the model on each device. This is useful for caching_allocator_warmup as we want to know how much cache we need to pre-allocate. cdS)Nrrrrrrz&get_total_byte_count..s1rNrT) is_weightr) rrur-rrr,rvparam_element_sizer@rrrDr) rirrtotal_byte_counttied_param_namesrrrr dtype_sizeparam_byte_countis_part_of_plans rget_total_byte_countrs3#99--27799@BBJemmG4::<<55 F ) ) ) --j99  #%88 ERRJJ++--J ;;==:5 w< 1) as cudaMalloc is not a bottleneck at all anymore - Loading speed bottleneck is now almost only tensor copy (i.e. changing the dtype) and moving the tensors to the devices. However, we cannot really improve on those aspects obviously, as the data needs to be moved/copied in the end. c\i|])\}}t||tj|*Sr)rrr)rrrs rr#z,caching_allocator_warmup..sG(5vXmntXuXu u|F##rN)rxpugArrg333333ArneuronrF)rrr)r,rrurMrrcurrent_device mem_get_infomemory_reservedmemory_allocatedrr1rr) rir rrrr byte_countaccelerator_modulerfree_device_memorytotal_device_memory unused_memoryr\s rr r s(9L9R9R9T9T "+E3I<XX/4466,g,g ;/ ) )!( !>uEEHZHkHklqHrHrrMM)M99'-7 m+m;;!1$'999!"JJ"/!2JJ Z)<})LMMJJ [E ! !  [H $ $  KJ!O,,EM&`e f f fY,g,grc JeZdZdZeeeeeeeeee d Z de de de ffd Z xZS)AttentionInterfacea_ Dict-like object keeping track of allowed attention functions. You can easily add a new attention function with a call to `register()`. If a model needs to locally overwrite an existing attention function, say `sdpa`, it needs to declare a new instance of this class inside the `modeling_.py`, and declare it on that instance. ) flash_attention_4flash_attention_3rr rzpaged|flash_attention_4zpaged|flash_attention_3zpaged|flash_attention_2z paged|sdpaz paged|eagerrrxrc|tdn|dkr||vrtd|dt||S)zcReturn the requested `attn_implementation`. Also strictly check its validity, and raise if invalid.Na You tried to access the `AttentionInterface` with a `config._attn_implementation` set to `None`. This is expected if you use an Attention Module as a standalone Module. If this is not the case, something went wrong with the dispatch of `config._attn_implementation`rrzP` is not a valid attention implementation registered in the `AttentionInterface`)rrKeyErrorrJr)rrrxrs r get_interfacez AttentionInterface.get_interfacegs  &   K     !G + +0C40O0Oy'yyy ww{{.888r)rrrrr9r;rAr:rBr7_global_mappingrrrrrs@rrrQs5440&#:#:#:24  O 9 9x 9H 9 9 9 9 9 9 9 9 9 9rrr c^eZdZdZedejfdZedejfdZdS)PreTrainedAudioTokenizerBasea Class that additionally defines the behavior of any `audio_tokenizer` to be added. Characteristic for any of them: 1. Encode raw audio into discrete audio codebooks (with x channels) 2. Decode from discrete audio codebooks back to raw audio It is possible that they can decode in different ways given a different representation but they are forced to support 2. nonetheless, e.g. see `DAC`. input_valuescdS)z Encode raw audio retrieved from a respective `FeatureExtractor` into discrete audio codebooks (with x channels) Nr)rrrrs rencodez#PreTrainedAudioTokenizerBase.encoder audio_codescdS)z6Decode from discrete audio codebooks back to raw audioNr)rrrrs rdecodez#PreTrainedAudioTokenizerBase.decoderrN) rrrrrrrrrrrrrrzsw 5<   ^ E%,EEE^EEErrr)rTN)NNNr(rdrr/rLrzrrpr rabcrrcollections.abcrr contextlibr dataclassesrr r r itertoolsr threadingr typingrrrrrzipfilerrhuggingface_hubrr packagingr safetensorsrsafetensors.torchrr*rryrrtorch.distributionsrtorch.utils.checkpointrrrrconfiguration_utilsr conversion_mappingr!core_model_loadingr"r#r$r%rr&dynamic_module_utilsr' generationr(r) integrationsr*r+r,r-r.integrations.accelerater/r0r1r2r3r4r5rr6integrations.eager_pagedr7integrations.finegrained_fp8r8integrations.flash_attentionr9integrations.flash_pagedr:integrations.flex_attentionr;rr<r=r>integrations.moer?integrations.peftr@integrations.sdpa_attentionrAintegrations.sdpa_pagedrBintegrations.tensor_parallelrCrDrErFrGrHrIloss.loss_utilsrJmodeling_flash_attention_utilsrKrLrMrNmodeling_rope_utilsrOmonkey_patchingrPrQ pytorch_utilsrR quantizersrSquantizers.autorTquantizers.quantizers_utilsrUsafetensors_conversionrVutilsrWrXrYrZr[r\r]r^r_r`rarbrcrdrerfrgrhrirjrk utils.genericrlrmrn utils.hubrorprqrrutils.import_utilsrsrtrurvrwrxryutils.loading_reportrzr{utils.output_capturingr|r}utils.quantization_configr~accelerate.hooksraccelerate.utilsrkernels.layer.moder_typingr is_availabler!smdistributed.modelparallel.torch modelparallelrnsmdistributed.modelparallelr SMP_VERSIONrrm get_loggerrrrrupperrrrrrrrrrrrrrrrrrruint8int8int16uint16rrint32uint32rfloat64int64uint64 float8_e4m3fn float8_e5m2r0rrrrr:rBr'rrRrerrbrhrrrrurrrrrrCrrLrfrrr rr rrrrrr sC  ######........%%%%%%(((((((($$$$$$$$HHHHHHHHHHHHHH OOOOOOOO!!!!!!666666999999++++++------$$$$$$111111<<<<<< +*****44444477777777vvvvvvvvvvvvvvFEEEEECCCCCCCCCCCCAAAAAA======??????QQQQQQQQQQ333333222222??????AAAAAA*))))) 544444BBBBBBBB,,,,,,######------======333333.jiiiiiiiiiddddddddddddKJJJJJJJHHHHHHHH999999=333333<<<<<<('''''''''''' %0==??&333333333FFFFFF - k : :mgmF>S>S S %  H % %z~~nc2288:: JNN#6<<BBDD%g&CK\]]]  $--------64.3.... ```###00U[0C$J0000.    1 1 1 J + * ; < = N ; < = = ; <" &-$6(- $ /k/k2;&/k $/k/k+ /k  #u|  /k/k/k/kdU\c")S ,DSN,S%,=N8O,TYZ^_bcf_gZhjnorjsZsTt,,,,>% #c(^%)-c5<.?)@% 4C>4C> )*%%%%*MS%,&'M0AM #u| MMMM`(&7(S(RWR^((((sS4Z326-1"V.V.#&#4t#;V. 4ZV.TzV.D[ V. t V.  V.%($JV.$d*V.t V. 49t TD[ ()V.V.V.V.@ (,PP  t #d *P3i$&P PTk P t P  P$P U[ ()PPPPfjjjjjjjjZj*j*j*j*j*j*j*j*Z;V;V;V;V;Vbi!57GYi;V;V;VDx(i(CDD&2*9*E*M*T*T[|+U++O'  YYYDY_YYY Y MM MdMryMMM M dry2@#)el":@t@@@@^b 48HSVZHZDIgOIg$Ig^ilp^pIgIgIgIgX"9"9"9"9"9)"9"9"9L/A.@.B.B+BBBEEEEE?EEEEEr