tjQpdZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlmZmZmZddlmZddlmZddlmZmZddlmZddlZddlZddlZddlm Z!dd l"m#Z#m$Z$dd l%m&Z&dd lm'Z'dd l(m)Z)m*Z*m+Z+m,Z,m-Z-dd l.m/Z/ddl0m1Z1ddl2m3Z3m4Z4m5Z5ddl6m7Z7m8Z8ddl9m:Z:m;Z;ddlm?Z?ddl@mAZAmBZBddlCmDZDddlEmFZFmGZGmHZHmIZImJZJddlKmLZLddlMmNZNmOZOddlPmQZQddlRmSZSmTZTmUZUddlVmWZWmXZXddlYmZZZddl[m\Z\m]Z]ddl^m_Z_m`Z`ddlambZbmcZcdd ldmeZedd!lfmgZgdd"lhmiZimjZjmkZkmlZlmmZmmnZnmoZompZpdd#lqmrZrmsZsmtZtmuZudd$lvmwZwmxZxmyZymzZzm{Z{m|Z|m}Z}m~Z~mZmZmZmZmZmZmZdd%lmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZdd&lmZmZmZdd'lmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZdd(lmZdd)lmZejgZemZerdd*lmZeZerddlZeærLddlmcmZddlmcmZddlmZdd lm/Ze&jeڦe&jekZer ddlm cmZnd+ZerddlmcmZdd,lvmZmZmZerdd-lmZer:dd.lmZmZdd/lmZdd0lmZmZmZmZmZmZmZmZmZdd1lmZeIrdd2lmZerddlZejeZd3Zd4Zd5Zd6Zd7Zd8Zd9Zed:;Gd<d=ZdS)>uc The Trainer class, to easily train a 🤗 Transformers from scratch or finetune it on a new task. N)CallableIteratorMappingpartial)Path) TYPE_CHECKINGAny)#get_reporting_integration_callbacks) CommitInfo ModelCard)version)nn) DataLoaderDatasetIterableDataset RandomSamplerSequentialSampler) __version__)PreTrainedConfig) DataCollatorDataCollatorWithPaddingdefault_data_collator) DebugOptionDebugUnderflowOverflow)get_fsdp_ckpt_kwargsupdate_fsdp_plugin_peft)SequenceFeatureExtractor)FeatureExtractionMixin)"ALL_HYPERPARAMETER_SEARCH_BACKENDSdefault_hp_search_backend)BaseImageProcessor)deepspeed_initdeepspeed_load_checkpointdeepspeed_sp_compute_lossis_deepspeed_availablepropagate_args_to_deepspeed)apply_liger_kernel)activate_neftunedeactivate_neftune)MIN_PEFT_VERSION)save_tpu_checkpointtpu_spmd_dataloaderwrap_model_xla_fsdp) LOSS_MAPPINGForCausalLMLoss)TrainingSummary)PreTrainedModel unwrap_model)!MODEL_FOR_CAUSAL_LM_MAPPING_NAMESMODEL_MAPPING_NAMES)GreedyLR get_scheduler)ProcessorMixin)PreTrainedTokenizerBase)CallbackHandlerDefaultFlowCallbackExportableStatePrinterCallbackProgressCallbackTrainerCallbackTrainerControl TrainerState)_OPTIMIZER_HANDLERSOptimizerContext_parse_optim_argsis_optimizer_factory)EvalLoopContainerIterableDatasetShard LabelSmootherLengthGroupedSamplerdistributed_broadcast_scalarsfind_batch_sizeget_model_param_countget_parameter_namesis_attention_mask_causal nested_detach nested_gatherreissue_pt_warningsremove_dummy_checkpoint safe_globalsset_rng_state_for_device)PREFIX_CHECKPOINT_DIRBestRunEvalLoopOutputEvalPredictionHPSearchBackend HubStrategyPredictionOutputRemoveColumnsCollator SaveStrategyTrainerMemoryTracker TrainOutput_is_peft_modelalign_special_tokens#compare_trainer_and_checkpoint_argsdefault_compute_objectivedenumpify_detensorizeenable_full_determinismfind_executable_batch_sizeget_last_checkpoint has_lengthload_sharded_checkpointnumber_of_argumentsrotate_checkpoints seed_workerset_seedsort_checkpoints speed_metricssuppress_progress_barsunwrap_peft_model"validate_quantization_for_training)OptimizerNames ParallelModeTrainingArguments)ADAPTER_CONFIG_NAMEADAPTER_SAFE_WEIGHTS_NAMEADAPTER_WEIGHTS_NAME CONFIG_NAMEGENERATION_CONFIG_NAMESAFE_WEIGHTS_INDEX_NAMESAFE_WEIGHTS_NAMEWEIGHTS_INDEX_NAME WEIGHTS_NAMEXLA_FSDPV2_MIN_VERSIONPushInProgresscan_return_losscheck_torch_load_is_safe find_labelshf_apiis_accelerate_availableis_datasets_availableis_in_notebookis_peft_availableis_sagemaker_dp_enabledis_sagemaker_mp_enabledis_torch_hpu_availableis_torch_mlu_availableis_torch_musa_availableis_torch_npu_availableis_torch_xla_availablelogging)requires)QuantizationMethod)NotebookProgressCallbackF)smp_forward_backwardsmp_forward_onlysmp_nested_concat) PeftModel) Acceleratorskip_first_batches)AcceleratorState) DataLoaderConfigurationDistributedDataParallelKwargsDistributedTypeGradientAccumulationPluginload_fsdp_modelload_fsdp_optimizerrelease_memorysave_fsdp_modelsave_fsdp_optimizer)clear_device_cache)DeepSpeedSchedulerWrapperztraining_args.binztrainer_state.jsonz optimizer.ptz scaler.ptz optimizer.binz scheduler.ptpytorch_model_fsdp)torch accelerate)backendsc4eZdZdZddlmZmZmZmZm Z m Z m Z  dde e jzdzdedzdedzd d d d d eezezezdzdede fdzdedzdeegefdzdeedzdeejjdzejjjdzfdee ejjee!e"ffdzdeej#ej#gej#fdzfdZ$ddZ%dee!e"ffdZ&ddZ'de(fdZ)dd e!e*zdzde(fdZ+de*de(fdZ,de(de-fd Z. dd"e*d#e!d$e-d%ee*gej/j0j1fdzd&e2d'e!dzde(fd(Z3dd e*dzdej/j0j1dzfd)Z4d e*dej/j0j1dzfd*Z5dd+Z6 dd"d,d#e!dzdd,fd-Z7dded#e!dzdefd.Z8d/e-ddfd0Z9ddejjfd1Z: dd/e-d2ejjdzdejjj;fd3Zde jdee!fd5Z?de@fd6ZA dd7e!e2zdzd8d9d:ee!dzdeBfd;ZC dd$e-dzdedzd7e!dzd8d9d:ee!dzdeBf d<ZDdee-e-ffd=ZEd>ZFd?ZGd@ZH dde jdAee!ej#e"zfdBej#e-zdzdej#fdCZI dde jdAee!ej#e"zfdDe2dBej#e-zdzdej#eej#e"fzf dEZJdeKjLfdFZMddHe2dzdeKjNfdIZO ddJej#dKej#e@zdzde jd8d9dLe@d:ee!dzdMe@dNe@dzddfdOZPdPeQdQe-dRejRdeeej#e-zdzffdSZSdTedRejRdej#e-zdzfdUZTdVej#e"zdej#e"zfdWZUdAee!ej#e"zfdee!ej#e"zffdXZVde jdAee!ej#e"zfdeeee!ej#e"zfffdYZWdede(dee-e-e-e-e-e-dze-ffdZZXdede-fd[ZYde-fd\ZZde-fd]Z[de-fd^Z\dde jd_e2de(dzde jfd`Z]daZ^dbZ_dcZ`dddZa dd e*ee!e*fzdzdfee!dzdge!dee!e@ffdhZb dde(d#e!die2dzdfee!dzdge!decf djZd dde*dfee!dzdge!deefdlZf dde jdAee!ej#e"zfdie2dfee!dzdeej#dzej#dzej#dzff dmZg dd8d9d:ee!dzdne2dee!e@ffdoZhd8d9de!fdpZide jd8d9ddfdqZjdree!e@fd8d9de2fdsZkdte!ddfduZldte!ddfdvZmdte!ddfdwZndd7e!de jdzddfdxZoddyZpdze!dzddfd{Zqdze!dzddfd|Zrdze!dzddfd}Zsdd~Ztde"ddfdZuddte!dzde2ddfdZvddte!dzdedzddfdZwddee!e@fdMe@dzddfdZxddZydAee!ej#e"zfde-fdZzdde!dzddfdZ{ dde!dzde!dzde!ee!zdzde!dzde!dzde!ee!zdzde!ee!zdzd"e!ee!zdzde!ee!zdzddfdZ| dde!dzde2de!dzde!dzde}f dZ~de!ddfdZddZ ddedgee!e@ffdzdeee!e@fge@fdzde-de!ee!zde!ezdzdedge!fdzdeeezfdZdd8d9de jfdZddZd8d9de-dree!e@fddfdZde!ddfdZde eezddfdZde eezdedzfdZde eezddfdZde2fdZde2fdZde jdRejRddfdZdS)Traineru Trainer is a simple but feature-complete training and eval loop for PyTorch, optimized for 🤗 Transformers. Args: model ([`PreTrainedModel`] or `torch.nn.Module`, *optional*): The model to train, evaluate or use for predictions. If not provided, a `model_init` must be passed. [`Trainer`] is optimized to work with the [`PreTrainedModel`] provided by the library. You can still use your own models defined as `torch.nn.Module` as long as they work the same way as the 🤗 Transformers models. args ([`TrainingArguments`], *optional*): The arguments to tweak for training. Will default to a basic instance of [`TrainingArguments`] with the `output_dir` set to a directory named *tmp_trainer* in the current directory if not provided. data_collator (`DataCollator`, *optional*): The function to use to form a batch from a list of elements of `train_dataset` or `eval_dataset`. Will default to [`default_data_collator`] if no `processing_class` is provided, an instance of [`DataCollatorWithPadding`] otherwise if the processing_class is a feature extractor or tokenizer. train_dataset (`torch.utils.data.Dataset` | `torch.utils.data.IterableDataset` | `datasets.Dataset`, *optional*): The dataset to use for training. If it is a [`~datasets.Dataset`], columns not accepted by the `model.forward()` method are automatically removed. Note that if it's a `torch.utils.data.IterableDataset` with some randomization and you are training in a distributed fashion, your iterable dataset should either use a internal attribute `generator` that is a `torch.Generator` for the randomization that must be identical on all processes (and the Trainer will manually set the seed of this `generator` at each epoch) or have a `set_epoch()` method that internally sets the seed of the RNGs used. If the dataset does not implement `__len__` (e.g. a streaming `IterableDataset`), `max_steps` must be set, since the total number of training steps cannot be inferred. eval_dataset (`torch.utils.data.Dataset` | dict[str, `torch.utils.data.Dataset`] | `datasets.Dataset`, *optional*): The dataset to use for evaluation. If it is a [`~datasets.Dataset`], columns not accepted by the `model.forward()` method are automatically removed. If it is a dictionary, it will evaluate on each dataset prepending the dictionary key to the metric name. processing_class (`PreTrainedTokenizerBase` or `BaseImageProcessor` or `FeatureExtractionMixin` or `ProcessorMixin`, *optional*): Processing class used to process the data. If provided, will be used to automatically process the inputs for the model, and it will be saved along the model to make it easier to rerun an interrupted training or reuse the fine-tuned model. model_init (`Callable[[], PreTrainedModel]`, *optional*): A function that instantiates the model to be used. If provided, each call to [`~Trainer.train`] will start from a new instance of the model as given by this function. The function may have zero argument, or a single one containing the optuna/Ray Tune trial object, to be able to choose different architectures according to hyperparameters (such as layer count, sizes of inner layers, dropout probabilities etc). compute_loss_func (`Callable`, *optional*): A function that accepts the raw model outputs, labels, and the number of items in the entire accumulated batch (batch_size * gradient_accumulation_steps) and returns the loss. For example, see the default [loss function](https://github.com/huggingface/transformers/blob/052e652d6d53c2b26ffde87e039b723949a53493/src/transformers/trainer.py#L3618) used by [`Trainer`]. compute_metrics (`Callable[[EvalPrediction], Dict]`, *optional*): The function that will be used to compute metrics at evaluation. Must take a [`EvalPrediction`] and return a dictionary string to metric values. *Note* When passing TrainingArgs with `batch_eval_metrics` set to `True`, your compute_metrics function must take a boolean `compute_result` argument. This will be triggered after the last eval batch to signal that the function needs to calculate and return the global summary statistics rather than accumulating the batch-level statistics callbacks (List of [`TrainerCallback`], *optional*): A list of callbacks to customize the training loop. Will add those to the list of default callbacks detailed in [here](callback). If you want to remove one of the default callbacks used, use the [`Trainer.remove_callback`] method. optimizers (`tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR]`, *optional*, defaults to `(None, None)`): A tuple containing the optimizer and the scheduler to use. Will default to an instance of [`AdamW`] on your model and a scheduler given by [`get_linear_schedule_with_warmup`] controlled by `args`. optimizer_cls_and_kwargs (`tuple[Type[torch.optim.Optimizer], dict[str, Any]]`, *optional*): A tuple containing the optimizer class and keyword arguments to use. Overrides `optim` and `optim_args` in `args`. Incompatible with the `optimizers` argument. Unlike `optimizers`, this argument avoids the need to place model parameters on the correct devices before initializing the Trainer. preprocess_logits_for_metrics (`Callable[[torch.Tensor, torch.Tensor], torch.Tensor]`, *optional*): A function that preprocess the logits right before caching them at each evaluation step. Must take two tensors, the logits and the labels, and return the logits once processed as desired. The modifications made by this function will be reflected in the predictions received by `compute_metrics`. Note that the labels (second parameter) will be `None` if the dataset does not have them. Important attributes: - **model** -- Always points to the core model. If using a transformers model, it will be a [`PreTrainedModel`] subclass. - **model_wrapped** -- Always points to the most external model in case one or more other modules wrap the original model. This is the model that should be used for the forward pass. For example, under `DeepSpeed`, the inner model is wrapped in `DeepSpeed` and then again in `torch.nn.DistributedDataParallel`. If the inner model hasn't been wrapped, then `self.model_wrapped` is the same as `self.model`. - **is_model_parallel** -- Whether or not a model has been switched to a model parallel mode (different from data parallelism, this means some of the model layers are split on different GPUs). - **place_model_on_device** -- Whether or not to automatically place the model on the device. Defaults to `True` unless model parallel, DeepSpeed, FSDP, full fp16/bf16 eval, or SageMaker MP is active. Can be overridden by subclassing `TrainingArguments` and overriding the `place_model_on_device` property. - **is_in_train** -- Whether or not a model is currently running `train` (e.g. when `evaluate` is called while in `train`) r )get_learning_ratesget_num_trainable_parametersget_optimizer_group log_metricsmetrics_format save_metrics save_stateNNNmodelargs data_collator train_datasetz3Dataset | IterableDataset | datasets.Dataset | None eval_datasetz6Dataset | dict[str, Dataset] | datasets.Dataset | Noneprocessing_class model_init.compute_loss_funccompute_metrics callbacks optimizersoptimizer_cls_and_kwargspreprocess_logits_for_metricsc T|0d}td|dt|}||_|jjrt |jjnt|jjd|_||_ | t|jj |_ |j |}t!j||j|-|||_|}n't+d|t-d||_|jjt2vrt-d|jjdt5|d |_t9|d dd t;|jD}tA|d krd |_n@tA|d kr-|jj!tEj!|dk|_|j#o|j$%dd |_&|j#rA|j'rt-d|j&s$|j(tRj*krt-d|j+ |j+|_+nG|js1|j's*|j,s#|j-s|j&s|j.st_rd |_+nd |_+|j+r:t9|ddt`j1kr|2||j!|jr d |j_3||_4||_ tk|j67|}tq|dr |j9|_:nNtwj<|j=j>}td|D|_:t9|j6dd}||j@dkr|jArd |_:tk|j }t|j}|jjC|n |jjC|_Ct|j|_Dtj%t9|ddtu|_G|jjHdkrat9|j jIdddkr"tjKdtd|_Mn't|jjH|_Mnd|_M|+t|ttfrt|nt}||n||_T||_U||_V||_W|jX|_X||_Y| |_Z| |_[| \|_\|_]| |_^|_tt|jjbz}|jjcr+d dldme}|}||gz}|f|| |n|| z} t| |j |jW|j\|j]|_h|i|jjjrtntd|_m|jjnr|o|jjpr tjr|jjsd t|_ut|w|xd|jhjy|jugzD|_zd |_{d|_|d|_}d|_~d |_d|_|j|_d |_|jh|j|jz|ju|_ut9|j d d|jj|j jI_|j#o|j$%d!d |_|jrytst-d"tj}tjtjtjt||d fd#$|j&o|j |_|j dS)%N tmp_trainerz1No `TrainingArguments` passed, using `output_dir=z`.) output_dirz<`Trainer` requires either a `model` or `model_init` argumentzK`Trainer` requires either a `model` or `model_init` argument, but not both.zThe model you have picked (a) cannot be used as is for training: it only computes hidden states and does not accept any labels. You should choose a model with a head suitable for your task like any of the `AutoModelForXxx` listed at https://huggingface.co/docs/transformers/model_doc/autoF hf_device_mapcg|]}|dv| S))cpudisk).0devices R/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/trainer.py z$Trainer.__init__..s$ooo&QW_nQnQnvQnQnQnr TrxlazZUsing --fsdp xxx together with --deepspeed is not possible, deactivate one of those flags.z.Using fsdp only works in distributed training.quantization_methodaccepts_loss_kwargsc3JK|]}|jtjjkVdSN)kindinspect Parameter VAR_KEYWORD)rks r z#Trainer.__init__..s@11<='+77111111rparallelism_config deepspeed loss_type problem_typemulti_label_classificationzsLabel smoothing is not compatible with multi-label classification. Disabling label smoothing for this training run.)epsilon)JITCheckpointCallbackexist_okc<g|]}t|t|Sr isinstancer=rcbs rrz$Trainer.__init__..M:   Q[\^`oQpQp    r)is_local_process_zerois_world_process_zerostateful_callbacksconfig xla_fsdp_v2z*FSDPv2 requires `torch_xla` 2.2 or higher.)fsdptensor) axis_names)loggerinforvrfull_determinismrfseedrnrr"create_accelerator_and_postprocessr_skip_memory_metrics_memory_trackerstartget_process_log_levelr set_verbosity_setup_devicesrcall_model_init RuntimeError ValueError __class____name__r6rsis_model_parallelgetattrsetrvalueslenrrr fsdp_configgetis_fsdp_xla_enabledis_deepspeed_enabled parallel_moderu DISTRIBUTEDplace_model_on_devicefp16_full_evalbf16_full_evalis_fsdp_enabledrrBITS_AND_BYTES_move_model_to_device_n_gpu model_wrappedrr acceleratorr4hasattrrmodel_accepts_loss_kwargsr signatureforward parametersany sp_backend sp_enabledr label_namesrr0r1_loss_shifts_labelslabel_smoothing_factorrwarningswarn UserWarninglabel_smootherrIrr:rrrrrrrneftune_noise_alpharrr optimizer lr_schedulerr_validate_argsDEFAULT_CALLBACKSr report_toenable_jit_checkpointtrainer_jit_checkpointr set_trainerr;callback_handler add_callback disable_tqdmr>DEFAULT_PROGRESS_CALLBACK hub_model_id push_to_hub init_hf_repo should_saveosmakedirsrrAcontrolrBrrrstate is_in_trainhp_namehp_search_backend current_flos_loggers_initialized_signature_columnstrain_batch_size_train_batch_size_created_lr_scheduler on_init_end use_cacheis_fsdp_xla_v2_enabledIS_XLA_FSDPV2_POST_2_2xrglobal_runtime_device_countxsset_global_meshMeshnparrayrangeis_fsdp_xla_v1_enabledstop_and_update_metrics)selfrrrrrrrrrrrrrr log_leveldevicesunwrapped_modelforward_paramspcmodel_to_inspectdefault_label_namesdefault_collatordefault_callbacksr jit_callback num_devicess r__init__zTrainer.__init__osD <&J KKZJZZZ [ [ [$ ;;;D 3793Mk ///S[\`\e\jSkSk   //1113DI4QRR ""$$$..00 i(((  =%",,,.."#abbb% !nooo(DO ? #': : :Jeo.FJJJ  +5111"' 5/4 0 0 <ooC0C0J0J0L0L,M,MoooG7||a)-&&W"")-)9U\'RS*=U=U)U&#'9#S1A1E1EeU1S1S 9 S(  p+ S0BlF^0^0^ !QRRR  % 1)-)CD & &  " .( .# .(,': .' . # . '(( .*/D & &)-D &  & ;4d;;?Q?```  & &udk : : :  ! ! DI # ,D,<,I,I%,P,PQQ ?$9 : : -<-PD * *$./FGGRN-011AOAVAVAXAX111..D * T%';T B B >bm{::r}:-2D *,TZ88)*:*DEE26)2G2O..UYU^Uj./?/IJJ$0#3G">"&"<"<">">  !2< ~M       ! !%$)!"&!%!6%*",88DJPTP\]]  4:x . . :*.)*=DJ  '&*i&^D4D4H4HX]4^4^#  & w) O !MNNN8::K  rwrxk0B0B'C'CkSTEUbtuuu v v v&*&>&btGbCb# 4466666rreturnc|j}tr|jrtd|jt jjjkrkt dt jjjd|jdt jjjt jjj|_|j r6|j /dtj |j jvrtd|j*|jdkr|jtd |jd |jt$jks|jr|jtd |j|jt1d |j|j|jt1d t7r||ju|jD] }|j}|jjD]0}t?|ddkr|ddj}n1||krtd|j s|j!r|j|jt1dtE|j#s2tEtI|j#ddrtKd|j&dkr%|j'dkrt(d|j).tU|j)s|j&dkrtd|j)NtW|j)tXj-j.j/r't(d|j0ddSdSdS)zJValidate constructor arguments and fail fast on incompatible combinations.zOSageMaker Model Parallelism does not support BF16 yet. Please use FP16 instead z(FP16 provided in SM_HP_MP_PARAMETERS is z+, but FP16 provided in trainer argument is z , setting to Ncompute_resultzWhen using `batch_eval_metrics`, your `compute_metrics` function must take a `compute_result` boolean argument which will be triggered after the last batch of the eval set to signal that the summary statistics should be returned by the function.noz%You have set `args.eval_strategy` to zx but you didn't pass an `eval_dataset` to `Trainer`. Either set `args.eval_strategy` to `no` or pass an `eval_dataset`. z`args.metric_for_best_model` must be provided when using 'best' save_strategy or if `args.load_best_model_at_end` is set to `True`.zSPassing both `optimizers` and `optimizer_cls_and_kwargs` arguments is incompatible.zPassing a `model_init` is incompatible with providing the `optimizers` argument. You should subclass `Trainer` and override the `create_optimizer_and_scheduler` method.paramsra[The model and the optimizer parameters are not on the same device, which probably means you created an optimizer around your model **before** putting on the device and passing it to the `Trainer`. Make sure the lines `import torch_xla.core.xla_model as xm` and `model.to(xm.xla_device())` is performed before the optimizer creation in your script.zPassing `optimizers` is not allowed if PyTorch FSDP is enabled. You should subclass `Trainer` and override the `create_optimizer_and_scheduler` method. collate_batchzRThe `data_collator` should be a simple callable (function, class with `__call__`).zHmax_steps is given, it will override any value given in num_train_epochszThe train_dataset does not implement __len__, so the number of training steps cannot be inferred; max_steps has to be specified. The total number of steps must be known in advance to bound the training loop and to configure the learning rate scheduler.zThe `train_sampling_strategy='z'` option is ignored when using an `IterableDataset`. Samplers cannot be used with IterableDataset as they require indexed access to the dataset.)1rrbf16rfp16smpr8cfgrwarningbatch_eval_metricsrrrr eval_strategyr save_strategyr^BESTload_best_model_at_endmetric_for_best_modelrr%rrr&rrr param_groupsrrrcallablerr TypeError max_stepsnum_train_epochsrrrirrutilsdatartrain_sampling_strategy)rPrparam model_device param_groupoptimizer_devices rr'zTrainer._validate_argsosy # $ $ /y t !rsssyCIM...7sy}?Q77@D 77"%)-"477  IM.   " t';'Gw'89M'N'N'YYY N   )d.@D.H.HTM^MfE8JEEE   !2 2 2d6Q 2)1 Z  ( 49Stuu u ? &DN,F$J[Jgj  " # # (B..00  $| #~:   {8,--11'28'A  $"7!";"; KKb c c c   )*T=O2P2P )UYUcghUhUhN    )j9KU[M]Mm.n.n ) KKn1Mnnn      * ) ) )rc |jj|jjd}|||jj |jj}n6t |jtr|jjp |jj }nd}d|i}|jj |jj |d<|jj |jj |d<|jj |jj |d<tdi|g|d<|jj3d }t|st!d |d |jj|d <t#|jd d|jjdkr|jjJtd r,|jjddlm}||jj|d <nAt+d|d j|jjkr|jj|d _tdr,ddlm}||jj|jj}||d<|S)z>Helper method to build accelerator-specific keyword arguments.)mixed_precisiondeepspeed_pluginNTfind_unused_parameters bucket_cap_mbbroadcast_buffers static_graphkwargs_handlersz1.12.0z'ParallelismConfig requires accelerate>=z1). Please upgrade accelerate to use this feature.rtp_sizer r)ParallelismConfig)rz5Requires accelerate>1.12.0 to use Tensor Parallelism.z1.2.0)TorchDynamoPlugin)backendmode dynamo_pluginr)rr{r|updateddp_find_unused_parametersrrr3is_gradient_checkpointinggradient_checkpointingddp_bucket_cap_mbddp_broadcast_buffersddp_static_graphrrr ImportErrorrrrrraccelerate.utilsrtorch_compile_backendtorch_compile_mode) rPkwargsr find_unused ddp_kwargsmin_accelerate_versionrrrs r_build_accelerator_argszTrainer._build_accelerator_argssg $y8 $ :   F 9 / ;)>KK  O 4 4  $zCgtyGghKKK. < 9 & 2*.)*EJ ' 9 * 6.2i.MJ* + 9 % 1)-)CJ~ &#@#N#N:#N#N"O  9 ' 3%- "*+ABB !H>THHH*.)ED% & 4:y$ / / ; @RUV@V@Vy+3*844^y3;@@@@@@5F5FtzOa5b5b5b12$%\]]]*+3tz7III59Z5G)*2 "7 + + 2 : : : : : :-- 7di>ZM%2D ! rcr i}|jjj|jjj}d|vr1|jjdkrt d|d|j_d|d<t di|}|jj gd}tdi fd|D}|jj|_ d}|r&|jj st d||_ d d}|jjd d lm}|di|jj}|||| }t%di||_|jj|_d t-j|jjvr*t3j|j|jj|_t9|jjdddu|_t9|jjdddu|_|jr1|jjj j!r|jj"rt d|jr0t9|jddtG|j|j|jj$r7|js|jr)|jj%r|jrdnd} t | d|jr5|jjj&j'dkr|jj(rt d|jj$r;|jr6dtS|jjj j*vrt ddSdSdS)zOCreate the accelerator and perform post-creation setup (FSDP, DeepSpeed, etc.).N num_stepsr zThe `AcceleratorConfig`'s `num_steps` is set but `gradient_accumulation_steps` is greater than 1 in the passed `TrainingArguments`If using the passed `AcceleratorConfig` is desired, do not set the `TrainingArguments` `gradient_accumulation_steps`.) split_batchesdispatch_batches even_batchesuse_seedable_samplerc<i|]}||Sr)pop)rrvaccelerator_configs r z>Trainer.create_accelerator_and_postprocess..s*SSSu(,,U33SSSr non_blockingzx`non_blocking` is enabled but `dataloader_pin_memory` is not. For the best performance, it's recommended to enable both.gradient_accumulation_kwargsr)FullyShardedDataParallelPlugin)dataloader_config fsdp_plugingradient_accumulation_pluginuse_gather_object)rr|rzThe activation_checkpointing in FSDP config and the gradient_checkpointing in training arg can't be set to True simultaneously. Please use FSDP's activation_checkpointing logic when using FSDP.hf_deepspeed_config DeepSpeedFSDPzJ can't be used with `save_only_model` along with `load_best_model_at_end`.zo`auto_find_batch_size` isn't supported yet with DeepSpeed Zero-3. Please consider using Zero-2, Zero-1, or FSDPSHARDED_STATE_DICTzWsave_only_model option is not compatible with FSDP state dict type 'SHARDED_STATE_DICT'r)+rrrgradient_accumulation_stepsrrto_dictr data_seedrdataloader_pin_memoryrrgrfsdp_plugin_argsrrrrrgather_for_metricsgather_functionrrr functoolsreval_use_gather_objectrr8r rractivation_checkpointingrr(save_only_modelrlr| zero_stageauto_find_batch_sizestrstate_dict_type) rPgrad_acc_kwargsrdataloader_paramsrrrrrwrapperrs @rrz*Trainer.create_accelerator_and_postprocesss 9 ' D P"i:WO / ) )y4q88 L 9H 8T 5() $ (B'T'TO'T'T$!Y9AACCjii3  SSSSARSSS  '+i&9#)--n==   ?  NNK   *6&=>>> 9 % 1 G G G G G G88VV49;UVVK++/#)E,  '....#/B '"3D4H"I"I"T T T#,#4$ 8X$$$D %,D,<,BDVX\$]$]ei$i!&t'7'=}dSS[__   %1J tyOo  '  $ EJkkFGssstt t  %  &7BaGG .HB  I % x$ x%D,<,B,N,^(_(___vww w  x x x x`_rc|jtd||jd|j|jdS)a@ Returns the training [`~torch.utils.data.DataLoader`]. Will use no sampler if `train_dataset` does not implement `__len__`, a random sampler (adapted to distributed training if necessary) otherwise. Subclass and override this method if you want to inject some custom behavior. Nz+Trainer: training requires a train_dataset.TrainingT)dataset description batch_size sampler_fn is_training)rr_get_dataloaderr@_get_train_samplerrPs rget_train_dataloaderzTrainer.get_train_dataloaderksQ   %JKK K##&"-. $   rct||jtdt|tr|nd}t |dr"||jvr|jjr |j|St|tr |j|n ||n|j}||d|jj |j |S)a Returns the evaluation [`~torch.utils.data.DataLoader`]. Subclass and override this method if you want to inject some custom behavior. Args: eval_dataset (`str` or `torch.utils.data.Dataset`, *optional*): If a `str`, will use `self.eval_dataset[eval_dataset]` as the evaluation dataset. If a `Dataset`, will override `self.eval_dataset`. If it is a [`~datasets.Dataset`], columns not accepted by the `model.forward()` method are automatically removed. Nz-Trainer: evaluation requires an eval_dataset.eval_eval_dataloaders Evaluation)rrrrdataloader_key) rrrrrrrdataloader_persistent_workersreval_batch_size_get_eval_sampler)rPrrs rget_eval_dataloaderzTrainer.get_eval_dataloaders  D$5$=LMM M*4L#)F)FRF D- . . :$"888 79).9 9,,, #D l + +'" ## $y0-) $   r test_datasetcR||d|jj|jS)a Returns the test [`~torch.utils.data.DataLoader`]. Subclass and override this method if you want to inject some custom behavior. Args: test_dataset (`torch.utils.data.Dataset`, *optional*): The test dataset to use. If it is a [`~datasets.Dataset`], columns not accepted by the `model.forward()` method are automatically removed. test)rrrr)rrrr)rPrs rget_test_dataloaderzTrainer.get_test_dataloaders5## y0- $   r dataloaderc |j}t|trt|jjSt|jS#tt t f$rt||jjzcYSwxYw)z Helper to get number of samples in a [`~torch.utils.data.DataLoader`] by accessing its dataset. When dataloader.dataset does not exist or has no length, estimates as best it can ) rrrHr NameErrorAttributeErrorrprper_device_train_batch_size)rPrrs r num_exampleszTrainer.num_exampless  K (G'#788 7:-5666z)** *>95 K K Kz??TY%JJ J J J Ks4A A 3BBFrrrrrrc:|j}tr2t|tjr|||}n||j|}tjj o|j j dk}|||j j |j j |j j|rdndd} t|tjjjs^|||| d<|j j| d<|j j| d<|r.t)t*|j j |j j | d <|jt3|fi| } |0|j jr$t5|d r | |j|<n || i|_| S) zACreate a [`~torch.utils.data.DataLoader`] from the given dataset.)rr forkN)r collate_fn num_workers pin_memorypersistent_workersmultiprocessing_contextsampler drop_lastprefetch_factor)rrankworker_init_fnr)rrrdatasetsr_remove_unused_columns"_get_collator_with_removed_columnsrrmps is_availablerdataloader_num_workersrrrsrtrdataloader_drop_lastdataloader_prefetch_factorrrm process_indexrpreparerrr) rPrrrrrrr should_forkrrs rrzTrainer._get_dataloaders* " " qz'8;K'L'L q11'{1SSGG CCDDVdoCppMn(5577`DI<\_`<` %'9;)9"&)"I1<'Fvv$   '5;#3#CDD %/9z'/B/B!),-1Y-K k *3793W / 0 6=TY-MTXT]Tk777!"23%--j.V.VDU.V.VWW   %$)*Q %t011 F9C&~66*8*)E&rc||j}|t|sdS|jjdkrt rBt |t jr(|jj|j vr||jjnd}nd}|j |j j dnd}t|jj |jjz|||S|jjdkrt|St!|S)z?Return the training sampler based on `train_sampling_strategy`.Ngroup_by_lengthrrlengthsmodel_input_name sequential)rrirrurrrrlength_column_name column_namesrmodel_input_namesrJr?rrr)rPrrrs rrzTrainer._get_train_samplers   .M  =(A(A 4 9 ,0A A A$&& :mXEU+V+V y3}7QQQ"$)">?? >B>S>_%7::ei ( *TY-RR%!1   Y ., > >$]33 3 // /rc|t|sdS|jjdkrtrBt |t jr(|jj|jvr||jjnd}nd}|j |j j dnd}t|jj |||S|jj dkrt|SdS)zNReturn the evaluation sampler, using sequential ordering when not distributed.Nrrrr )rirrurrrrrrrrrJr world_sizer)rPrrrs rrzTrainer._get_eval_samplers  z,'?'? 4 9 ,0A A A$&& :lHDT+U+U y3|7PPP!!=>> >B>S>_%7::ei ( )$!1   9 1 $ $$\22 24rc|j|j}t|jr@t|jdr|j}n|jjj}t j|j}t|j |_|xjttddg|j zz c_dSdS)zTPopulate `_signature_columns` from the model's forward signature if not already set.Nget_base_modellabel label_ids)r>rrarr base_modelrrrlistrkeysrr)rPrVrs r _set_signature_columns_if_neededz(Trainer._set_signature_columns_if_needed4s  " *#z dj)) C4:'788C'+z'@'@'B'B$$(,z'<'B$)*:*BCCI&*9+?+D+D+F+F&G&GD #  # #tC+0FIY0Y,Z,Z'['[ [ # # # # + *rzdatasets.Datasetc z|jjsS||j}t t jt |z }t|dkrx|dnd|d}t d|d|j j j dd |d d |d |j j j d fd |D}t|dkr.Ws&MMM18L3L3L13L3L3LrzGNo columns in the dataset match the model's forward method signature: (z-). The following columns have been ignored: [zp]. Please check the dataset and model. You may need to set `remove_unused_columns=False` in `TrainingArguments`.z1.4.0type format_kwargs)r columnsr )rremove_unused_columnsrr>rrrrrrrrrjoinrrparserr set_formatformatremove_columns)rPrrsignature_columnsignored_columnsdset_descriptionrs ` rrzTrainer._remove_unused_columnsDs y. N --/// 3s7#7883?P;Q;QQRR   ! # #%0%8rr>Y >Y>Y>Y  KK7)977J(177TXT]T]^mTnTn77yy1177IMI]If777   NMMM/MMM w<<1  @Z^ZcZcduZvZv@@=AYY=W=W@@@  =- . .w1G1G G G   ^F+WGN[jLk    N))/:: :rc|jjs|S||j}t ||t ||jjj}|S)z=Wrap the data collator in a callable removing unused columns.)rrrr model_name) rrrr>r]rrrr)rPrrrremove_columns_collators rrz*Trainer._get_collator_with_removed_columnsgsdy. !  --/// 3"7'/#z+4 # # # '&rnum_training_stepscZ|||dS)aZ Setup the optimizer and the learning rate scheduler. We provide a reasonable default that works well. If you want to use something else, you can pass a tuple in the Trainer's init through `optimizers`, or subclass and override this method (or `create_optimizer` and/or `create_scheduler`) in a subclass. rN)create_optimizercreate_scheduler)rPrs rcreate_optimizer_and_schedulerz&Trainer.create_optimizer_and_schedulerys4  1CDDDDDrc4 ||jn|}|jR|| fd|D|jjd fd|Dddg}|j |j\}}n||j|\}}t|r||fi||_nYd|vr| d}d|vr| d}d|vr| d}||fi||_d t|vr,| d dd krd dl }|j j}d }|D]} t#| t$jr|t)d | Dz }t.d| d|dz d|| dd dit.d| dt.d|dz dt7rt9j|j|_|jS)aO Setup the optimizer. We provide a reasonable default that works well. If you want to use something else, you can pass a tuple in the Trainer's init through `optimizers`, or subclass and override this method in a subclass. Returns: `torch.optim.Optimizer`: The optimizer instance. Nc0g|]\}}|v |j|Sr requires_gradrnpdecay_parameterss rrz,Trainer.create_optimizer..s8"aqL\G\G\abapG\G\G\G\r)ra weight_decayc0g|]\}}|v |j|Srr#r%s rrz,Trainer.create_optimizer..s8"aqP`G`G`efetG`G`G`G`rraroptimizer_dict bitsandbytes optim_bitsrc\i|])}||*Sr)data_ptrnumel)rr's rrz,Trainer.create_optimizer..s*']']']A aggii']']']rzskipped z: izM paramsweight zbitsandbytes: will optimize z in fp32z skipped: )rr%get_decay_parameter_namesnamed_parametersrr)rget_optimizer_cls_and_kwargsrFrrrr-optimGlobalOptimManager get_instancemodulesrr Embeddingsumrrrrregister_module_overridedebugrreDistributedOptimizer) rPr opt_modeloptimizer_grouped_parameters optimizer_clsoptimizer_kwargsr-managerskippedmoduler(s @rrzTrainer.create_optimizers)#(-DJJU > !#==iHH &/&@&@&B&B%)I$: &/&@&@&B&B%(  , (,8262O/ //262S2STXT]_h2i2i/ /$M22 a!0!O!O>N!O!O ///3C3G3G3Q3Q0...3C3G3G3P3P0$'7773C3G3GHX3Y3Y0!./K!`!`O_!`!`]!3!3338H8L8L\[_8`8`de8e8e####&,?LLNN'//11VVF!&",77V3']']IZIZI\I\']']']'d'd'f'f#g#gg $Rv$R$R5$R$R$RSSS88LZ\K]^^^ %TF%T%T%TUUU A%AAABBB " $ $ F 5dnEEDN~rr%c*|j|8tr#tjjjr |jj}n|j}t|jj ||j |||jj |_d|_ |jS)ad Setup the scheduler. The optimizer of the trainer must have been set up either before this method is called or passed as an argument. Args: num_training_steps (int): The number of training steps to do. Returns: `torch.optim.lr_scheduler.LRScheduler`: The learning rate scheduler instance. N)r%num_warmup_stepsrscheduler_specific_kwargsT) r&rrer8rfrdr%r8rlr_scheduler_typeget_warmup_stepslr_scheduler_kwargsrA)rPrr%s rrzTrainer.create_schedulers   $ *,,/1C/ $ 8II $I - +#!%!;!;+7799!< ?7 # # %llnnGs) A,/A''A,resume_from_checkpointtrial$optuna.Trial | dict[str, Any] | Noneignore_keys_for_evalc|durd}|j|j}d|_|j|jrt |jnt|j| ||_ d\|_ |_ |j r ||j |j|j |_|jjrt%|j |jj|js|jr.|js'|j ||j |j|jr |j |jt5|jt8t:fr/t=|j drt?|j |j|j %tC|j |j |j"|_#|$|tJj&|j'vr.|j(dkrtSdtU|j t5|tVr0|r.tY|j-}|tSd |j-d |t]s#|j/s|j0s|1|tej3thj56|tn}|j8|j9r |j8|_:tw|j<|j:|j9}|j=rt}ntj@}|5||||| cdddS#1swxYwYdS) a Main training entry point. Args: resume_from_checkpoint (`str` or `bool`, *optional*): If a `str`, local path to a saved checkpoint as saved by a previous instance of [`Trainer`]. If a `bool` and equals `True`, load the last checkpoint in *args.output_dir* as saved by a previous instance of [`Trainer`]. If present, training will resume from the model/optimizer/scheduler states loaded here. trial (`optuna.Trial` or `dict[str, Any]`, *optional*): The trial run or the hyperparameter dictionary for hyperparameter search. ignore_keys_for_eval (`list[str]`, *optional*) A list of keys in the output of your model (if it is a dictionary) that should be ignored when gathering predictions for evaluation during the training. Returns: [`~trainer_utils.TrainOutput`]: Object containing the global step count, training loss, and metrics. FNTr)gradient_checkpointing_kwargsrr z\Currently --debug underflow_overflow is not supported under DP. Please use DDP with torchrunz/No valid checkpoint found in output directory ())rrirjrl)Arrrr9rrrfrrnrrr%r&r rrruse_liger_kernelr)liger_kernel_configr rrrgradient_checkpointing_enablernrrr:r9rrbr$r*rneftune_hook_handle_hp_search_setuprUNDERFLOW_OVERFLOWr?n_gpurrboolrhrrr r_load_from_checkpointrBload_from_jsonr5pathrTRAINER_STATE_NAMEr?rr@rg_inner_training_loopr2rq contextlib nullcontext)rPrirjrlrr8inner_training_looprXs rtrainz Trainer.train7s. "U * *%) " ""$$$y ? &262G ` #DI . . .XVZV_M`M`--e44DJ0: -DND-) D**4:t{CCC!%D  9 % J tz49+H I I I   @4#6 @@V @[_[j[r  & &tz4; ? ? ?  & w J 4 4SWSu 4 v v v d+.E~-V W W D\c J] ]  D !T-B C C C  # /'7 DD\^b^n'o'oD $ e$$$  )TZ 7 7zA~~!r'tz222 ,d 3 3 g8N g%8%I%I "%- !eSWSb!e!e!efff ! -+,, CT5N CW[Wk C**+ABBB / =SUg0h0hiiE%1d6O1).)?&8  %t'=t?X  +/*:X$&&& @V@X@X   &&'=%9                   s3MMMc\|j|jr|||}|jrt |}|||\}}} } } } } || ||||\}}| | ||\}}t dt d| dt d|dt d|dt d|j j d|j j |jkr#t d|jdt d| dt d |jt d | dt d t!|d d|Wt d|d|jj|j js!t d|d|dt)j}|jj|_t/jd|j|_d|_|jj|_||j||j|j |_ |j!r|"||d tG||D]`}|j$|j |j|j |_ |%|||| ||||||| |j j&rna|'|| |S)z\Run the actual training loop: forward, backward, optimizer step, logging, and checkpointing.z***** Running training ***** Num examples = ,z Num Epochs = z Num update steps per epoch = z( Instantaneous batch size per device = zA Training with DataParallel so batch size has been adjusted to: zE Total train batch size (w. parallel, distributed & accumulation) = z Gradient Accumulation steps = z Total optimization steps = z# Number of trainable parameters = T)trainable_onlyNz/ Resuming training from checkpoint with epoch z and global step z& Fast-forwarding the dataloader past z epochs and z1 batches to resume from the exact training state.r+r)skip_scheduler) repochtrain_dataloadersteps_in_epochnum_update_steps_per_epochrjrl start_timeriepochs_trainedsteps_trained_in_current_epoch)(r free_memoryr_update_auto_batch_sizerrDr.set_initial_training_values_init_training_state_prepare_for_trainingrrrrr@rrMr8 global_stepignore_data_skiptimenum_input_tokens_seen_initial_num_input_tokens_seenrrr_tr_loss_total_loss_scalar_globalstep_last_logged zero_gradr-on_train_beginr7 eval_on_start _evaluaterMon_epoch_begin _run_epochshould_training_stop_finalize_training)rPrrrirjrlrrrrrnum_train_samplestotal_train_batch_sizerrqrrrrrs rr|zTrainer._inner_training_loops $$&&&  $ 5  ( ( 4 4 44466  & E23CDD   , ,T3C D D  &   "  :>9R9R 13CE[]b: : 66#'"<"+;<<  E0?? 4:W[WcddDL OO!1-+E%9%'=-/M    |0  &&u.?LLLrctd|jj|jgzD|_|du|j_|j|j_|j|j |d}d}|tj tj |trtjtj |t|_t!|j |j|t%|jj|z}|j js|jj|z}||j jz}|j||||||fS)zxInitialize TrainerState, optionally restoring from checkpoint. Returns (epochs_trained, steps_trained_in_current_epoch).c<g|]}t|t|Srrrs rrz0Trainer._init_training_state..rr)rNr)rBr-rr7r8is_hyper_param_searchr@r? compute_stepsrr5rzisfilerr{ryrc_load_callback_stateintrrrinit_training_references)rPrqrrrrirjrrs rrzTrainer._init_training_statest"  !2< ~M       ,1+< (&*&< #   I666)*& ! -"'.. GLL/1C D D3 3 -&4RW\\BXZl5m5mnnDJ / 4: F F F  % % ' ' ' !7;U!UVVN9- X151GJd1d..$)2WW. ++D)=MuUUU===rc 6tp |jp|j}|jo#t|jjjdddk}|rd}|jrd|_d|_|j rt||\|_ |_|s| | |j}||ju}|r|r|jr.t!|jrt#|j|j|j|j}| ||j|j |_ n|j rWt'|jjdkr:|j|j|j |j\}|_ |_nR|j|j|j \}|_ n$|j|j |_ ||||_|js|jr|x|_|_|j r |j|_|jr:t/|jdr%t0j|jdt|jd d}|-|jd kr"|jr|j||}||j r+t=|j|t!|j n0ts|jr|||j| ||!|d D]&} tE|j#| t|| '||j#_$||fS) zkWrap model, create optimizer and scheduler, and run accelerator.prepare. Returns (model, train_dataloader). fsdp_versionr FNrDummySchedulergeneraterrload_module_strict)rr%r&)%rrrrrr8rrAr&r r$r%r _wrap_modelrrrarrr rrrrdistrregister_fsdp_forward_methodrrdeepspeed_ulysses_dl_adapterr%rx_load_optimizer_and_scheduler _load_scalersetattrr-r) rPrqrridelay_optimizer_creationis_fsdp2ruse_accelerator_preparerUattrs rrzTrainer._prepare_for_training s#:#<#<#p@X#p\`\p 'qWT5E5K5WYgij-k-kop-p  -', $  % / $D ).D &  $ c0>tXa0b0b0b -DND-' $  ! ! # # #  !344#(4:"5 # F' ]'JN4:,F,FJ+DJ8HIII(00<<%%e,,,!%!1!9!9$.!I!I* ]tD4E/F/F/OSc/c/c;?;K;S;SJ0A<<8t~t'8'8)-(8(@(@T^(\(\%t~~!-55dnEEDN ;;;#   44#; 4/4 3DJ+  $ 0!/DN   Otz:.. O 66tz:NNNT%';T B B >bm{::r}:#/LLM]_dee  " -( W)&(>WefjfpWqWqSq)** Wd.B W**+A4CUVVV  . ./E F F F   4 5 5 5; F FD D)4t1D1D E E E E1A.&&&rc  d} d} d}d}d}|| krV| T| dkr3|jjs't|| }| dz } | |jjz}d}n| dkr|| t |dr||t|}||jjz}|dkr |jj}t||D]}||dz kr |jjn|}| |||jj \}}t||_ |r|| d}t|D]C\}}| dz } | dz|jjzdkp| dz|k}|jj|| |jjzdkr0|j|j|j|j|_|jjjdds0|jjt0jks|t|dz kr t4j}n t9j|jj| }|5||||}dddn #1swxYwY|jj rdtCsVtEj#|stEj$|r.|xj%|j%d|jj&z|j'z z z c_%nI|j%j |j kr$tQd |j%j d |j |xj%|z c_%|xj)tU|+|z c_)|,||rd} |jj-dkr|.|} |/|| } |j0|j|j|j|_|j12|j3|j|j|j|_|4}|jj5sItm|j7tDj8j7j9ttfs|j72|;|jxj&dz c_&|| dz|z z|j_<|j=|j|j|j|_|>|j%| |||||| n0|j?|j|j|j|_|jj@s |jjArnE|jj@s |jjArntCrtjC| dkrAtEd|jj&d|jjFdd|j_A|jG|j|j|j|_|>|j%| |||||| dS)z&Run one full pass over the dataloader.NFrr T set_epochsync_each_batch)rz0Calculated loss must be on the original device: z but device in use is ) grad_norm)rTzXThere seems not to be a single sample in your epoch_iterator, stopping training at step zI! This is expected if you're using an IterableDataset and set num_steps (z.) higher than the number of available samples.)Hrrrr_load_rng_staterriterrMget_batch_samplesrr#current_gradient_accumulation_steps enumeratergradient_state_set_sync_gradientsr- on_step_beginr8r7 plugin_kwargsrdistributed_typer DEEPSPEEDr}r~rrno_sync training_steplogging_nan_inf_filterrrisnanisinfrrrrr<floatfloating_point_ops_track_num_input_tokens max_grad_norm_clip_grad_norm_get_grad_normon_pre_optimizer_stepr%stepon_optimizer_steprhoptimizer_step_was_skippedrr&r8rcr7rr on_step_end_maybe_log_save_evaluateon_substep_endshould_epoch_stoprxm mark_steprrgrq on_epoch_end)rPrrrrrrjrlrrirrrrrT rng_to_syncnum_update_steps_trainedepoch_iterator remainder update_step num_batches batch_samplesnum_items_in_batchiinputs do_sync_step sync_context tr_loss_steps rrzTrainer._run_epochsF    $% N " "'='I-11$):T1#56FHf#g#g 59+ITYMr+r(" /144$$%;<<< #[ 1 1 .  & &u - - -.//#TY%JJ >> =I !!9;UVV^ ^ K9DIcfgIg9h9h 55nw 150F0F~Wbdhdmdt0u0u -M-8;=7I7ID 4 $$$%;<<<# '}55J J  6  $qDI,QQUVVv[_bc[chvZv  /CCLQQQ$)??1DD#'#8#F#FtyRVR\^b^j#k#kDL$3AEEFWY^__\'8OV>hjr=stt5 -22444OO%%%J**a/**',qN/J'JDJ$#'#8#D#DTYPTPZ\`\h#i#iDL11 !,"&32    $(#8#G#G SWS]_c_k#l#lDL<1T\5VE|- 1R   " # #  LNNN !88 NNdJ*dd#z3ddd    15DL -,99$)TZQUQ]^^  %% M     ' & sJJ J ctd|xj|jz c_t |jjd}|j|z }td|||jj }| |jj |d<||d<|j ||||jjr |jj|t'|||jj}|jjr|jjs|jjd krc|D]`}t.j||jjs4td |d t5j|d a|j|j|j|j|_||j tC|j"|j#|j$d|_%tM|jj||S)zMFinalize training: metrics, best-model loading, cleanup. Returns TrainOutput.zU Training completed. Do not forget to share your model on huggingface.co/models =) gMbP?r num_samplesr total_flos train_lossN)rbest_model_checkpointr zDeleting older checkpoint [z] due to args.save_total_limitT) ignore_errorsF)'rrrrremaxr8rrprq store_flosrrrOlogrrlr_load_best_modelro_get_output_dirr4save_total_limitr5rzsamefileshutilrmtreer- on_train_endr7_finish_current_pushr$r+rrsrr9r`) rPrjrreffective_global_steprmetricscheckpoints_sorted checkpoints rrzTrainer._finalize_training's8 oppp 4=#5#5#7#77 #DJ$:E B B,/DD   )j*      $ 5  *  44W===  9 + $ 0P0\  ! ! # # #-++E22$*Jj    9  BTZ%E%QVZV_VptuVuVu0 B B w'' DJ4TUUBKK hj h h hiiiM*DAAAA,99$)TZQUQ]^^  !!###  # / tz4+CTEU V V V 4:1:wGGGrrrc||||\}}|5|t|jdr2t |jjr|j||}t rjt|||jj }| |jj cdddS|5||||}dddn #1swxYwY~|jj|jj|jjzdkrit't(jjr8tt(jdrt(ji}|jjt4jt4jfvr||d<|jjdkr|}|j r||j! ||j"z }|j#j$tJj&krd|d <|j#j'|fi|| cdddS#1swxYwYdS) ak Perform a training step on a batch of inputs. Subclass and override to inject custom behavior. Args: model (`nn.Module`): The model to train. inputs (`dict[str, torch.Tensor | Any]`): The inputs and targets of the model. The dictionary will be unpacked before being fed to the model. Most models expect the targets under the argument `labels`. Check your model's documentation for all accepted arguments. Return: `torch.Tensor`: The tensor with training loss on this batch. rNrrclear_graph_cacherTr F scale_wrt_gas)( _prepare_context_parallel_inputsrrr%ro_prepare_inputsrrrr reduce_meandetachtorcompute_loss_context_manager compute_losstorch_empty_cache_stepsr8rrrrrrr r8rtLOMOADALOMOrhrvmeanrrrrrrrbackward)rPrrr cp_contextloss_mblossrs rrzTrainer.training_stepXs;2"BB5&QQ FZ\\+ !+ ! KKMMMt~w// 'HT^=Q4R4R '$$&&&))&11F&(( K.ufdi>cdd**,,3355889IJJ+ !+ !+ !+ !+ !+ !+ !+ !2244 _ _((K](^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 1=J*TY-NNRSSS"$$$>%22442L_9`9`2I//111Fy>#68N"OOO*.*A*A*C*C'y""yy{{2 G6H6PVZVlVtdFF0O4MMM*/' %D  %d 5 5f 5 5 5;;==W+ !+ !+ !+ !+ !+ !+ !+ !+ !+ !+ !+ !+ !+ !+ !+ !+ !+ !s=CJ1 J1!E: J1E J1 E EJ11J58J5return_outputsc t|jdd}|6|jdkr+|jr$|jjrt |j||||S|j|jd|vr| d}nd}|j ri}|||d<i||}|di|}|j6|t d||||} n@||j |} t| r| jjn| } | t#jvr|||d } n|||} nt'|t(rdd |vr`t+d d |d d |dt'|t(r|d n|d} |jjr_|j s|jrQ|O|jj} t|jddx} | |jz} | |jjdkr| n |jjz} |r| |fn| S)a6 How the loss is computed by Trainer. By default, all models return the loss in the first element. Args: model (`nn.Module`): The model to compute the loss for. inputs (`dict[str, torch.Tensor | Any]`): The input data for the model. return_outputs (`bool`, *optional*, defaults to `False`): Whether to return the model outputs along with the loss. num_items_in_batch (Optional[torch.Tensor], *optional*): The number of items in the batch. If not passed, the loss is computed using the default batch size reduction logic. Returns: The loss of the model along with its output if return_outputs was set to True Subclass and override for custom behavior. If you are not using `num_items_in_batch` when computing your loss, make sure to overwrite `self.model_accepts_loss_kwargs` to `False`. Otherwise, the loss calculation might be slightly inaccurate when performing gradient accumulation. rNrlabelsrz|Trainer: `compute_loss_func` is defined but `labels=None`. Your custom loss function will still be called with labels=None. r T) shift_labelsrzJThe model did not return a loss from the inputs, only the following keys: rz,. For reference, the inputs it received are .rr r)rrrrrtrainingr&r#rrrrrgr4rar _get_namer5rrdictrrrraverage_tokens_across_devices num_processesrrv) rPrrrrrUrroutputsrrSr loss_scales rrzTrainer.compute_losss6T%';T B B >bm{::r}:QUQ[Qd:,T-=ufn^`aa a   +t/E/QW_ciWiWiZZ))FFF  ) *F!-/A+,))&)F%//&//  ! -~X))#5*DD  ".;;EBBO"/221*0::<<<$..00  >EGGGG**7F*NN**7F;;'4(( V7-B-B xxx //xx]`]e]eflfqfqfsfs]t]txxx '1$&?&?O76??WQZD I 3 L/ L373I L#.)7Jd.0DdKKKXrz) $)/Q"6"6JJDIO KD"0:gd:rctj}|}t|tjs|||S)zF A helper wrapper to group together context managers. )r} ExitStackautocast_smart_context_managerrr~ enter_context)rP ctx_stack autocast_ctxs rrz$Trainer.compute_loss_context_managersR(** ::<< , (>?? 2  # #L 1 1 1rT cache_enabledc(tjS)z A helper wrapper that creates an appropriate context manager for `autocast` while feeding it the desired arguments, depending on the situation. We rely on accelerate for autocast, hence we do nothing here. )r}r~)rPr-s rr)z&Trainer.autocast_smart_context_managers %'''rtr_lossrrrrTc |jjr5|jj|jkrt rt ji} t||j j  } ||z}| |jj|jz z | d<|3t|tjr| n|| d<||| d<n|| d<|xj| z c_|jj|_||| |d} |jjrS|||} || |} |j jt0jkr | |j_|jjrH||||j|j |j|j|_dSdS)z\Log metrics, run evaluation, and save checkpoints if the current training state requires it.rNrrT)rrj)r7 should_logr8rrrrrrQrr rrerrTensorrhrrrshould_evaluater_determine_best_metricrjr^rkr4_save_checkpointr-on_save) rPr/rrrjrrlrrTlogstr_loss_scalarris_new_best_metrics rrz Trainer._maybe_log_save_evaluate s < " 'tz'=@\'\'\%''  %'D+7DI4KLLQQSSXXZZN w G)TZ-CdFb-bcDL$8B9el8[8[$jINN$4$4$4aj[!((5_%%(,(?(?(A(A_%  # #~ 5 # #+/:+AD ( OO    HHT: & & & < ' >nnU,@AAG!%!.nsj$$$  &...))/)xabb11x $$$rc3dK|]+}|dV,dS)N)ner=)rrs rrz2Trainer._get_num_items_in_batch..vs8(^(^64)<)<)>)>(^(^(^(^(^(^rr rr)rrrr=rprrr#rrgatherrrvrrddim unsqueezeexpandrnon_data_parallel_size)rPrrrcount_num_items_in_batchlabels_for_countrUs` rr>zTrainer._get_num_items_in_batchQs"    " M!,,  .6)5 ! $  $$$$ "/ $$$ &)(^(^M](^(^(^%^%^""~.      )y6 K9'!++)-)9)@)@ASAVAVW]A^A^)_)_)c)c)e)e&1$$&849?%J"122 Y%7%:%:6%B%B"9?Q&&+=+A+A+C+Cq+H+H);)E)Ea)H)H)O)OPTPYP_ac)d)d& !13GNNN2Y);r?X)X&!!s'AA0/A0rtc`t|tr6t|fd|DSt|tt fr$t|fd|DSt|t jrdjj i}j ret j |st j |r=| djjjji|jdi|S|S)z| Prepares one `data` before feeding it to the model, be it a tensor or a nested list/dictionary of tensors. cBi|]\}}||Sr_prepare_input)rrvrPs rrz*Trainer._prepare_input..s-RRRTQq$"5"5a"8"8RRRrc3BK|]}|VdSrrO)rrQrPs rrz)Trainer._prepare_input..s1CCd11!44CCCCCCrrdtyper)rrr itemstuplerrr2rrr is_floating_point is_complexrrr8r| hf_ds_configrSr)rPrtrs` rrPzTrainer._prepare_inputs/ dG $ $ %4::RRRRTZZ\\RRRSS S udm , , %4::CCCCdCCCCC C el + + % 01F( ge.Ed.K.K guO_`dOeOe g w(8(>(O(\(b(b(d(defff47$$V$$ $ rc||}t|dkr+tdd|jd|S)z Prepare `inputs` before feeding them to the model, converting them to tensors if they are not already and handling potential state. rzThe batch received was empty, your model won't be able to train on it. Double-check that your training dataset contains keys expected by the model: rr)rPrrrr>)rPrs rr zTrainer._prepare_inputssl $$V,, v;;!  nILRVRiIjIjnnn   rc<t|jdd|jjjr|jjjdkrt |dr-|jjdkrtd|jjdd|vrkt d d |vrMd }tj |d d | }|ddddf|d<d|vrt dtj|dd|dj|ddd|d<g}g}d|vr0||d|dd |vr0||d |dd|vr0||d|dd|vrt|dds-|d}t+|stdd|_|jrL|d}|dkr+|||dn d|vr8|d0||d|dt1|jj||t5||fSt6j|fS)as Prepare inputs for context parallelism by setting up buffers and validation. Args: model: The model being trained inputs: Input tensors to prepare Returns: tuple: (context_manager, prepared_inputs) where context_manager is either the context parallelism wrapper or a no-op context rNrrsdpazIContext parallelism is supported only with SDPA attention, you are using rrz7Shift labels not found in the inputs, shifting manuallyrrD)rr )valuer position_idsz9Position IDs not found in the inputs, generating manually input_idsrrrattention_mask_attn_mask_causal_checkedFzContext parallelism only supports causal attention masks. The provided attention_mask is not causal. Please ensure your data uses causal masking (lower triangular) or remove the attention_mask to use the model's default causal masking.Tr)buffersbuffer_seq_dimsno_restore_buffers)rrr cp_enabled cp_backendrr_attn_implementationrr warning_oncer functionalpad contiguousrarangesizerrIr;rOr`rGrmaybe_context_parallelrr}r~)rPrr _ignore_indexrrarbr_s rr z(Trainer._prepare_context_parallel_inputss D$&:D A A M 3> N2=HH5(++|8FBB(MhmhtiJMMM"//''(abbb6))(, !#!2!26(3CVS`!2!a!a171221I1I1K1K~.V++##$_```).;',,Q//{8K8R***& ,11!44b99~&G Of$$vk2333&&q)))6!!vh/000&&q)))''vn5666&&q)))6))t%@%HH :&,,<%=N3NCC(f 6:D21%+,<%=N%))++q00~666'..q1111''F>,B,Nvn5666&&q))) 7 /#&w<<    %v--rc|j}|dk}t|rt|nd}||}|}|dkr|||z}|Rt ||jzt||jzdkzd}|rtj |j |z}|ry| |} |jdkr"||zt||zdkz} ||z} n|tj |j } | ||j z} nE|jdkr#tj } |}||jz} |j|z} ntd|j||n ||jz} | || | || |fS)a' Calculates and returns the following values: - `num_train_epochs` - `num_update_steps_per_epoch` - `num_examples` - `num_train_samples` - `total_train_batch_size` - `steps_in_epoch` (total batches per epoch) - `max_steps` rNr zYargs.max_steps must be set to a positive value if dataloader does not have a length, was )rqrirget_total_train_batch_size get_sp_sizerrrmathceilrrrsysmaxsizer) rPrrrq epoch_basedlen_dataloaderrsp_sizerrrrrrs rrz#Trainer.set_initial_training_values sN !m ,6z,B,BLZ!%!@!@!F!F""$$ Q;;>5+g5N  %),$"BBnt'GG!KLLM** &  Z Id&;>X&XYY  ,,Z88L~!!#,0J#JS ::Q>NN$ %.0F$F!!#'9T-B#C#C $($5$5j$A$ADDY$Y!! ^a  "{ )2 &1DNBL $1G G  %N%% ,:+E9W[WwKw  &   "    rc|j|z|z|z}|j|jz|zS)aCCalculates total batch size (micro_batch * grad_accum * dp_world_size). Accounts for all parallelism dimensions: TP, CP, and SP. Formula: dp_world_size = world_size // (tp_size * cp_size * sp_size) Where: - TP (Tensor Parallelism): Model layers split across GPUs - CP (Context Parallelism): Sequences split using Ring Attention (FSDP2) - SP (Sequence Parallelism): Sequences split using ALST/Ulysses (DeepSpeed) All dimensions are separate and multiplicative: world_size = dp_size * tp_size * cp_size * sp_size )r get_tp_size get_cp_sizerqr@r)rPr dp_world_sizes rrpz"Trainer.get_total_train_batch_sizeS sU4+;+;+=+==AQAQASASSW[WgWgWiWii %(HH=XXrcXt|jdddS|jj}|jS)zGet the sequence parallel sizerNr )rrrrxrPrUs rrqzTrainer.get_sp_sizee 0 4#%94 @ @ H1!4B: rcXt|jdddS|jj}|jS)zGet the context parallel sizerNr )rrrcp_sizer~s rr{zTrainer.get_cp_sizem rrct|jddx}|S|jrGt|jddx}r/|jdiddSdS)zGGet the tensor parallel size from either the model or DeepSpeed config._tp_sizeNrtensor_parallel autotp_sizer )rrr rrr)rPmodel_tpdeepspeed_configs rrzzTrainer.get_tp_sizeu s  J== =H JO  $ \gdiQfhl>m>m*m*: \#*../@"EEII-YZ[[ [qrr c\|j|d|ur|StrAt|tjjr|St j||jjS|jj dkr6t|dds%t|ddstj |}|s|S|j rt||j|j}nOt!rAtj|t't)jdg}|S) zMWrap `model` for distributed training if needed (DDP, FSDP, SageMaker, etc.).Fkeep_torch_compile)backward_passes_per_stepr is_loaded_in_8bitis_loaded_in_4bitSMDATAPARALLEL_LOCAL_RANK) device_ids)rr4rrrerDistributedModelrrrvrr DataParallelrr/rDrparallelDistributedDataParallelrr5getenv)rPrr rs rrzTrainer._wrap_model s=   ( (5 ( I IQV V VL " $ $ o%!;<<  ' Hmnnn n IOa  E#6>> E#6>> OE**E L  # 'ty$:UVVEE $ & & K773ry1L'M'M#N#N"O8E rcj||_t|j|jrt |jd|_|j|_|jrb|jj}|jtd|jj z|j_t|j |jd||j_dSdS)zwFree memory, reset model wrapping, and update DeepSpeed config for the new batch size when using `auto_find_batch_size`T) recursiver )rN) r@rrrr4rr rrrrvr(r)rPr original_bss rrzTrainer._update_auto_batch_size s",t)***   B%djDAAADJ!Z  $ @)?K484JcRSUYU^UdNeNe4eDI 1 '(8$)Z^ _ _ _ _4?DI 1 1 1  @ @rc,|jjdkrdSt|jdd}||vrtddS|jjdkrd|vr|d}n|jJt|jdr5|jj )|||jj k}nOtd || }n|| }tj ||jj tj }|jxj|j|z c_dS) z>Count input tokens seen (all or non-padding) and update state.r`Nmain_input_namer^zTried to track the number of tokens seen, however the current model is not configured properly to know what item is the input. To fix this, add a `main_input_name` attribute to the model class you are using. non_paddingr_ pad_token_idz\Could not determine method to count non-padding tokens, falling back to counting all tokens.)rrS)rinclude_num_input_tokens_seenrrrrgr=rrrr2r as_tensorrint64r8rrrFre)rPrr input_tokenss rrzTrainer._track_num_input_tokens s 9 2d : : F!$*.?MM & ( ( NNR    F 9 2m C C6))%&67;;== %1D1>BB2)6B & 74;P;] ]bbdd r &o6<<>> !/288::L|DI33DI4KLL L//0@0@0B0BDID[\\\rc|:|j|td}|jjt jkr$t|dr|}|S)z+Return the gradient norm as a Python float.Ninfre) rrrrrrrrre)rPrrs rrzTrainer._get_grad_norm sn  (889I9I9K9KUSX\\ZZI   ,0I I Iy&)) -%NN,, rr ignore_keysmetric_key_prefixc P|du}|r|n|j}t|trTi}|D];\}}||r|n|||d|}||<|S|j||} |j rt| } tj } | | d|j dnd||} |jj|jjz} |d| jvr| | j|dz } | jt%|| | jt)j| j| z || jt.j|jjvr%t5jt9j|j|j|j |j!| j|_!|j"| j| jS) a Run evaluation and returns metrics. The calling script will be responsible for providing a method to compute metrics, as they are task-dependent (pass it to the init `compute_metrics` argument). You can also subclass and override this method to inject custom behavior. Args: eval_dataset (`Dataset` | dict[str, `Dataset`], *optional*): Pass a dataset if you wish to override `self.eval_dataset`. If it is a [`~datasets.Dataset`], columns not accepted by the `model.forward()` method are automatically removed. If it is a dictionary, it will evaluate on each dataset, prepending the dictionary key to the metric name. If you pass a dictionary with names of datasets as keys and datasets as values, evaluate will run separate evaluations on each dataset. This can be useful to monitor how training affects other datasets or simply to get a more fine-grained evaluation. When used with `load_best_model_at_end`, make sure `metric_for_best_model` references exactly one of the datasets. If you, for example, pass in `{"data1": data1, "data2": data2}` for two datasets `data1` and `data2`, you could specify `metric_for_best_model="eval_data1_loss"` for using the loss on `data1` and `metric_for_best_model="eval_data2_loss"` for the loss on `data2`. ignore_keys (`list[str]`, *optional*): A list of keys in the output of your model (if it is a dictionary) that should be ignored when gathering predictions. metric_key_prefix (`str`, *optional*, defaults to `"eval"`): An optional prefix to be used as the metrics key prefix. For example the metrics "bleu" will be named "eval_bleu" if the prefix is "eval" (default) Returns: A dictionary containing the evaluation loss and the potential metrics computed from the predictions. The dictionary also contains the epoch number which comes from the training state. Nr?)rrrrT)rprediction_loss_onlyrr_model_preparation_timer)#rrr"rTevaluaterrrrrDr.revaluation_looprrrrrrprrrrsrrTPU_METRICS_DEBUGr?r master_printmetmetrics_reportr- on_evaluater8r7rO) rPrrroverridereval_dataset_name _eval_datasetdataset_metricseval_dataloaderroutputtotal_batch_sizes rrzTrainer.evaluate sUX t+'/F||T5F lD ) ) G4@4F4F4H4H 0 00!="&--2:!Q@Q +):&P&P=N&P&P#0## ////N ""$$$22<@@  & C1/BBOY[[ %% $*.)=)E4#/&   94ty7KK 8 8 8FN J J &.,=)V)V)VW WJ !".)F$69I$IJJ            (DIO ; ; OC.00 1 1 1,88DJPTP\^d^lmm  44V^DDD~rrc @|j}||n|j}|jr|jt |dd\}}||jd}t|jj dkr||jurtj } |js|j r&|jj s|j |n|j|d}ttj | z d|_|j r||_||jur||_|jr |j|_|js[|jr'|t*j|j }n-|jr&|t*j|j }|jj} t6d |d t;|r1t6d ||nt6d t6d| t?|dr(tA|j!r|!t?|j"dr2tA|j"j!r|j"!||j#_$tK|dd} tM|jj'd} tM|jj'd} tM|jj'd}tM|jj'd}d}i}d}tQ|D]P\}}tS|}| ||z }| |} |*||||\}}}tK|jdd}d|j+vr|,||nd}t[rt]j/|=|0|1| }| 2||Y|j3|dd}|0|}|jj4r|dkr|2|||j3|dd}|v|j3|dd}|j5|5||}|0|}|jj4r|dkr| 2||<|0|}|jj4r|dkr|2||j#6||j7|j8|_8|jj4r|j9[|Y|W|jj:j;}i}d|j+vr|nd|d<d|j+vr|nd|d<|9tyd$||d||}~~~~t*j=>|j?|dz|j?zdkrr| @| @|@|@~~~~t*j=>R|jjA|_0| B} | B} |B}|B}t;| rt| }nYt| trtK| dddkr| j}n't;|r||}n|}|dkr|dkr|}|j9S| Q|O|jj4sCd|j+vr| nd|d<d|j+vr|nd|d<|9tyd$| |d|}n|i}t|}t| trA| r?tjH| IJ||d<nFt| tjKr,| IJ||d<t?|d r |j||d!<t|LD]7}|M|d"s|N|||d"|<8t| |||#S)%z Prediction/evaluation loop, shared by `Trainer.evaluate()` and `Trainer.predict()`. Works both with or without labels. NrT)r inferenceF)r )evaluation_mode)rSrz ***** Running z *****rz Num examples: Unknownz Batch size = rrrD) padding_indexrrr^rr )rG pad_index Predictionrlosses) predictionsr)r_r_lossmodel_preparation_timerr?)rrrrr)Prrr rr$rrrr_modelsrr torch_compiler prepare_modelroundrrr9r rrfloat16rrbfloat16rrrrirrrorr%r-rrrGeval_do_concat_batchesrrLprediction_stepinclude_for_metricsrPrrrrrepeataddpad_across_processesrhron_prediction_stepr8r7rrend_of_dataloaderrYcuda empty_cacheeval_accumulation_stepsto_cpu_and_numpyr get_arraysrrHrerrK concatenaterrendarrayr startswithrrX)rPrrrrrrr?rrrr all_losses all_preds all_labels all_inputsreval_set_kwargsobserved_num_examplesrrobserved_batch_sizerlogitsrr inputs_decode is_last_step batch_kwargsrkeys rrzTrainer.evaluation_loopQ s y7K7W33]a]v  $ N)?!$1MMMDAq  e << t' ( (A - -%4:2E2EJ,Q151EQNRiNeQ ((///%33E43PP  +0 j0H!*L*LD '# #" DJ&&%*"( 4!%!3 K" Ku}T[II$ Ku~dkJJY.  :{:::;;; j ! ! 3 KKKD,=,=j,I,IKK L L L L KK1 2 2 2 2j22333 5& ! ! huz&:&:  JJLLL 4>6 * * "x8K/L/L " N   ! ! !0:-z9d;; 'ty'GW[\\\ %di&FVZ[[[ &ty'GW[\\\ &ty'GW[\\\ !"&j11C )C )LD&"1&"9"9 ".%)<<%%!4J&*%9%9%I]kv%9%w%w "FFF%dj2C[QQO@HDLd@d@d##F?$;<<>v1X\>]]!)>>v1X\>]]5A!??OOF--f55y3*{l7R7RMM&)))!--f55y3+{l7R7RNN6***0CCD$*VZVbccDLy+ )'38JvOa#'#3#B#TL#%L7=AY7Y7YVV_cL*7?4C[7[7[VVaeL*"22&\6V\\|\\'33G FFF &&((((-9tax4Kg>gkl>l>l++---**,,,++---++---FFF &&((( $/B **,, ((** **,, **,,  l # # 4l++KK &: ; ; 4 Vdfg@h@hkl@l@l&3KK*%% 4"// ;; 3 !   5 9 9/K  ,%&I0'7=@X6X6X ^bOH %6>$BZ6Z6Z `dOH %**^9 ^^o^^GG_G(00 j$ ' ' LJ L35>*3M3M3R3R3T3T3Y3Y3[3[G(/// 0 0  BJ / / L3=??3D3D3I3I3K3KG(/// 0 41 2 2 aEIE`G(AAA B '' I IC>>%6"9"9"9:: I8? C8H8H,44s445)zSZhsttttrrc |j||}tj}||d||}|jj|jjz}|d|jvr||j|dz }|j t|||j tj |j |z |j|j|j|j|j|_|j|jt'|j|j|jS)af Run prediction and returns predictions and potential metrics. Depending on the dataset and your use case, your test dataset may contain labels. In that case, this method will also return metrics, like in `evaluate()`. Args: test_dataset (`Dataset`): Dataset to run the predictions on. If it is an `datasets.Dataset`, columns not accepted by the `model.forward()` method are automatically removed. ignore_keys (`list[str]`, *optional*): A list of keys in the output of your model (if it is a dictionary) that should be ignored when gathering predictions. metric_key_prefix (`str`, *optional*, defaults to `"test"`): An optional prefix to be used as the metrics key prefix. For example the metrics "bleu" will be named "test_bleu" if the prefix is "test" (default) If your predictions or labels have different sequence length (for instance because you're doing dynamic padding in a token classification task) the predictions will be padded (on the right) to allow for concatenation into one array. The padding index is -100. Returns: *NamedTuple* A namedtuple with the following keys: - predictions (`np.ndarray`): The predictions on `test_dataset`. - label_ids (`np.ndarray`, *optional*): The labels (if the dataset contained some). - metrics (`dict[str, float]`, *optional*): The potential dictionary of metrics (if the dataset contained labels). r)rrrrr)rrr)rrrrrrrrrrrprrrrsr- on_predictr8r7rOr\rr)rPrrrtest_dataloaderrrrs rpredictzTrainer.predict sTH ""$$$22<@@Y[[ %% ;bs&   94ty7KK 8 8 8FN J J &.,=)V)V)VW WJ !".)F$69I$IJJ       ,77 4:t|]c]kll  44V^DDDF,>&JZdjdrssssrct|jdkrdntfd|jD}d}||j}t|jdko|}|4t |jdrt|jj ddgng|s|rIttfd |jD}t|d kr|d}nd}tj 5trt|} |s|rt!| t"r6| d } tfd | D} n| d} | d d} | } t-| } nd} t!| t"r.tfd | D} n| } t-| } n\|s|r|5|g|jj}||d|\} }dddn #1swxYwY| } t!|t"r.tfd|D} n|d d} n{d} |5|di}dddn #1swxYwYt!|t"r.tfd|D} n|} dddn #1swxYwY|r| ddfSt| } t| d kr| d} | | |fS)a Perform an evaluation step on `model` using `inputs`. Subclass and override to inject custom behavior. Args: model (`nn.Module`): The model to evaluate. inputs (`dict[str, torch.Tensor | Any]`): The inputs and targets of the model. The dictionary will be unpacked before being fed to the model. Most models expect the targets under the argument `labels`. Check your model's documentation for all accepted arguments. prediction_loss_only (`bool`): Whether or not to return the loss only. ignore_keys (`list[str]`, *optional*): A list of keys in the output of your model (if it is a dictionary) that should be ignored when gathering predictions. Return: tuple[Optional[torch.Tensor], Optional[torch.Tensor], Optional[torch.Tensor]]: A tuple with the loss, logits and labels (each being optional). rFc3FK|]}|duVdSrr)rrrs rrz*Trainer.prediction_step..{ s4AvAv`a&**Q--W[B[AvAvAvAvAvAvr return_lossNrkeys_to_ignore_at_inferencepast_key_valuesc3BK|]}|VdSrr)rnamers rrz*Trainer.prediction_step.. s/(W(WdD)9)9(W(W(W(W(W(Wrr rc32K|]\}}|dgzv |VdSrNrrrrQrs rrz*Trainer.prediction_step.. s;)o)o1qXcgmfnXnOnOn!OnOnOnOn)o)orc3*K|] \}}|v |VdSrrrs rrz*Trainer.prediction_step.. s2)d)d1qXcOcOc!OcOcOcOc)d)drT)rrc32K|]\}}|dgzv |VdSrrrs rrz*Trainer.prediction_step.. s;&h&hTQQ\`f_gQgHgHgqHgHgHgHg&h&hrc3*K|] \}}|v |VdSrrrs rrz*Trainer.prediction_step.. s2&]&]TQQ\H\H\qH\H\H\H\&]&]rr)rrallrrr rrrrrPrUrno_gradrrrr"rTrrrrrr>rrrr)rPrrrr has_labelsrloss_without_labelsr raw_outputsr logits_mbrrrr%s ` ` rrzTrainer.prediction_step] s<"$"233q88UUcAvAvAvAveieuAvAvAv>v>v jj//  .K!$"233q8H[%%f--  tz8,, !%dj&79VYjXkll    , "5(W(W(W(WdFV(W(W(W#W#WXXF6{{aF ]__( )( )&((' ).uf== :!4:!+t444"-f"5$))o)o)o)o 8I8I8K8K)o)o)o$o$o "-a.$/O "..007799==??D.y99FFD!+t440$))d)d)d)d 8I8I8K8K)d)d)d$d$d $/ .y99FF)!4)::<<-1-I-I6(TXT]Td-e-e*(,(9(9!6$Se):)) g  ;;==--//D!'400-!&&h&h&h&hW]]__&h&h&h!h!h!(D::<<22"'%//&//222222222222222!'400)!&&]&]&]&]W]]__&]&]&]!]!]!(Q( )( )( )( )( )( )( )( )( )( )( )( )( )( )( )T  &$% %v&& v;;!  AYFff%%s]"D:N,>J& N,&J* *N,-J* .B N,; M N,M N,M AN,,N03N0rc ||}|||jj|t |jt jjjtfr|s|j j }| dsd|} |j ||nK#t$r>}td|dt|d|d|d}~wwxYw|S)zSRun evaluation, report to HP search, and step ReduceLROnPlateau/GreedyLR if needed.reval_9The `metric_for_best_model` training argument is set to 'W', which is not found in the evaluation metrics. The available evaluation metrics are: zX. Please ensure that the `compute_metrics` function returns a dictionary that includes 'zM' or consider changing the `metric_for_best_model` via the TrainingArguments.N)r_report_to_hp_searchr8rrr&rr8rcr7rrmrrKeyErrorrr)rPrjrlrrmetric_to_checkexcs rrzTrainer._evaluate s?--,@-AA !!%)?III t(5;+C+UW_*` a a " #i=O"--g66 <";/";"; !&&w'?@@@@   `P_``=A',,..=Q=Q``n}```   s B22 C:<9C55C:c|j||jtjkr|j}nj|jtjkr0ddl}|j}n%|jtj krddl }|j j }|j | |nd|}tj|jj|}n |jj}|S)zIReturn the output directory, accounting for hyperparameter search trials.Nrzrun-)r;rZOPTUNAnumberRAYray.tunetune get_context get_trial_idWANDBwandbrunidr:r5rzrrr)rPrjrun_idrayrrun_namerun_dirs rrzTrainer._get_output_dir s  ! -%2C%)???'?+>>>--//<<>>'?+@@@ .2l.Ft||E***OSYOOHgll49#7BBGGi*Grc.td|jj}|j||||}t j||}| |d|j j tj tjtjfvr|jjrt#rt%jdnO|j jt*jkrt/jn!t3rt5jtd|jj}t j||}t j|r ||j_|j js?||||| ||j j!rd|j"j#|j$gzDD]|}|j%j&} |} tO|jj(| tRr&|jj(| *| m| |jj(| <}|j+t j|tX|j j-r|.||j j!r)t_||j j0|jjddSdS) zSSave model checkpoint, optimizer, scheduler, scaler, RNG states, and trainer state.-N)rjT_internal_callrlc<g|]}t|t|Srrrs rrz,Trainer._save_checkpoint..& s:Q[\^`oQpQpr)rrr use_mtime)1rVr8rr;rrr5rzr save_modelrrjr^STEPSEPOCHrkbest_global_steprr rendezvousr rur rbarrierrreexistsrr_save_optimizer_and_scheduler _save_scaler_save_rng_stater4r-rr7rrrrrr; save_to_jsonr{r2_push_from_checkpointrlr) rPrrjcheckpoint_folderrrbest_checkpoint_folderbest_checkpoint_dirrcb_namecb_states rr5zTrainer._save_checkpoint s 5OOtz7MOO  ! )em OO   &&U&33W\\'+<==   4888 I # (:L sn# 9 * 6"i=O"--g66 <";/";"; &7    ^P_^^=A',,..=Q=Q^^^  &*Y%@MrzzbgHz%-:>):U)gv[`af[g[g &x dj&<== *)5 &9*|/A>>>  ! ^  &2D4DdjR\  `t`v`v     &2D4DdnVZV`bl    Y " ^ Jt~0022BGLL^4\4\ ] ] ])-(A) *  8K K G % I ! 1. 12O 1+,, 1 (555 e 4,779927<< Tb;c;cddd e e e e e e e e e e e e e e e  0 0 0 0 0 1 1 1 1 1 1s&AFF  F AQ$$Q(+Q(c  |jj}n#t$rYdSwxYw|dStrt jdt jd5}t j|jj tj |tt|dddn #1swxYwY|jjrtst jd5}t#j|jj tj |tdddn #1swxYwYt|dSdSdS)z-Save the gradient scaler state if one exists.Nsaving_scaler_stateTrJ)rscalerrrrr r rSrCrPr5rzr SCALER_NAMErRrr4r)rPrr`rZs rr$zTrainer._save_scaler s %,FF    FF  > F ! # # 5 M/ 0 0 0(555 5(/::<c>cddd#O444 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 9  1)?)A)A 1(555 h 4+2==??jZeAfAfggg h h h h h h h h h h h h h h h  0 0 0 0 0 1 1 1 1s2  A$C  CCAE&&E*-E*c  ||j}tjt}tjt }tjt }tjt}tjt}tjt}tjt} tj oqtfdtj DpDtjtjtd} tj r fdtj Dng} | r|jst#ddtd|||| ||fDs| s| st#dt$d d tj|rNt)j|} | j} | 1| t.kr&t$d | d t.d tj|s!tj|s| rt3rt5jtdddS|jr2t9|jjj|j|fitAdStj|r"tBj"#|d}n%tItEj%|dd}|&|d}~|'|dStQ|r:tS|drtS|drtj*r|j+}tY|dkrt$d|d}| rX| D]>}tj|}|-||||k?|.|dS|-|ddSt$dt ddSt$dt^ddSta|t3}t3s|'|dSdS)z/Load model weights from a checkpoint directory.Nc3K|]J}tjtj|?t|vVKdSr)r5rzisdirrFSDP_MODEL_NAMEr folder_nameris rrz0Trainer._load_from_checkpoint.. sa7==.Dk!R!RSS;.rz.binc g|]}tjtj|?tjtj|t sCtjtj|t |Sr)r5rzrdrrryrxrfs rrz1Trainer._load_from_checkpoint.. s   7==.Dk!R!RSS  GNN27<<0F Ui#j#jkk   w~~bgll3I;Xq&r&rss     rzCheckpoint found at z* is only supported when using PyTorch FSDPc3TK|]#}tj|V$dSr)r5rzr)rfs rrz0Trainer._load_from_checkpoint.. sD  q!!      rz!Can't find a valid checkpoint at zLoading model from rz9You are resuming training from a checkpoint trained with z- of Transformers but your current version is zJ. This is not recommended and could yield to errors or unwanted behaviors.Frztagrload_optimizerrrT map_location weights_onlyactive_adapters load_adapterr zFMultiple active adapters detected will only consider the first adapterr) is_trainablejThe intermediate checkpoints of PEFT may not be saved correctly, consider using a custom callback to save i in corresponding saving folders. Check some examples here: https://github.com/huggingface/peft/issues/968Could not load adapter model, make sure to have PEFT >= installedstrict)1rr5rzrrzryrxrr~r}r|rdrlistdirrrerrrrrfrom_json_filetransformers_versionrrgrrerirrr8rr safetensorsr load_filerloadload_state_dict_issue_warnings_after_loadrarr"rqrrr set_adapterr,rj)rPrir config_fileadapter_weights_fileadapter_safe_weights_file weights_fileweights_index_filesafe_weights_filesafe_weights_index_file is_fsdp_ckptadapter_subdirsrcheckpoint_versionrP load_resultrqactive_adapter subdir_namepeft_ids ` rrxzTrainer._load_from_checkpoint s =JEgll#9;GG !w||,BDXYY$&GLL1GIb$c$c!w||$:LII W\\*@BTUUGLL)?ARSS"$',,/EG^"_"_w}}%;<<  #%:.D#E#E   ^ w~~bgll+AoC[C[C[\\]] *w}}344     #%:.D#E#E       x 4 xv4Jvvvww w   !%&+(-       [ [ [YAWYYZZ Z C*@CCCDDD 7>>+ & & %4[AAF!'!< !-2D 2S2S=Pb==@K=== 7>>, ' ': =27>>:K+L+L: =P\: =&(( =*/\5af% =$*6$*  +,, 7>>"344a!,!2!>"899&+&;O?++a//'oppp%4Q%7N&f+:ssK&(gll3I;&W&WG!..w S^bpSp.rrrr)).99999**+A>`d*eeeeeNNbDXbbb vZjvvvwwwww2%9OXoXqXqrrrK*,, =// <<<<< = =rc td|jjd|jjdt j|jjt}t j|jjt}t j|jjt}t j|jjt}tr|j n|j}|jr6t!|j |jjt#|j dS|jr$rC} |j |j!r+d |j |j"j#d} t?| | d} ~ wwxYwd dl$m%} | gg}ntdtdd}ntdtLdd}n|t j'|r"tPj)*|d} n%tWtSj,|dd} |-| d}ts|r|.|dSdSdSt jt j|jjt^sLt jt j|jjt`rOtc||jjt}ts|.|dSdStd|ddS)zFLoad the best model found during training based on the tracked metric.zLoading best model from z (score: z).rTFrkrqrrrr zCDetected multiple active adapters, will only consider the first onez0When using prompt learning PEFT methods such as z, setting load_best_model_at_end=True can lead to errors, it is recommended to set this to False and to load the model manually from the checkpoint directory using PeftModel.from_pretrained(base_model, ) after training has finished.N)_IncompatibleKeysrtrurvrwrrrnrxz#Could not locate the best model at zi, if you are running a distributed training on multiple nodes, you should activate `--save_on_each_node`.)2rrr8rr1r5rzrrr}ryrxrrrr r%rarrrrrr"rerirrqrrgrrr peft_configis_prompt_learning peft_typer\torch.nn.modules.modulerr,rr}rr~rrrrr|r~rj) rPbest_model_pathbest_safe_model_pathbest_adapter_model_pathbest_safe_adapter_model_pathrrhas_been_loadedrrmsgrrPs rrzTrainer._load_best_modelq s ttz/OttZ^ZdZptttuuu',,tz'GVV!w||DJ,LN_``"$',,tz/OQe"f"f')w||DJ4TVo'p'p$&=&?&?O""TZ  $_  %" 0'5dj'A'A#A        !Y ) &2  0  '(( KKK GNN? + +Q w~~233Q w~~566Q w~~:;; Q  #O&((= A*9$!#( "%((3Ku&788%0WUN=[=[%0).)>q)Au45599"NN+pqqq7>>*ABB4bgnnUqFrFr4* % 2 24:3SUc d d d d#/ * * *#(#4^#D#W !*)8+0+<^+L+V+\)8)8)8%(+7s*;*;$D$) *RQQQQQ*;*;B*C*CKK"NN!jL`!j!j!j /4OOsWgsss+0w~~&:;;h%0%6%@%@AU^c%@%d%d 0222%*Zebf%g%g%g #("7"7 E"J"JK.00A_A33K@@@@@AAAA W^^BGLL)IKbcc d d hjhohvhv GLL9;M N Ni i 2tz7@W@Y@YK+,, =// <<<<< = = NNPoPPP     s4 L M">MM"rcl|dS|jjdkrp|jj}tj|d|d}tj|s td|ddSn[tj|d}tj|stddSt5ttj |d }dddn #1swxYwYtj|d tj |d tj |d t%rt'j|d|jjt*jk}tjrt3dtj||t5rt3dtj||t9rt3dtj||t=rt3dtj||tArt3dtj!||dSdS)z9Restore random number generator states from a checkpoint.Nr r<r=z$Didn't find an RNG file for process zr, if you are resuming a training that wasn't launched in a distributed fashion, reproducibility is not guaranteed.r;zDidn't find an RNG file, if you are resuming a training that was launched in a distributed fashion, reproducibility is not guaranteed.Trpr5r6rrCUDANPUHPUMLUMUSA)"rrrr5rzrrrrrTrrrr>setstaterK set_state set_rng_staterrr rur rrrUrr7rr8rr9rr:)rPrrrng_filecheckpoint_rng_stateis_distributeds rrzTrainer._load_rng_state s   F 9 ! # # I3Mw||J0P]0P0P0PQQH7>>(++  c=ccc  w||J@@H7>>(++  B ^^ K K $ & & &#(:hT#J#J#J  K K K K K K K K K K K K K K K ,X6777 09::: ""#7#>??? ! # # :  1%8 9 9 90L4LL : " " $ $ _ $VUZ9M~ ^ ^ ^ ! # # ] $UEI7K^ \ \ \ ! # # ] $UEI7K^ \ \ \ ! # # ] $UEI7K^ \ \ \ " $ $ _ $VUZ9M~ ^ ^ ^ ^ ^ _ _s.%DD#&D#c  dS|jrt|jtst jd5}t |jtj tj tddddn #1swxYwYt|dStr:t!jtj t"dzntj tj t"ptj tj t&pKtj o,t+fdtjD}|jrGt!jtj d|jjdt"n|}|rtj tj trRt5r|jrnt tj tj d |jjd |jjdt"d d }|d }nHt tj tj t"d d }t jd5}t tj tj td d }dddn #1swxYwYt|t9j||jjt9j||jj|j||j|dStr fd}|j !|n|jjdkr |jjnd }|j"r. ss'7==j+)N)NOO*0055a8KGrz rank*-of-rrrHrrnr%c|tjtjt ddS)NTr)rrerr5rzrrR)modoptrs r opt_load_hookz.opt_load_hookPs=++CHRW\\*n5]5]gk,l,l,lmmmmmrr ))r rr&rr rSrrrrr5rzrrTrRrglobrRrrrdrrzrNrrrrrsend_cpu_data_to_devicerr%rregister_post_step_hookrrrr8rrr)rPrrZcheckpoint_file_existsoptimizer_statelr_scheduler_staterros ` rrz%Trainer._load_optimizer_and_schedulers!   F  $ d/1JKK 5,D999_,...%55 27<< N#K#KZ^___ $O444 F'(( DIbgll:~>>E F F Frw||JGGHH 7>>"',,z;M"N"NOO GMM*--+-:j+A+A $* (DIbgll:/b49;O/b/bR`/b/bcc d d d'  "A 5bgnnRW\\*n5]5]&^&^A 5%''? 5.,...&+j &(sty/F(s(sDIL`(s(scq(s(s&+%) '''O'6k&BOO,...&+j Z@@ucg'''O,D999_,...). Z@@ucg***& $O444*?DI>}MMMM 8 , N J&  344 122266!J " Z H HWcrv ,D999_,...%55 27<< N#K#KZ^___ $O44444CA 5A 5A 5A 5s8A B((B,/B,A N((N,/N,:A V&&V*-V*c |dStjtj|t}|rtt rt jd5}ttj tj|tdd}dddn #1swxYwYt|tj ||jj|jj|dSt jd5}t|jjtj tj|tddddn #1swxYwYt|dSdS)z If scaler state exists, load it.NTrJrrnr)r5rzrrrarr rSrrrrRrrrrrr`r)rPrrrZ scaler_states rrzTrainer._load_scalerps   F!# Z0U0U!V!V ! 5&'' 5,D999_,...#(: Z==E`d$$$L $O444*<9IJJJ '77 EEEEE,D999_,...$+;; 27<< K#H#HW[\\\ $O44444% 5 5s&-A CC C1A%F""F&)F&c |jjsdSg}g}|jj|jgz}|jjD]T\ }t|ts|g}t fd|Dr fd|D}t||D]\}}| di}| di} t|d i|} | D]\} } t| | | t|tr| |_n|| |jt| t$d?| Vt)|dkr1t$dd |d |D]}|j|dS) zLIf callback states exist and were passed in, restore their states if enabledNc38K|]}|jjkVdSrrrrcallbackstored_callbacks rrz/Trainer._load_callback_state..s.eeh8%./Aeeeeeerc4g|]}|jjk|Srrrs rrz0Trainer._load_callback_state..s1!)8CUC^bqCqCqHCqCqCqrr attributeszPContinuing training from checkpoint, restoring any callbacks that were passed inrzPCheckpoint included callbacks not included in current configuration. Ignoring. (r ror)r'restore_callback_states_from_checkpointr-rr7r8rrTrrrziprr rrAr;remove_callbackrrrrgrr.)rP not_found new_callbacksoriginal_callbacksrt duplicatesr callback_datarr new_callback attributer\rs @rrzTrainer._load_callback_statesVy@  F  !2< ~M%)Z%B%H%H%J%J 2 2 !OTdD)) veeeeRdeeeee 2-? 03:t/D/D N N+Hm(,,VR88D!.!2!2<!D!DJ#14>>#9#9D#9#9L,6,<,<,>,>@@( 5 i????!(N;;;'3 %,,\:::)99$|:L:LMMMM noooo  1111 y>>A   NNzcgclclmvcwcwzzz   & 9 9H  ! . .x 8 8 8 8 9 9rrct|jdkr|jjft|jjdkrIt |jt |jjkr|jn#t d|jdt|jdkr%t d|jddSdS)zGLog warnings for missing or unexpected keys after loading a checkpoint.rNz8There were missing keys in the checkpoint model loaded: rz;There were unexpected keys in the checkpoint model loaded: ) r missing_keysr_keys_to_ignore_on_saver tie_weightsrrgunexpected_keys)rPrs rrz"Trainer._issue_warnings_after_loads {' ( (A - - 2> :;;a?? 011S9[5\5\\\ &&((((uZeZruuuvvv {* + +q 0 0 NNlkNilll      1 0rrcz| |jj}tr/t|j|j|j|j|j|ntrtj |d|j }|jj r|||ttj|dn|jridt)|jjjjvrB|j|j}|jj r|||n|jr dt5t7j|j jjv}|j j!"di"d dd k}|r2tG|jr|r|j $d }n|j|j }|jj r|||n#tJ$rtL'd |jj r||itQ|jj |tRtTg|j |Yn%wxYw|jj r|||jj+r%|s%|+d |jj,dSdSdS)z Will save the model, so you can reload it using `from_pretrained()`. Will only save from the main process. NTrrPzuser_content.ptFULL_STATE_DICTrNzero_optimizationstagerrOz| stage3_gather_16bit_weights_on_model_save=false. Saving the full checkpoint instead, use zero_to_fp32.py to recover weightsz Model save)commit_messagerevision)-rrrr-rrrrNrr5r6rrPr4_saverrzrtouchrrr8rrget_state_dictr rrrrXrrrrrra$_zero3_consolidated_16bit_state_dictrrrgrSrr}r2 hub_revision)rPrrrPr\zero3_shardings rrzTrainer.save_modelsQ  -J ! # #* #  DIt'79NPTPkmw    % & && # K T 2 2 2 2+6688Jy$ > :* === j*;<< = = C C E E E E  ! # C(8(>(J(Z$[$[[[!-<>>>> ?Y " # JJz " " " 9  [ [   L49CY  Z Z Z Z Z [ [ [ [s&C2IBK$#K$rPc|||n |jj}tj|dtd|t stfn ttf}t|j |s||j }t|j |j d|r7|j |j d||n~tdtj|tj|t(d d i n|j |||j|j|na|jZt/|jd rE|jj9td |jj|t!j|jtj|t4dS)zHSave model weights, configuration, and processing class to `output_dir`.NTrzSaving model checkpoint to FrrzETrainer.model is not a `PreTrainedModel`, only saving its state dict.rpt)metadata tokenizerzWSaving Trainer.data_collator.tokenizer by default as Trainer.processing_class is `None`)rrr5r6rrrr3rrrrPrr4save_pretrainedr}r save_filerzrr}rrrrrCTRAINING_ARGS_NAME)rPrrPsupported_classess rrz Trainer._saves!$.#9ZZty?S  J.... >*>>???6G6I6Ik_..P_ajOk$*&788 J!!Z2244 $*77 W\7]]_pqq  --djU-SScc:d cddd!++ Z9J K KW_aeVf, J & &zj & I I I  ,  ! 1 1* = = = =   **K88 +",8 KKq r r r   ( 8 8 D D D  49bgll:7IJJKKKKKrr7c|jj|jj|d<|jjdkrJ|jj|d<|9|jj|jz }|td||i|d|jji}|jj ||j |j|j|j ||_ dS)a8 Log `logs` on the various objects watching training. Subclass and override this method to inject custom behavior. Args: logs (`dict[str, float]`): The values to log. start_time (`Optional[float]`): The start of training. Nrr`rr) num_tokensr)r8rrrrrrrpr log_historyr;r-on_logr7)rPr7rcurrent_session_num_tokensrs rrz Trainer.log's :  ' J,DM 9 2d : :,0J,LD( )%-1Z-MPTPs-s* M':Jdeeefff9D9&$*"899 %%f---,33DItz4|jjtjkra|jxjt |jg|jj z c_d|_dS|jxj|jz c_d|_dS)zGStore the number of floating-point operations that went into the model.rrN) rr rur r8rrKr<rr=rers rrzTrainer.store_flos?s 9 "l&> > > J ! !-t/@.A$)JZ[[[__aaffhh  ! !!"D    J ! !T%6 6 ! ! !D   rct|jddx}|vrNt|jdr9d||z|jdzSdS)a For models that inherit from [`PreTrainedModel`], uses that method to compute the number of floating point operations for every backward + forward pass. If using another model, either implement such a method in the model or subclass and override this method. Args: inputs (`dict[str, torch.Tensor | Any]`): The inputs and targets of the model. Returns: `int`: The number of floating-point operations. rr^num_parametersT)exclude_embeddingsr)rrrr2r)rPr main_inputs rrzTrainer.floating_point_opsJsz"$*.?MM MJRX X X]d J(^ ^ Xvj)//111DJ4M4Mae4M4f4ff fqrtokench|sdS|jj1t|jjj}n |jj}||n |jj}t |||jj d}|j |_d|_ dS)zE Initializes a git repo in `self.args.hub_model_id`. NT)rprivater) rrr1rrabsoluter hub_tokenr create_repohub_private_reporepo_idpush_in_progress)rPr repo_namerepo_urls rr3zTrainer.init_hf_repo_s ))++  F 9 ! )TY122;;==BII .I* 0C88''  Hbmq'rr$, $rlanguagelicensetagsrfinetuned_fromtasks dataset_tags dataset_argsc :|sdStj|jjd} d} tj| rtj| j d} | dk} tj| j j } |8| 6t|tr|g}| D]}||vr||tj||||||||||  }|}t%| d5}||dddn #1swxYwY| r>|j|j|jjdSdS)a Creates a draft of a model card using the information available to the `Trainer`. Args: language (`str`, *optional*): The language of the model (if applicable) license (`str`, *optional*): The license of the model. Will default to the license of the pretrained model used, if the original model given to the `Trainer` comes from a repo on the Hub. tags (`str` or `list[str]`, *optional*): Some tags to be included in the metadata of the model card. model_name (`str`, *optional*): The name of the model. finetuned_from (`str`, *optional*): The name of the model used to fine-tune this one (if applicable). Will default to the name of the repo of the original model given to the `Trainer` (if it comes from the Hub). tasks (`str` or `list[str]`, *optional*): One or several task identifiers, to be included in the metadata of the model card. dataset_tags (`str` or `list[str]`, *optional*): One or several dataset tags, to be included in the metadata of the model card. dataset (`str` or `list[str]`, *optional*): One or several dataset identifiers, to be included in the metadata of the model card. dataset_args (`str` or `list[str]`, *optional*): One or several dataset arguments, to be included in the metadata of the model card. Nz README.mdF library_namepeft) rrrrrrrrrw)rr5rzrrrr"rrrtrrrrr;r2 from_trainer to_model_cardopenwriterr4rcreate_or_update_model_card)rPrrrrrrrrrmodel_card_filepathis_peft_libraryr existing_tagsrltraining_summary model_cardrjs rcreate_model_cardzTrainer.create_model_cardqsJ))++  F gll49+?MM 7>>- . . )$>*=>>CGGWWL*f4O&N+>??DIMM$=dC((" 6D())C$ C(((*7 !)%%    &3355 %s + + q GGJ                    h   ) )$* 5 5 Q QRVR[Rf g g g g g h hs.EEEEnd of trainingrblockingrc |j|j|j|j|dd}|\|jjrP|jjt|jj j }n%|jj dd}||n |jj }|j| ||d|sdSt!|jdddd |vrg|d <t%|d t&r |d g|d <|jjD]'}||d vr|d |(|jd d|i|| |jj}|t3|j|jj ||| d t6d g| S)u Upload `self.model` and `self.processing_class` to the 🤗 model hub on the repo `self.args.hub_model_id`. Parameters: commit_message (`str`, *optional*, defaults to `"End of training"`): Message to commit while pushing. blocking (`bool`, *optional*, defaults to `True`): Whether the function should return only when the `git push` has finished. token (`str`, *optional*, defaults to `None`): Token with write permission to overwrite Trainer's original args. revision (`str`, *optional*): The git revision to commit from. Defaults to the head of the "main" branch. kwargs (`dict[str, Any]`, *optional*): Additional keyword arguments passed along to [`~Trainer.create_model_card`]. Returns: The URL of the repository where the model was pushed if `blocking=False`, or a `Future` object tracking the progress of the commit if `blocking=True`. rN/r)rTr model_tagsrr-*r folder_pathrr run_as_futureignore_patternsrr)r- on_push_beginrr8r7rr4r1rrrrrr3rrrrrrrr;rrrr upload_folderrV)rPrrrrrr model_tags rr2zTrainer.push_to_hubs 6 ++DItz4<PPPZZ d33  $)"7 y%-!$)"677< !Y399#>>rB * 0C   $   E  * * * t,,,))++  F 4:|T 2 2 >V##!#v&.#.. 2"(.!1v!Z2 5 5 F6N226N)))444??*????  y-H !!###xx%%% ,)&,!&;#?#?#?@&   rr(c |r|jjtjkrdS|jjs"|j|jsdS|j |j|j |j |jj }tttt g}t"t$fD]}t&j||}t&j|r||t1|5}t3j|}dddn #1swxYwYt9t;|d}||tAr'|tBtDtFg|D]} t&jt&j|| rQtIj%t&j|| t&j|| |j&|j&'|tQj)|jt&j|tT|jj+tXj-krd|j j.} ndt_|j j0} tc2|j3|| |jj4ddtjdg|jj6} | g} |jjtj7tj8fvr|jjtj7krd nts|j:} tc2|j3|| | d z|jj4d|jj6 }| ||j|jrtw| |_dS|jj<| dS) zDPush model and checkpoint files to the Hub from a checkpoint folder.N weight_mapzTraining in progress, step zTraining in progress, epoch Trrrzlast-checkpointz , checkpoint)rr path_in_reporrrr)=rr hub_strategyr[ENDhub_always_pushris_doner-rr8r7rrzr{rr}r~r|r5rzrrr;r jsonloadsreadrrrextendrrwryrxrcopyrrrrCrrjr^rrrrrrr1rrVr CHECKPOINTALL_CHECKPOINTSrrrjobs)rPr(rmodeling_files index_file index_pathrjindex shard_files modeling_filermodel_push_job push_jobsr"checkpoint_pushs rr'zTrainer._push_from_checkpoint s))++ ty/E/X/X Fy( T-B-NW[WlWtWtWvWv-N F ++DItz4<PPPY) %'=|M^_-/FG 3 3J&7DDJw~~j)) 3%%j111*%%1 Jqvvxx00E111111111111111"3u\':'A'A'C'C#D#DEE %%k222    j  ! !#68LNg"h i i i+ u uMw~~bgll+Y%Z Z Z%)Y%;{?U%U%U!![_`q[r[r[w %hh44)-)->i)"/5O   _ - - -  (D,A,I,I,K,K ($29$=$=D ! ! !  ! & - -i 8 8 8 8 8s#'EE E ct|dsdS|jN|js7td|jdSdSdS)z5Wait for any in-progress push to the Hub to complete.rNz\Waiting for the current checkpoint push to be finished, this might take a couple of minutes.)rrr&rrwait_until_doners rrzTrainer._finish_current_pushRsst/00  F  ,T5J5R5R5T5T , KKv w w w  ! 1 1 3 3 3 3 3 - , , ,rminimizehp_spacez optuna.Trialcompute_objectiven_trials directionrr:c J|t}t|}t|}|||_|jt d||jn||_||_ |tn||_ |j |||fi|} d|_| S)a Launch a hyperparameter search using `optuna` or `Ray Tune`. The optimized quantity is determined by `compute_objective`, which defaults to a function returning the evaluation loss when no metric is provided, the sum of all metrics otherwise. To use this method, you need to have provided a `model_init` when initializing your [`Trainer`]: we need to reinitialize the model at each new run. This is incompatible with the `optimizers` argument, so you need to subclass [`Trainer`] and override the method [`~Trainer.create_optimizer_and_scheduler`] for custom optimizer/scheduler. Args: hp_space (`Callable[["optuna.Trial"], dict[str, float]]`, *optional*): A function that defines the hyperparameter search space. Will default to [`~trainer_utils.default_hp_space_optuna`] or [`~trainer_utils.default_hp_space_ray`] depending on your backend. compute_objective (`Callable[[dict[str, float]], float]`, *optional*): A function computing the objective to minimize or maximize from the metrics returned by the `evaluate` method. Will default to [`~trainer_utils.default_compute_objective`]. n_trials (`int`, *optional*, defaults to 100): The number of trial runs to test. direction (`str` or `list[str]`, *optional*, defaults to `"minimize"`): If it's single objective optimization, direction is `str`, can be `"minimize"` or `"maximize"`, you should pick `"minimize"` when optimizing the validation loss, `"maximize"` when optimizing one or several metrics. If it's multi objectives optimization, direction is `list[str]`, can be List of `"minimize"` and `"maximize"`, you should pick `"minimize"` when optimizing the validation loss, `"maximize"` when optimizing one or several metrics. backend (`str` or [`~training_utils.HPSearchBackend`], *optional*): The backend to use for hyperparameter search. Will default to optuna or Ray Tune, depending on which one is installed. If all are installed, will default to optuna. hp_name (`Callable[["optuna.Trial"], str]]`, *optional*): A function that defines the trial/run name. Will default to None. kwargs (`dict[str, Any]`, *optional*): Additional keyword arguments for each backend: - `optuna`: parameters from [optuna.study.create_study](https://optuna.readthedocs.io/en/stable/reference/generated/optuna.study.create_study.html) and also the parameters `timeout`, `n_jobs` and `gc_after_trial` from [optuna.study.Study.optimize](https://optuna.readthedocs.io/en/stable/reference/generated/optuna.study.Study.html#optuna.study.Study.optimize) - `ray`: parameters from [tune.run](https://docs.ray.io/en/latest/tune/api_docs/execution.html#tune-run). If `resources_per_trial` is not set in the `kwargs`, it defaults to 1 CPU core and 1 GPU (if available). If `progress_reporter` is not set in the `kwargs`, [ray.tune.CLIReporter](https://docs.ray.io/en/latest/tune/api/doc/ray.tune.CLIReporter.html) is used. Returns: [`trainer_utils.BestRun` or `list[trainer_utils.BestRun]`]: All the information about the best run or best runs for multi-objective optimization. Experiment summary can be found in `run_summary` attribute for Ray backend. NzXTo use hyperparameter search, you need to pass your model through a model_init function.) r"rZr!ensure_availabler;rrdefault_hp_spacer<r:rdr=r) rPr<r=r>r?rr:r backend_objbest_runs rhyperparameter_searchzTrainer.hyperparameter_search\sz ?/11G!'**8ACC $$&&&!( ? "j 9A8H 44h  >O>W!:!:]n";?49GGGG!%rct|j}|dkr|}n+|dkr||}ntd|td|S)zdInvoke `model_init` to get a fresh model instance, optionally conditioned on a hyperparameter trial.rr z'model_init should have 0 or 1 argument.Nz"model_init should not return None.)rkrr)rPrjmodel_init_argcountrs rrzTrainer.call_model_initst1$/BB ! # #OO%%EE A % %OOE**EEHII I =CDD D rc||_|j|dS|jtjkr||}nE|jtjkr|}|ddn|jtjkr|}|D]\}}t|j |st d|d9t|j |d}|t||}t|j |||jtjkr"td|j|jtjkrtd||jr|j jt)d|jdd lm}dd lm}||j j|j _|j j|j ||j j |j _t=|j j |_!|"dS) zRSet up training arguments and accelerator state for a hyperparameter search trial.NrzTrying to set zY in the hyperparameter search but there is no corresponding field in `TrainingArguments`.zTrial: zW&B Sweep parameters: z7For sweeps with deepspeed, `args.deepspeed` must be setr)DeepSpeedPlugin)HfTrainerDeepSpeedConfig)rX)#_trialr;rZrr<r rrrTrrrrgrr rrrar rrrrrrI#transformers.integrations.deepspeedrJrtrainer_config_processr|r _reset_stater?r@r)rPrjrarr\old_attrrIrJs rrtzTrainer._hp_search_setupss  ! )U] F  !_%; ; ;]]5))FF  #': : :F JJw % % % %  #'< < <F ,,.. + +JC49c** ,S,,,ty#t44H#&Xu-- DIsE * * * *  !_%; ; ; KK0%,00 1 1 1  !_%: : : KK888 9 9 9  $ .y"* !Z[[[   ( ( * * * 9 8 8 8 8 8 T T T T T T,D,DTYEX,Y,YDI ) I ) @ @ K K K)8diFc)d)d)dDI &    + + - - -"&!; //11111rrc:|j|dS|}|||_|jtjkrddl}t|dr|j sk| |j|| r>|j |j|j|j|jdSdSdS|jtjkrddl}t)j5}d}|jjr5|||jj|}|j|d<|j ||ddddS#1swxYwYdSdS)zHReport intermediate metrics to the active hyperparameter search backend.Nrstudy)checkpoint_dir objective)r)r;r+r=rSrZroptunarrQ_is_multi_objectivereport should_pruner-rrr8r7 TrialPrunedr r tempfileTemporaryDirectoryr4_tune_save_checkpointr  Checkpointfrom_directory)rPrjrrrTrtemp_checkpoint_dirrs rrzTrainer._report_to_hp_searchs  ! )U] F,,..//88  !_%; ; ; MMMug&& /u{/N/N/P/P / T^T222%%''/)66ty$*dl[[[,&,...  / / / /// #': : : OOO,.. @2E! <+Y..>Q.RRR!$!4!C!CDW!X!XJ'+~ $J???  @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @; :sA*FFFrRctj|td|jj}||d|jjr|j |jj d<|j tj|ttj|jtj|t"tj|jtj|t&dSdS)z@Save a checkpoint during a Ray Tune hyperparameter search trial.rTrrAN)r5rzrrVr8rrrr4r7rr&r{rrCr%rPrRr&rT)rPrRrs rr[zTrainer._tune_save_checkpoint sW\\.5J2e2eTZMc2e2eff   4888 9  a>Bl>P>P>R>RDJ )*: ; J # #BGLL=O$P$P Q Q Q Jt~0022BGLL^4\4\ ] ] ] Jt(3355rw||JP^7_7_ ` ` ` ` `  a arrc:|j|dS)ag Add a callback to the current list of [`~transformers.TrainerCallback`]. Args: callback (`type` or [`~transformers.TrainerCallback]`): A [`~transformers.TrainerCallback`] class or an instance of a [`~transformers.TrainerCallback`]. In the first case, will instantiate a member of that class. N)r-r.rPrs rr.zTrainer.add_callbacks! **844444rc6|j|S)aK Remove a callback from the current list of [`~transformers.TrainerCallback`] and returns it. If the callback is not found, returns `None` (and no error is raised). Args: callback (`type` or [`~transformers.TrainerCallback]`): A [`~transformers.TrainerCallback`] class or an instance of a [`~transformers.TrainerCallback`]. In the first case, will pop the first member of that class found in the list of callbacks. Returns: [`~transformers.TrainerCallback`]: The callback removed, if found. )r- pop_callbackras rrczTrainer.pop_callback%s$11(;;;rc:|j|dS)a Remove a callback from the current list of [`~transformers.TrainerCallback`]. Args: callback (`type` or [`~transformers.TrainerCallback]`): A [`~transformers.TrainerCallback`] class or an instance of a [`~transformers.TrainerCallback`]. In the first case, will remove the first member of that class found in the list of callbacks. N)r-rras rrzTrainer.remove_callback5s! --h77777rc"|jjdkS)z Whether or not this process is the local (e.g., on one machine if training in a distributed fashion on several machines) main process. r)rlocal_process_indexrs rrzTrainer.is_local_process_zeroBs y,11rcltrtjdkS|jjdkS)z Whether or not this process is the global main process (when training in a distributed fashion on several machines, this is only going to be `True` for one process). r)rrerrrrs rrzTrainer.is_world_process_zeroIs2 # $ $ #8::? "y&!++rct|ddtddS||}|jjt jkr&t|dr| dSdSdS)zJMove the model to the specified device, re-tying weights on XLA if needed.rNzZThe model is already on multiple devices. Skipping the move to device specified in `args`.r) rrrgrrr ruTPUrr)rPrrs rrzTrainer._move_model_to_deviceTs 5/4 0 0 < NNl    F   9 "l&6 6 675-;X;X 6         7 6 6 6r) NNNNNNNNNNrNN)r]Nr)NFN)NNN)NNNNN)FN)T)TN)NNr)Nr)F)NF) NNNNNNNNN)rTNN)NNr:r;NN)rjrkr]N)r __module__ __qualname____doc__trainer_pt_utilsrrrrrrrr3rModulervrr:r#r r9rrYr"rr@rUrr8 Optimizerr&LambdaLRr rr r2r\r'rrrrrrrrrrsrtSamplerrwrrrrrrr r LRSchedulerr staticmethodr7r5rrhr`rr|rrrrrrr}r(rAbstractContextManagerr)rrrrr>rPr r rrprqr{rzrrrrrrrXrr\rrrrr5r4r%r#r$rxrrrrrrrrrrrr3rr r2r'rrZrWrErrtrr[r.rcrrrrrrrrrsc^^B59)--1OSQU <@-1CG26dp^bei%~7~7*T1~7 $&~7$d* ~7 M ~7 O ~72  !   ~7S/12T9~7$d?~7">"2D"89D@~7(4/~7 %+/$6 8P8Y\`8``a!~7"#(U[-B(CT#s(^(S"TW["[#~7$(0u|0Lel0Z'[^b'b%~7~7~7~7@QQQQf94S>9999vjxjxjxjx\ j    (% % g 0D% PZ% % % % N  J    $ Kz Kc K K K K&LP!%)0000 0 gY (8(@@ADH 0  0d 0 0000d00$0%+JZJbeiJi0000>g%+:J:RUY:Y:\\\\"FJ!;!;)!;8;d !; !;!;!;!;F'''WZ]aWa'mu''''$ E E E E E EJJek.CJJJJZRV!!"%!27+2G$2N!  ! -!!!!<+<_W[E[glmprumugv\<  ry  T#Y     E@598<15 kk #d T 1k6k#3i$. k  kkkk^"&)--18<15 [M[M$J[M $&[M!$d [M 6 [M #3i$. [M [M[M[M[Mz> sCx>>>>>`'`'`'Dc c c J/H/H/Hj9= G!G!yG!S%,,,-G!"L3.5 G!  G!G!G!G!Z %8< W;W;yW;S%,,,-W; W; "L3.5 W; elC/0 0 W;W;W;W;r j.B    ((D4K(S]St(((( '+0^0^0^<%'$.0^y 0^ 6 0^  0^#3i$.0^0^t|0^ 0^0^0^0^f1&1581BG,1 tU\C'$.. /1111"<"T<"5<<"TYT`cfTfimTm<"<"<"<"|5<##5%,:L$ d3 s0B+B&C SRWR^adRdMdHe    \.Y\.(,S%,2D-D(E\. xc5<##5566 7\.\.\.\.|D %D 3=D sCc3d C7 8D D D D LY/@YSYYYY$SS S       d z\`O` lnlu    D@@@*___B]]]    =A(,!' aaS'\ 22T9a#Y%a a c5j  aaaaN-1(,!' MuMuMuMu#Tk Mu #Y% Mu  Mu MuMuMuMu`ek;t;t#;t26s)d2B;t^a;t ;t;t;t;tD)- g&g&yg&S%,,,-g&# g& #Y% g& u|d"EL4$79LL M g&g&g&g&Z % 5#3i$.  c5j  @%KPS&CbiC8^CcgCCCCJ$"d3:.>$"Gm$"rv$"$"$"$"L1i#1i$1i1i1i1if@1@1@1@1@1@1D1s1t1111.==C= TXHX=dh====BhhhhT+_#*+_+_+_+_+_Zf5d f5tf5f5f5f5P5sTz5d55556%9%9%9%9Ncd$8[8[S4Z8[8[Y]8[8[8[8[t%L%Ld %Ltd{%LVZ%L%L%L%LR__S%Z(_edl_d____0 " " " "c5<#3E.E)F3*%%#*%%%%%( $"'+!%%)(,/3*./3HhHh*HhtHhDIo$ Hh $J Hh d HhT#Y%HhDIo,HhtCy4'HhDIo,Hh HhHhHhHhX&7 # P P d P P Tz P * P  P P P P dC9sC9tC9C9C9C9J4444IMHL%/04:>OON+T#u*-==>EO$T#u*%5$6$=>EO O c? O &- O>*C/047O 4= OOOOb  %K WYW`    42424242l@;@CF@QUVY[`V`Qa@ @@@@6 aC aD a a a a 5T/%:_%L 5QU 5 5 5 5rrtrYrr collections.abcrrrrpathlibrtypingr r integrationsr r6rKsafetensors.torchr}rtorch.distributed distributedrhuggingface_hubr r packagingrrtorch.utils.datarrrrrr rconfiguration_utilsrdata.data_collatorrrr debug_utilsrrdistributed.fsdprr!feature_extraction_sequence_utilsrfeature_extraction_utilsr rEr!r"image_processing_utilsr#integrations.deepspeedr$r%r&r'r(integrations.ligerr)integrations.neftuner*r+integrations.peftr,integrations.tpur-r.r/loss.loss_utilsr0r1 modelcardr2modeling_utilsr3r4models.auto.modeling_autor5r6 optimizationr7r8processing_utilsr9tokenization_utils_baser:trainer_callbackr;r<r=r>r?r@rArBtrainer_optimizerrCrDrErFrmrGrHrIrJrKrLrMrNrOrPrQrRrSrTrU trainer_utilsrVrWrXrYrZr[r\r]r^r_r`rarbrcrdrerfrgrhrirjrkrlrmrnrorprqrrrs training_argsrtrurvrsrwrxryrzr{r|r}r~rrrrrrrrrrrrrrrrrrrutils.import_utilsrutils.quantization_configrr(r0utils.notebookrrtorch_xla.core.xla_modelcore xla_modelrtorch_xla.debug.metricsr?rrtorch_xla.runtimeruntimerF torch_xla XLA_VERSIONrrEtorch_xla.distributed.spmdspmdrH!smdistributed.modelparallel.torch modelparallelrerrrrrrrraccelerate.staterrrrrrrrrrraccelerate.utils.memoryrrrT get_loggerrrrr{rRrarrTrerrrrrs    7777777777%%%%%%%%   11111111cccccccccccccc111111\\\\\\\\\\<<<<<<<<KKKKKKKKGGGGGG<<<<<<````````666666322222FFFFFFFF//////[[[[[[[[[[::::::::&&&&&&9999999921111111,,,,,,<<<<<<                     "@KJJJJJJJJJ:)(((((999999)),>9888888 8OOO #))))))))))))))))))""""""444444*W];77=7=I_;`;``0/////////"\333333333[[[[[[[[[[?::::::::111111                      ;:::::?>>>>>>MMM  H % %)) $&  ]A ]A ]A ]A ]A ]A ]A  ]A ]A ]A r