tjP"NdZddlmZddlmZddlmZmZerddlm Z m Z m Z m Z m Z mZmZmZdZnBddlmZed Z ed Z ed Z ed Z ed Z edZdededefdZdZedeGddeZdgZdS)zxLSTM configuration.)strict)PreTrainedConfig)auto_docstringis_xlstm_available)BackendModeTypeChunkwiseKernelType DtypeTypeSequenceKernelTypeStepKernelTypeWeightModeTyperound_up_to_next_multiple_ofxLSTMLargeConfigT)Literal)traintrain_with_padding inference)chunkwise--native_autogradzparallel--native_autograd)float32bfloat16float16native_sequence__nativenative)singlefusedx multiple_ofreturnc8t||zdz |z|zS)z0Rounds up x to the next multiple of multiple_of.)int)rrs k/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/xlstm/configuration_xlstm.pyrr1s#Q_q([8KGHHHFzNX-AI/xLSTM-7b) checkpointcveZdZUdZdZdZeed<dZeed<dZ edzed<d Z eed <dZ edzed <d Z eed <dZ eed<dZeed<dZeed<dZeed<dZeed<dZeed<dZeed<dZeed<dZeed<dZeed<d Zeed!<d"Zeed#<dZeed$<d%Z e!ed&<dZ"eed'<d(Z#e!ed)<d*Z$eed+<d"Z%eed,<d-Z&eed.<d/Z'eed0<d1Z(e)ed2<dZ*eed3<d4Z+edzed5<d6Z,edzed7<d8Z-ee.ezdzed9<d:Z/eed;<fd<Z0e1d=Z2e1d>Z3e1d?Z4e1d@Z5dAZ6xZ7S)B xLSTMConfigaN num_blocks (int, optional, *optional*, defaults to 32): Number of blocks of the xLSTM model, use num_hidden_layers if None. num_heads (int, optional, *optional*, defaults to 8): Number of heads for the xLSTM Layer/Cell. use_bias (bool, optional, *optional*, defaults to `False`): Whether to use biases in the xLSTM model. norm_reduction_force_float32 (bool, optional, *optional*, defaults to `True`): Whether to force the float32 norm reduction op to be done in fp32 precision. add_out_norm (bool, optional, *optional*, defaults to `True`): Whether to add an output norm after the blocks before the LMHead. qk_dim_factor (float, optional, *optional*, defaults to 0.5): Scale factor for the query and key dimension. v_dim_factor (float, optional, *optional*, defaults to 1.0): Scale factor for the value dimension. chunkwise_kernel (ChunkwiseKernelType, optional, *optional*, defaults to `"chunkwise--native_autograd"`): Kernel type for chunkwise processing mode. sequence_kernel (SequenceKernelType, optional, *optional*, defaults to `"native_sequence__native"`): Kernel type for sequence processing mode. step_kernel (StepKernelType, optional, *optional*, defaults to `"native"`): Kernel type for step processing mode. mode (BackendModeType, optional, *optional*, defaults to `"inference"`): Operation mode (inference is needed for generation). chunk_size (int, optional, *optional*, defaults to 64): Internal chunk size. return_last_states (bool, optional, *optional*, defaults to `True`): If to return the last states / cache internally. Needed as True for generation. autocast_kernel_dtype (DtypeType, optional, *optional*, defaults to `"bfloat16"`): Kernel dtype for the states. inference_state_dtype (DtypeType, optional, *optional*, defaults to `"float32"`): Kernel dtype for states in inference. ffn_proj_factor (float, optional, *optional*, defaults to 2.667): Size factor of the post-up projection gated Feed Forward network. ffn_round_up_to_multiple_of (int, optional, *optional*, defaults to 64): Size factor round value of the post-up projection gated Feed Forward network. gate_soft_cap (float, optional, *optional*, defaults to 15.0): Gate soft cap scale. output_logit_soft_cap (float, optional, *optional*, defaults to 30.0): Output logit soft cap scale. weight_mode (`Literal`, *optional*, defaults to `"single"`): Whether parallel linear layers are separated or fused (single). max_inference_chunksize (int, optional, *optional*, defaults to 16384): Limit the chunk size for inference to save memory. Example: ```python >>> from transformers import xLSTMConfig, xLSTMModel >>> # Initializing a xLSTM configuration >>> configuration = xLSTMConfig() >>> # Initializing a model (with random weights) from the configuration >>> model = xLSTMModel(configuration) >>> # Accessing the model configuration >>> configuration = model.config ```xlstmi vocab_sizei hidden_sizeN embedding_dim num_hidden_layers num_blocks num_headsFuse_biasTnorm_reduction_force_float32tie_word_embeddings add_out_normgư>norm_epsg? qk_dim_factorg? v_dim_factorrchunkwise_kernelrsequence_kernelr step_kernelrmode@ chunk_sizereturn_last_statesrautocast_kernel_dtypeepsrinference_state_dtypegtV@ffn_proj_factorffn_round_up_to_multiple_ofg.@ gate_soft_capg>@output_logit_soft_capr weight_mode use_cacher pad_token_idr bos_token_id eos_token_idi@max_inference_chunksizec |j|jn|j|_|j|jn|j|_|j|jn|j|_|j|jn|j|_t jdi|dS)N)r)r*r,r-super __post_init__)selfkwargs __class__s r"rOzxLSTMConfig.__post_init__s/3/?/K4++QUQc373E3QT//W[Wg;?;Q;]!7!7cgcr-1_-H$//dNd'''''''r#c>t|j|jzdSNr;)r)rr)r5rPs r"qk_dimzxLSTMConfig.qk_dims*+  t1 1    r#c>t|j|jzdSrT)rr)r6rUs r"v_dimzxLSTMConfig.v_dims*+  t0 0    r#c |j|jzSN)rVr/rUs r" qk_head_dimzxLSTMConfig.qk_head_dims{dn,,r#c |j|jzSrZ)rXr/rUs r" v_head_dimzxLSTMConfig.v_head_dimszT^++r#ctrtdid|jd|jd|jd|jd|jd|jd|jd|j d |j d |j d |j d |j d |jd|jd|jd|jd|jd|jd|jd|jd|jd|jd|jd|jS|S)Nr(r*r-r/r0r3r4r1r5r6r7r8r9r:r<r=r>r?r@rArBrCrDrErM)external_xlstmrr(r)r,r/r0r3r4r1r5r6r7r8r9r:r<r=r>r?r@rArBrCrDrErUs r"to_xlstm_block_configz!xLSTMConfig.to_xlstm_block_configs  #??".. 11..    ".. .2-N-N#00".."&!6!6!% 4 4!,, YY!" ??#$$(#:#:%&'+&@&@'(HH)*'+&@&@+.!% 4 4/0-1,L,L14#0056'+&@&@78!,,9 >Kr#)8__name__ __module__ __qualname____doc__ model_typer(r!__annotations__r)r*r,r-r/r0boolr1r2r3r4floatr5r6r7r r8r r9r r:rr<r=r>r r?r@rArBrCrDrEr rFrGrHrJlistrKrOpropertyrVrXr[r]r` __classcell__)rRs@r"r&r&8s9 9 vJJK $M3:$$$s!Jd !!!IsHd)- $--- %%%%L$HeM5L%,H)HHH*CO'CCC"*K***'D/'''J####'19111C'09000"OU"""'))))M5#'5'''"*K***It L#*    L#*   +,L#S /D(,,,#(S((((((((  X   X --X-,,X,!!!!!!!r#r&N)rdhuggingface_hub.dataclassesrconfiguration_utilsrutilsrrxlstm.xlstm_large.modelrr r r r r rrr_typingrr!r&__all__rMr#r"rrs......33333377777777                    NNHIO! %89I !:;X&N./NII#I#IIIIN+,,,]]]]]"]]-,]@ /r#