tj]UrddlmZddlmZddlZddlmZddlmZmZddl m Z m Z ddl m Z dd lmZmZmZdd lmZdd lmZmZdd lmZmZdd lmZmZddlmZmZddl m!Z!ddl"m#Z#m$Z$m%Z%ddl&m'Z'm(Z(ddl)m*Z*ddl+m,Z,Gddej-Z.edGddej-Z/Gddej-Z0dZ1edd:dZ2dej3d e4d!ej3fd"Z5 d;d$ej-d%ej3d&ej3d'ej3d(ej3dzd)e6d*e6d+e!e#fd,Z7ee2Gd-d.ej-Z8Gd/d0eZ9e$Gd1d2eZ:e$Gd3d4e:Z;e$Gd5d6e:e Z<Gd7d8ee:Z=gd9Z>dS)<)Callable)OptionalN)nn)ACT2CLSACT2FN)Cache DynamicCache)GenerationMixin)use_kernel_forward_from_hubuse_kernel_func_from_hubuse_kernelized_func)create_causal_mask)GenericForTokenClassificationGradientCheckpointingLayer)BaseModelOutputWithPastCausalLMOutputWithPast)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuple)maybe_autocastmerge_with_config_defaults)capture_outputs) ApertusConfigc$eZdZfdZdZxZS) ApertusMLPct||_|j|_|j|_t j|j|jd|_t j|j|jd|_t|j |_ |j dkr"td|j |_ dSdS)NFbiasxieludtype)super__init__config hidden_sizeintermediate_sizerLinearup_proj down_projr hidden_actact_fnrr(selfr+ __class__s j/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/apertus/modeling_apertus.pyr*zApertusMLP.__init__,s  !-!'!9y!143IPUVVV 4#94;KRWXXXV./   ' '!'*>>>DKKK ( 'cx||||SN)r0r2r/)r4xs r6forwardzApertusMLP.forward7s*~~dkk$,,q//::;;;r7)__name__ __module__ __qualname__r*r; __classcell__r5s@r6r"r"+sG ? ? ? ? ?<<<<<<epsreturnNcttjt j||_||_dS)z= ApertusRMSNorm is equivalent to T5LayerNorm N)r)r*r Parametertorchonesweightvariance_epsilon)r4r,rEr5s r6r*zApertusRMSNorm.__init__=sD l5:k#:#:;; #r7 hidden_statesc|j}|tj}|ddd}|tj||jzz}|j||zS)NT)keepdim) r(torIfloat32powmeanrsqrtrLrK)r4rM input_dtypevariances r6r;zApertusRMSNorm.forwardEs|#) %((77  $$Q'',,R,>>% Ht?T4T(U(UU {]--k::::r7cHt|jjd|jS)Nz, eps=)tuplerKshaperL)r4s r6 extra_reprzApertusRMSNorm.extra_reprLs& )**II$2GIIIr7)rD) r<r=r>floatr*rITensorr;r\r?r@s@r6rCrC;s$$$$$$$$$$;U\;el;;;;JJJJJJJr7rCc eZdZUejed<d deffd Ze d dedzde dde dzde d e ffd Z ejed ZxZS)ApertusRotaryEmbeddinginv_freqNr+ct|j|_|j|_||_|jjd|_|j}|jdkrt|j}||j|\}|_ | d|d| d| ddS)N rope_typedefaultraF) persistentoriginal_inv_freq) r)r*max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenr+rope_parametersrccompute_default_rope_parametersrattention_scalingregister_bufferclone)r4r+device rope_init_fnrar5s r6r*zApertusRotaryEmbedding.__init__Ss "("@$*$B! 4[A!%!E >Y & &.t~>L+7< V+L+L($( ZeDDD 0(..2B2BuUUUUUr7roz torch.deviceseq_lenrFz torch.Tensorc|jd}t|ddp|j|jz}d}d|t jd|dtj|tj|z zz }||fS) a Computes the inverse frequencies according to the original RoPE implementation Args: config ([`~transformers.PreTrainedConfig`]): The model configuration. device (`torch.device`): The device to use for initialization of the inverse frequencies. seq_len (`int`, *optional*): The current sequence length. Unused for this type of RoPE. Returns: Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE). rope_thetahead_dimNg?rrOr')ror() rjgetattrr,num_attention_headsrIarangeint64rRr])r+rorqbasedimattention_factorras r6rkz6ApertusRotaryEmbedding.compute_default_rope_parameterscs&%l3fj$//c63EIc3c U\!S!5;???BB&X]XcBddgjj k )))r7cN|jddddf|jddd|j}|dddddf}t |jjtr|jjdkr |jjnd}t|d5||z dd}tj ||fd }| |jz}||jz} dddn #1swxYwY||j | |j fS) NrrPrmpscpuF) device_typeenabledrOrzr')rar]expandr[rRro isinstancetypestrr transposerIcatcosrlsinr() r4r: position_idsinv_freq_expandedposition_ids_expandedrfreqsembrrs r6r;zApertusRotaryEmbedding.forwards!M$4-8>>@@GG HZ[\H]_acdeehhijiqrr ,QQQaaaZ 8 > > @ @'1!(-'E'Ek!(-[`J`J`ahmmfk  U C C C 5 5&,,..1F1L1L1N1NNYYZ[]^__E)UEN333C''))d44C''))d44C  5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 vvAGv$$cff17f&;&;;;s BE&&E*-E*r9)NNN)r<r=r>rIr^__annotations__r r* staticmethodrintrZr]rkno_gradrr;r?r@s@r6r`r`PslVV}VVVVVV '++/"**$*(*t* ~u$ % ***\*:U]__ < <_ < < < <