tjRddlZddlmZddlmZddlZddlmZddlm Z ddl m Z m Z ddl mZddlmZdd lmZmZmZdd lmZmZdd lmZmZdd lmZmZdd lmZddl m!Z!m"Z"m#Z#ddl$m%Z%m&Z&ddl'm(Z(ddl)m*Z*Gddej+Z,Gddej+Z-Gddej+Z.dej/de0dej/fdZ1 d8dej+dej/d ej/d!ej/d"ej/dzd#e2d$e2d%ee!fd&Z3d'Z4d9d(Z5Gd)d*ej+Z6Gd+d,eZ7e"Gd-d.eZ8e"Gd/d0e8Z9e"Gd1d2e8eZ:Gd3d4ee8Z;Gd5d6ee8Zcttjt j||_||_dSN)super__init__nn Parametertorchonesweightvariance_epsilon)self hidden_sizeeps __class__s h/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/helium/modeling_helium.pyr#zHeliumRMSNorm.__init__0sB l5:k#:#:;; #cT|j}|tj}|ddd}|tj||jzz}|jtj|z|S)NT)keepdim) dtypetor&float32powmeanrsqrtr)r()r* hidden_states input_dtypevariances r.forwardzHeliumRMSNorm.forward5s#) %((77  $$Q'',,R,>>% Ht?T4T(U(UU  u}-- =AA+NNNr/cHt|jjd|jS)Nz, eps=)tupler(shaper))r*s r. extra_reprzHeliumRMSNorm.extra_repr<s& )**II$2GIIIr/)r)__name__ __module__ __qualname__r#r=rA __classcell__r-s@r.rr/se$$$$$$ OOOJJJJJJJr/rc eZdZUejed<d deffd Ze d dedzde dde dzde d e ffd Z ejed ZxZS)HeliumRotaryEmbeddinginv_freqNconfigct|j|_|j|_||_|jjd|_|j}|jdkrt|j}||j|\}|_ | d|d| d| ddS)N rope_typedefaultrIF) persistentoriginal_inv_freq) r"r#max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenrJrope_parametersrLcompute_default_rope_parametersrattention_scalingregister_bufferclone)r*rJdevice rope_init_fnrIr-s r.r#zHeliumRotaryEmbedding.__init__Cs "("@$*$B! 4[A!%!E >Y & &.t~>L+7< V+L+L($( ZeDDD 0(..2B2BuUUUUUr/rXz torch.deviceseq_lenreturnz torch.Tensorc|jd}t|ddp|j|jz}d}d|t jd|dtj|tj|z zz }||fS) a Computes the inverse frequencies according to the original RoPE implementation Args: config ([`~transformers.PreTrainedConfig`]): The model configuration. device (`torch.device`): The device to use for initialization of the inverse frequencies. seq_len (`int`, *optional*): The current sequence length. Unused for this type of RoPE. Returns: Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE). rope_thetahead_dimNg?rr1r4)rXr4) rSgetattrr+num_attention_headsr&arangeint64r5float)rJrXrZbasedimattention_factorrIs r.rTz5HeliumRotaryEmbedding.compute_default_rope_parametersSs&%l3fj$//c63EIc3c U\!S!5;???BB&X]XcBddgjj k )))r/cN|jddddf|jddd|j}|dddddf}t |jjtr|jjdkr |jjnd}t|d5||z dd}tj ||fd }| |jz}||jz} dddn #1swxYwY||j | |j fS) Nrr2rmpscpuF) device_typeenabledr1rfr_)rIrdexpandr@r5rX isinstancetypestrr transposer&catcosrUsinr4) r*x position_idsinv_freq_expandedposition_ids_expandedrkfreqsembrtrus r.r=zHeliumRotaryEmbedding.forwardqs!M$4-8>>@@GG HZ[\H]_acdeehhijiqrr ,QQQaaaZ 8 > > @ @'1!(-'E'Ek!(-[`J`J`ahmmfk  U C C C 5 5&,,..1F1L1L1N1NNYYZ[]^__E)UEN333C''))d44C''))d44C  5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 vvAGv$$cff17f&;&;;;s BE&&E*-E*r!NNN)rBrCrDr&Tensor__annotations__rr# staticmethodrintr?rdrTno_gradrr=rErFs@r.rHrH@slVV|VVVVVV &*+/"**t#*(*t* ~u$ % ***\*:U]__ < <_ < < < <