tjddlmZddlmZddlmZddlZddlmZddlm Z ddl m Z dd l mZmZdd lmZdd lmZmZmZdd lmZdd lmZddlmZddlmZmZmZm Z ddl!m"Z"m#Z#ddl$m%Z%m&Z&ddl'm(Z(ddl)m*Z*m+Z+m,Z,m-Z-ddl.m/Z/m0Z0ddl1m2Z2ddl3m4Z4ddl5m6Z6m7Z7edGddej8Z9Gddej8Z:Gdd ej8Z;Gd!d"ej8Z<Gd#d$ej8Z=d%Z>Gd&d'ej8Z?Gd(d)ej8Z@Gd*d+ej8ZAd,ZBed-dYd.ZCd/ejDd0eEd1ejDfd2ZF dZd4ej8d5ejDd6ejDd7ejDd8ejDdzd9eGd:eGd;e(e*fd<ZHeeCGd=d>ej8ZIGd?d@eZJe+GdAdBe&ZKe+GdCdDe&ZLGdEdFej8ZMe+GdGdHeKZNe+GdIdJeKeZOe+dKLeGdMdNe ZPe+dOLeGdPdQeZQe+dRLGdSdTeLZRe+dULGdVdWeLeZSgdXZTdS)[)Callable) dataclass)OptionalN)nn)initialization)ACT2FN)Cache DynamicCache)GenerationMixin)use_kernel_forward_from_hubuse_kernel_func_from_hubuse_kernelized_func)create_causal_mask)FlashAttentionKwargs)GradientCheckpointingLayer)BaseModelOutputWithPastBaseModelOutputWithPoolingCausalLMOutputWithPast ModelOutput)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tupletorch_compilable_check)maybe_autocastmerge_with_config_defaults)capture_outputs) AutoModel) AriaConfigAriaTextConfigRMSNormcTeZdZd deddffd ZdejdejfdZdZxZ S) AriaTextRMSNormư>epsreturnNcttjt j||_||_dS)z> AriaTextRMSNorm is equivalent to T5LayerNorm N)super__init__r Parametertorchonesweightvariance_epsilon)self hidden_sizer, __class__s d/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/aria/modeling_aria.pyr0zAriaTextRMSNorm.__init__6sD l5:k#:#:;; # hidden_statesc|j}|tj}|ddd}|tj||jzz}|j||zS)Nr#T)keepdim) dtypetor2float32powmeanrsqrtr5r4)r6r; input_dtypevariances r9forwardzAriaTextRMSNorm.forward>s|#) %((77  $$Q'',,R,>>% Ht?T4T(U(UU {]--k::::r:cHt|jjd|jS)Nz, eps=)tupler4shaper5r6s r9 extra_reprzAriaTextRMSNorm.extra_reprEs& )**II$2GIIIr:)r+) __name__ __module__ __qualname__floatr0r2TensorrGrL __classcell__r8s@r9r*r*4s$$$$$$$$$$;U\;el;;;;JJJJJJJr:r*c(eZdZdZfdZdZxZS)AriaProjectorMLPa! Feed-Forward Network module for the Aria Projector. Args: in_features (`int`): Input embedding dimension. hidden_features (`int`): Hidden dimension of the feed-forward network. output_dim (`int`): Output dimension. cttj||d|_tj||d|_t d|_dS)NFbiasgelu_new)r/r0rLinear linear_in linear_outr act)r6 in_featureshidden_features output_dimr8s r9r0zAriaProjectorMLP.__init__VsY ;eLLL)OZeLLL*%r:c|||}||}|SN)r]r[r\)r6r;s r9rGzAriaProjectorMLP.forward\s6 !>!>??  66 r:rMrNrO__doc__r0rGrRrSs@r9rUrUIsQ  &&&&& r:rUc6eZdZdZddedeffd Zd dZxZS) AriaCrossAttentionzv Aria Cross-Attention module. Args: config (`AriaConfig`): The configuration to use. rconfig dropout_ratec0t|jj}|jj}||_t j||d|_t j||d|_ t j||d|_ t j ||d|_ t j|||_ t j||_t j||_t j||_dS)NFrWT) batch_first)r/r0 vision_configr7num_attention_heads num_headsrrZq_projk_projv_projMultiheadAttentionmultihead_attnlinearDropoutdropout LayerNorm layer_norm layer_norm_kv)r6rgrhr7rmr8s r9r0zAriaCrossAttention.__init__ks *6 (< "i [uEEE i [uEEE i [uEEE !3KX\]]]i [99 z,// ,{33\+66r:Nc\|||}||}||}||}|||||\}}|||}|S)a Forward pass of the AriaCrossAttention module. Args: key_value_states (`torch.Tensor`): Input tensor for key and value. hidden_states (`torch.Tensor`): Input tensor for query. attn_mask (`torch.Tensor`, *optional*, defaults to None): Attention mask. Returns: torch.Tensor: Output tensor after cross-attention.  attn_mask)rnrwrxrorprrrurs) r6key_value_statesr;r{querykeyvalue attn_output_s r9rGzAriaCrossAttention.forward|s  DOOM::;;--.>??kk*++ ,--,,UC),TT Qll4;;{#;#;<< r:)rrb) rMrNrOrdr&rPr0rGrRrSs@r9rfrfbsn77z7777777"r:rfcTeZdZdZdeffd ZddejdejdzfdZxZ S) AriaProjectora  Aria Projector module. This module projects vision features into the language model's embedding space, enabling interaction between vision and language components. Args: config (`AriaConfig`): Configuration object for the model. rgc$t|j|_|jj|_|jj|_|jj|_ |j j|_ |j j|_ tjtj|j|j|_t'||_tj|j|_t/|j|j |j |_dSrb)r/r0projector_patch_to_query_dictpatch_to_query_dictrkr7r^rlrmkv_dim text_configr_r`rr1r2zeros'max_value_projector_patch_to_query_dictr}rf cross_attnrvrwrU feed_forwardr6rgr8s r9r0zAriaProjector.__init__s #)#G !/;-A*6 %1= ,8\%+f.\^b^n"o"opp ,V44,t'788,T-=t?SUYUdeer:Nr|r{cb|jd|jd}}||jvr-td|d|jd|j|}|jd|d|dd}|X||jd}|d d| dd}| |||}| | |}|S) a Forward pass of the Projector module. Args: key_value_states (`torch.Tensor`): Input tensor of shape (batch_size, num_patches, kv_dim). attn_mask (`torch.Tensor`, *optional*, default is None): Attention mask. Returns: `torch.Tensor`: Output tensor of shape (batch_size, query_number, output_dim). rr%zNumber of patches z: not found in patch_to_query_dict amongst possible values .Nr=rz)rJrKeyErrorkeysr} unsqueezerepeatrepeat_interleavermexpandsizerrrw) r6r|r{ batch_size num_patches query_numqueries attention_outouts r9rGzAriaProjector.forwardsR#3"8";=M=STU=VK d6 6 6O[OOlpmEmJmJmLmLOOO ,[9 *ZiZ(22155< >?? r:rb) rMrNrOrdr&r0r2rQrGrRrSs@r9rrsfffffff( PTATr:rc.eZdZdZdeffd ZdZxZS)AriaSharedExpertsMLPa/ Shared Expert MLP for shared experts. Unlike routed experts, shared experts process all tokens without routing. This class reconfigures the intermediate size in comparison to the LlamaMLP. Args: config (`AriaTextConfig`): Configuration object for the Aria language model. rgct||_|j|_|j|jz|_t j|j|j|j|_ t j|j|j|j|_ t j|j|j|j|_ t|j |_dS)NrW)r/r0rgr7intermediate_sizemoe_num_shared_expertsrrZmlp_bias gate_projup_proj down_projr hidden_actact_fnrs r9r0zAriaSharedExpertsMLP.__init__s  !-!'!9F>>+auz:K:RSSSK ;0A"BCCN0344  !*- Q/eCi(l6>*#=>>uSy Mr:c(eZdZdZfdZdZxZS)AriaGroupedExpertsGemmaP Grouped GEMM (General Matrix Multiplication) module for efficient expert computation. This module utilizes the grouped_gemm library (https://github.com/fanshiqing/grouped_gemm) for optimized performance. If the grouped_gemm library is not installed, it gracefully falls back to a sequential GEMM implementation, which may be slower but ensures functionality. Args: in_features (`int`): Number of input features. out_features (`int`): Number of output features. groups (`int`): Number of expert groups. ct||_||_||_t jtj||||_ dSrb) r/r0r^rgroupsrr1r2emptyr4)r6r^rrr8s r9r0zAriaGroupedExpertsGemm.__init__$sS &( l5;v{L#Q#QRR r:cRt||j|S)au Perform grouped matrix multiplication. Args: input (`torch.Tensor`): Input tensor of shape (num_tokens, in_features). tokens_per_expert (`torch.Tensor`): Number of tokens assigned to each expert. Returns: torch.Tensor: Output tensor of shape (num_tokens, out_features). )rr4cpu)r6inputrs r9rGzAriaGroupedExpertsGemm.forward+s-'  K  ! ! # #   r:rcrSs@r9rrsV SSSSS       r:rcDeZdZdeddffd ZdZdejfdZxZ S) AriaExpertsrgr-Nct||_t|j|jdz|j|_t|j|j|j|_dS)Nr#) r/r0rgrr7rmoe_num_expertsfc1fc2rs r9r0zAriaExperts.__init__@sd  )&*VYZ>Z\b\rss)&*BFDVX^Xnoor:ctj||jjd\}}tj|d}||fS)Nr%)krr=r)r2topkrgmoe_topkr functionalsoftmax)r6 router_logits top_logits top_indicesscoress r9route_tokens_to_expertsz#AriaExperts.route_tokens_to_expertsFsH"'*]dk>RXY"Z"Z"Z K&&zr&::F""r:c||\}}|j}tj|tj|jjd|jjdz |}|}| d}tj |} | d| |jj z} | | |} tj| dd\} } tj| | z} || |}tj|jd|jj z|df|j|j}|d| || d|jj |d}||dzd}|S)Nrr%)binsminmaxr=r#rr)rr?r2histcflattenr@rArgrviewargsort index_selectrrchunkrrsilurrrJrr index_copy_rsum)r6r;r top_k_index top_k_weightsoriginal_dtyperindicesflatten_indicessorted_indicespermuted_tokens fc1_output projectiongate expert_outputunpermuted_tokensrs r9rGzAriaExperts.forwardKs%)%A%A-%P%P" ]$*!K    ! ! $ $U] 3 3, +a/    "^    !,,r**77'44Q$+J^8^__XXo/@AA  ;z1"=== D]'' 33d: ->?? !K   #dk&: :M