§
    ‚Štj�"  ã                   óÞ  — d Z ddlZddlmZ ddlmZ ddlmZ ddlm	Z	 ddl
mZ ddlmZmZmZ d	d
lmZ d	dlmZmZmZmZmZmZmZ  ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z G d„ de¦  «        Z G d„ de¦  «        Z G d„ de¦  «        Z G d„ de¦  «        Ze G d„ de¦  «        ¦   «         Ze G d„ de¦  «        ¦   «         Z e G d„ de¦  «        ¦   «         Z!g d¢Z"dS )z%HyperCLOVAX modular model definition.é    N)Ústricté   )ÚCache)ÚCausalLMOutputWithPast)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleé   )ÚGraniteConfig)ÚGraniteAttentionÚGraniteDecoderLayerÚGraniteForCausalLMÚGraniteModelÚGranitePreTrainedModelÚGraniteRMSNormÚGraniteRotaryEmbeddingz,naver-hyperclovax/HyperCLOVAX-SEED-Think-14B)Ú
checkpointc                   ód   ‡ — e Zd ZU dZdZdZedz  ed<   dZe	dz  ed<   dZ
eed<   ˆ fd„Zd	„ Zˆ xZS )
ÚHyperCLOVAXConfiga@  
    embedding_multiplier (`float`, *optional*, defaults to `1.0`):
        Scaling factor applied to the token embedding outputs. Used in MuP to control the
        scale of the embedding activations.
    logits_scaling (`float`, *optional*, defaults to `1.0`):
        Scaling factor **multiplied** to the final logits before loss computation or sampling.
        Used in MuP to ensure consistent output scale across model sizes. Note: unlike
        [`GraniteConfig`], this is a multiplier, not a divisor.
    residual_multiplier (`float`, *optional*, defaults to `1.0`):
        Scaling factor applied to each sub-layer output before adding to the residual stream.
        Used in Maximal Update Parametrization (MuP) to stabilize training across model sizes.
    attention_multiplier (`float`, *optional*, defaults to `head_dim ** -0.5`):
        Scaling factor applied to attention logits before softmax, replacing the standard
        `1 / sqrt(head_dim)` scaling. Set explicitly for MuP-based training; when `None`,
        defaults to the standard value.
    use_post_norm (`bool`, *optional*, defaults to `True`):
        Whether to apply an extra RMSNorm after each sub-layer output (Peri-Layer Normalization).

    ```python
    >>> from transformers import HyperCLOVAXModel, HyperCLOVAXConfig

    >>> # Initializing a HyperCLOVAX style configuration
    >>> configuration = HyperCLOVAXConfig()

    >>> # Initializing a model from the configuration
    >>> model = HyperCLOVAXModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```ÚhyperclovaxNÚhead_dimÚattention_multiplierTÚuse_post_normc                 ó    •— | j         €| j        | j        z  | _          t          ¦   «         j        di |¤Ž | j        €| j         dz  | _        d S d S )Ng      à¿© )r   Úhidden_sizeÚnum_attention_headsÚsuperÚ__post_init__r   )ÚselfÚkwargsÚ	__class__s     €úq/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/hyperclovax/modular_hyperclovax.pyr    zHyperCLOVAXConfig.__post_init__P   sd   ø€ ð Œ=Ð Ø Ô,°Ô0HÑHˆDŒMà�‰ŒÔÐ'Ð' Ð'Ð'Ð'ð Ô$Ð,Ø(,¬°tÑ(;ˆDÔ%Ð%Ð%ð -Ð,ó    c                 ól   — | j         | j        z  dk    r t          d| j         › d| j        › d�¦  «        ‚dS )zCValidates that `hidden_size` is divisible by `num_attention_heads`.r   zThe hidden size (z6) is not a multiple of the number of attention heads (z).N)r   r   Ú
ValueError)r!   s    r$   Úvalidate_architecturez'HyperCLOVAXConfig.validate_architecture]   s[   € àÔ˜dÔ6Ñ6¸!Ò;Ð;Ýð7 DÔ$4ð 7ð 7ØÔ2ð7ð 7ð 7ñô ð ð <Ð;r%   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typer   ÚintÚ__annotations__r   Úfloatr   Úboolr    r(   Ú__classcell__©r#   s   @r$   r   r   $   s—   ø€ € € € € € ðð ð> €Jà€Hˆc�D‰jÐÐÑð *.Ð˜% $™,Ð-Ð-Ñ-ð €M�4ÐÐÑð<ð <ð <ð <ð <ðð ð ð ð ð ð r%   r   c                   ó   — e Zd ZdS )ÚHyperCLOVAXRMSNormN©r)   r*   r+   r   r%   r$   r5   r5   f   ó   € € € € € Ø€Dr%   r5   c                   ó   — e Zd ZdS )ÚHyperCLOVAXRotaryEmbeddingNr6   r   r%   r$   r9   r9   j   r7   r%   r9   c                   ó   — e Zd ZdS )ÚHyperCLOVAXAttentionNr6   r   r%   r$   r;   r;   n   r7   r%   r;   c                   óÒ   ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 ddej        dej        dz  dej        dz  d	e	dz  d
e
dz  deej        ej        f         dz  dee         dej        fd„Zˆ xZS )ÚHyperCLOVAXDecoderLayerÚconfigÚ	layer_idxc                 ó4  •— t          ¦   «                              ||¦  «         |j        rt          |j        |j        ¬¦  «        nt          j        ¦   «         | _        |j        rt          |j        |j        ¬¦  «        nt          j        ¦   «         | _	        d S )N)Úeps)
r   Ú__init__r   r5   r   Úrms_norm_epsÚnnÚIdentityÚ
post_norm1Ú
post_norm2)r!   r>   r?   r#   s      €r$   rB   z HyperCLOVAXDecoderLayer.__init__s   s“   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+ð PVÔOcÐvÕ˜vÔ1°vÔ7JÐKÑKÔKÐKÕikÔitÑivÔivð 	Œð PVÔOcÐvÕ˜vÔ1°vÔ7JÐKÑKÔKÐKÕikÔitÑivÔivð 	Œˆˆr%   NFÚhidden_statesÚattention_maskÚposition_idsÚpast_key_valuesÚ	use_cacheÚposition_embeddingsr"   Úreturnc           
      óB  — |}|                       |¦  «        } | j        d||||||dœ|¤Ž\  }}	|                      |¦  «        }||| j        z  z   }|}|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }||| j        z  z   }|S )N)rH   rI   rJ   rK   rL   rM   r   )Úinput_layernormÚ	self_attnrF   Úresidual_multiplierÚpost_attention_layernormÚmlprG   )
r!   rH   rI   rJ   rK   rL   rM   r"   ÚresidualÚ_s
             r$   ÚforwardzHyperCLOVAXDecoderLayer.forward}   sÓ   € ð !ˆØ×,Ò,¨]Ñ;Ô;ˆà)˜4œ>ð 
Ø'Ø)Ø%Ø+ØØ 3ð
ð 
ð ð
ð 
Ñˆ�qð Ÿš¨Ñ6Ô6ˆØ  =°4Ô3KÑ#KÑKˆð !ˆØ×5Ò5°mÑDÔDˆØŸš Ñ/Ô/ˆØŸš¨Ñ6Ô6ˆØ  =°4Ô3KÑ#KÑKˆØÐr%   )NNNFN)r)   r*   r+   r   r.   rB   ÚtorchÚTensorÚ
LongTensorr   r1   Útupler   r   rW   r2   r3   s   @r$   r=   r=   r   sø   ø€ € € € € ð
Ð0ð 
¸Sð 
ð 
ð 
ð 
ð 
ð 
ð /3Ø04Ø(,Ø!&ØHLðð à”|ðð œ tÑ+ðð Ô&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5¤<°´Ð#=Ô>ÀÑEðð Ð+Ô,ðð 
Œðð ð ð ð ð ð ð r%   r=   c                   ó   — e Zd ZdS )ÚHyperCLOVAXPreTrainedModelNr6   r   r%   r$   r]   r]   Ÿ   ó   € € € € € à€Dr%   r]   c                   ó   — e Zd ZdS )ÚHyperCLOVAXModelNr6   r   r%   r$   r`   r`   ¤   r^   r%   r`   c                   óè   — e Zd Zee	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dedz  dej	        dz  dej        dz  d	e
dz  d
eej        z  dee         defd„¦   «         ¦   «         ZdS )ÚHyperCLOVAXForCausalLMNr   Ú	input_idsrI   rJ   rK   Úinputs_embedsÚlabelsrL   Úlogits_to_keepr"   rN   c	           
      ój  —  | j         d||||||dœ|	¤Ž}
|
j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        | j        j        z  }d}|� | j        d||| j        j	        dœ|	¤Ž}t          |||
j        |
j        |
j        ¬¦  «        S )a&  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, HyperCLOVAXForCausalLM

        >>> model = HyperCLOVAXForCausalLM.from_pretrained("naver-hyperclovax/HyperCLOVAX-SEED-Think-14B")
        >>> tokenizer = AutoTokenizer.from_pretrained("naver-hyperclovax/HyperCLOVAX-SEED-Think-14B")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me? Are you okay?" The man was confused and answered, "Yes." Then the woman asked.
        ```)rc   rI   rJ   rK   rd   rL   N)Úlogitsre   Ú
vocab_size)Úlossrh   rK   rH   Ú
attentionsr   )ÚmodelÚlast_hidden_stateÚ
isinstancer.   ÚsliceÚlm_headr>   Úlogits_scalingÚloss_functionri   r   rK   rH   rk   )r!   rc   rI   rJ   rK   rd   re   rL   rf   r"   ÚoutputsrH   Úslice_indicesrh   rj   s                  r$   rW   zHyperCLOVAXForCausalLM.forward«   sþ   € ð> �$”*ð 
ØØ)Ø%Ø+Ø'Øð
ð 
ð ð
ð 
ˆð  Ô1ˆÝ8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆà—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAÀDÄKÔD^Ñ^ˆàˆØÐØ%�4Ô%Ðp¨V¸FÈtÌ{ÔOeÐpÐpÐioÐpÐpˆDå%ØØØ#Ô3Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r%   )NNNNNNNr   )r)   r*   r+   r
   r	   rX   rZ   rY   r   ÚFloatTensorr1   r.   r   r   r   rW   r   r%   r$   rb   rb   ©   sú   € € € € € àØð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ð6
ð 6
àÔ# dÑ*ð6
ð œ tÑ+ð6
ð Ô&¨Ñ-ð	6
ð
  ™ð6
ð Ô(¨4Ñ/ð6
ð Ô  4Ñ'ð6
ð ˜$‘;ð6
ð ˜eœlÑ*ð6
ð Ð+Ô,ð6
ð 
 ð6
ð 6
ð 6
ñ „^ñ Ôð6
ð 6
ð 6
r%   rb   )r   r]   r`   rb   )#r,   rX   Útorch.nnrD   Úhuggingface_hub.dataclassesr   Úcache_utilsr   Úmodeling_outputsr   Úprocessing_utilsr   Úutilsr   r	   r
   Úgranite.configuration_graniter   Úgranite.modeling_graniter   r   r   r   r   r   r   r   r5   r9   r;   r=   r]   r`   rb   Ú__all__r   r%   r$   ú<module>r      s±  ðð ,Ð +à €€€Ø Ð Ð Ð Ð Ð Ø .Ð .Ð .Ð .Ð .Ð .à  Ð  Ð  Ð  Ð  Ð  Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø &Ð &Ð &Ð &Ð &Ð &Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð €ÐIÐJÑJÔJØð=ð =ð =ð =ð =˜ñ =ô =ñ „ñ KÔJð=ð@	ð 	ð 	ð 	ð 	˜ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð!7ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð+ñ 	ô 	ð 	ð*ð *ð *ð *ð *Ð1ñ *ô *ð *ðZ ð	ð 	ð 	ð 	ð 	Ð!7ñ 	ô 	ñ „ð	ð ð	ð 	ð 	ð 	ð 	�|ñ 	ô 	ñ „ð	ð ð9
ð 9
ð 9
ð 9
ð 9
Ð/ñ 9
ô 9
ñ „ð9
ðxð ð €€€r%   