§
    ‚Štj“%  ã                   ó>  — d dl mZ d dlZd dlmZ d dlmZ ddlmZ ddlm	Z	 ddl
mZ dd	lmZ dd
lmZ ddlmZ ddlmZmZmZ ddlmZmZmZmZmZmZmZmZmZm Z  ddl!m"Z"  ej#        e$¦  «        Z% ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z& G d„ de"¦  «        Z' G d„ de¦  «        Z( G d„ de¦  «        Z) G d„ de¦  «        Z* G d„ de¦  «        Z+ G d„ de¦  «        Z, G d „ d!e¦  «        Z- G d"„ d#e¦  «        Z. G d$„ d%e¦  «        Z/g d&¢Z0dS )'é    )ÚCallableN)Ústrict)Únné   )ÚACT2CLS)ÚCache)ÚPreTrainedConfig)ÚRopeParameters)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚloggingé   )
ÚLlamaAttentionÚLlamaDecoderLayerÚLlamaForCausalLMÚLlamaForTokenClassificationÚ
LlamaModelÚLlamaPreTrainedModelÚLlamaRMSNormÚLlamaRotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forward)ÚNemotronMLPz!swiss-ai/Apertus-8B-Instruct-2509)Ú
checkpointc            	       ó¦  ‡ — e Zd ZU dZdZdgZdZdddddddddœZd	gd
gfddgdgfdgdgfdœZdZ	e
ed<   dZe
ed<   dZe
ed<   dZe
ed<   dZe
ed<   dZe
dz  ed<   dZeed<   dZe
ed<   dZeed<   dZeed <   d!Zeed"<   d#Ze
dz  ed$<   d%Ze
dz  ed&<   d'Ze
ee
         z  dz  ed(<   d)Zeed*<   dZee z  dz  ed+<   d)Z!eed,<   d-Z"ee
z  ed.<   ˆ fd/„Z#ˆ xZ$S )0ÚApertusConfigaz  
    ```python
    >>> from transformers import ApertusModel, ApertusConfig

    >>> # Initializing a Apertus-8B style configuration
    >>> configuration = ApertusConfig()

    >>> # Initializing a model from the Apertus-8B style configuration
    >>> model = ApertusModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```ÚapertusÚpast_key_valuesç    `ãfAÚcolwiseÚreplicated_with_grad_allreduceÚrowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.q_normzlayers.*.self_attn.k_normzlayers.*.self_attn.o_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormi   Ú
vocab_sizei   Úhidden_sizei 8  Úintermediate_sizeé    Únum_hidden_layersÚnum_attention_headsNÚnum_key_value_headsÚxieluÚ
hidden_acti   Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangegñhãˆµøä>Úrms_norm_epsTÚ	use_cacher   Úpad_token_idé   Úbos_token_idr   Úeos_token_idFÚtie_word_embeddingsÚrope_parametersÚattention_biasç        Úattention_dropoutc                 óŠ   •— | j         €| j        | _         | j        €dddddddœ| _         t          ¦   «         j        di |¤Ž d S )	NÚllama3r!   g       @i    g      ð?g      @)Ú	rope_typeÚ
rope_thetaÚfactorÚ original_max_position_embeddingsÚlow_freq_factorÚhigh_freq_factor© )r2   r1   r>   ÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s     €úi/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/apertus/modular_apertus.pyrL   zApertusConfig.__post_init__e   sf   ø€ ØÔ#Ð+Ø'+Ô'?ˆDÔ$àÔÐ'à%Ø(ØØ48Ø#&Ø$'ð$ð $ˆDÔ ð 	�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'ó    )%Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚdefault_thetaÚbase_model_tp_planÚbase_model_pp_planr,   ÚintÚ__annotations__r-   r.   r0   r1   r2   r4   Ústrr5   r6   Úfloatr7   r8   Úboolr9   r;   r<   Úlistr=   r>   r
   Údictr?   rA   rL   Ú__classcell__©rO   s   @rP   r   r   .   s
  ø€ € € € € € ðð ð €JØ#4Ð"5ÐØ€Mà%.Ø%.Ø%.Ø%EØ%EØ%.Ø )Ø"+ð	ð 	Ðð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ðð Ðð €J�ÐÐÑØ€K�ÐÐÑØ"Ð�sÐ"Ð"Ñ"ØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø&*Ð˜˜t™Ð*Ð*Ñ*Ø€J�ÐÐÑØ#(Ð˜SÐ(Ð(Ñ(Ø#Ð�uÐ#Ð#Ñ#Ø€L�%ÐÐÑØ€IˆtÐÐÑØ €L�#˜‘*Ð Ð Ñ Ø €L�#˜‘*Ð Ð Ñ Ø+,€L�#˜˜Sœ	‘/ DÑ(Ð,Ð,Ñ,Ø %Ð˜Ð%Ð%Ñ%Ø48€O�^ dÑ*¨TÑ1Ð8Ð8Ñ8Ø €N�DÐ Ð Ñ Ø%(Ð�u˜s‘{Ð(Ð(Ñ(ð(ð (ð (ð (ð (ð (ð (ð (ð (rQ   r   c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )Ú
ApertusMLPc                 ó<  •— t          ¦   «                              |¦  «         t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _        |j        dk    r"t          d         |j
        ¬¦  «        | _        d S d S )NF)Úbiasr3   )Údtype)rK   Ú__init__r   ÚLinearr-   r.   Úup_projÚ	down_projr4   r   rh   Úact_fn)rM   ÚconfigrO   s     €rP   ri   zApertusMLP.__init__v   sŒ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý”y Ô!1°4Ô3IÐPUÐVÑVÔVˆŒÝœ 4Ô#9¸4Ô;KÐRWÐXÑXÔXˆŒØÔ Ò'Ð'Ý! 'Ô*°´Ð>Ñ>Ô>ˆDŒKˆKˆKð (Ð'rQ   )rR   rS   rT   ri   rb   rc   s   @rP   re   re   u   s8   ø€ € € € € ð?ð ?ð ?ð ?ð ?ð ?ð ?ð ?ð ?rQ   re   c                   ó   — e Zd ZdS )ÚApertusRMSNormN©rR   rS   rT   rJ   rQ   rP   rp   rp   ~   ó   € € € € € Ø€DrQ   rp   c                   ó   — e Zd ZdS )ÚApertusRotaryEmbeddingNrq   rJ   rQ   rP   rt   rt   ‚   rr   rQ   rt   c                   óÈ   ‡ — e Zd Zddededz  fˆ fd„Z	 ddej        deej        ej        f         dej        dz  de	dz  d	e
e         d
eej        ej        f         fd„Zˆ xZS )ÚApertusAttentionNrn   Ú	layer_idxc                 óÈ   •— t          ¦   «                              ||¦  «         t          | j        |j        ¦  «        | _        t          | j        |j        ¦  «        | _        d S ©N)rK   ri   rp   Úhead_dimr7   Úq_normÚk_norm©rM   rn   rw   rO   s      €rP   ri   zApertusAttention.__init__‡   sM   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+Ý$ T¤]°FÔ4GÑHÔHˆŒÝ$ T¤]°FÔ4GÑHÔHˆŒˆˆrQ   r'   Úposition_embeddingsr(   r    rN   Úreturnc                 óv  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }	|                      |¦  «                             |¦  «                             dd¦  «        }
|                      |¦  «        }|                      |	¦  «        }	|\  }}t          ||	||¦  «        \  }}	|�| 
                    |	|
| j        ¦  «        \  }	}
t          j        | j        j        t           ¦  «        } || ||	|
|f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Néÿÿÿÿr:   r   r@   )ÚdropoutÚscaling)Úshaperz   Úq_projÚviewÚ	transposeÚk_projÚv_projr{   r|   r   Úupdaterw   r   Úget_interfacern   Ú_attn_implementationr   ÚtrainingrA   rƒ   ÚreshapeÚ
contiguousÚo_proj)rM   r'   r~   r(   r    rN   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss                   rP   ÚforwardzApertusAttention.forwardŒ   sâ  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆØ—{’{ <Ñ0Ô0ˆØ—[’[ Ñ,Ô,ˆ
à&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(rQ   ry   )rR   rS   rT   r   r[   ri   ÚtorchÚTensorÚtupler   r   r   r›   rb   rc   s   @rP   rv   rv   †   sä   ø€ € € € € ðIð I˜}ð I¸¸t¹ð Ið Ið Ið Ið Ið Ið )-ð()ð ()à”|ð()ð # 5¤<°´Ð#=Ô>ð()ð œ tÑ+ð	()ð
  ™ð()ð Ð+Ô,ð()ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð()ð ()ð ()ð ()ð ()ð ()ð ()ð ()rQ   rv   c                   óÒ   ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 ddej        dej        dz  dej        dz  d	e	dz  d
e
dz  deej        ej        f         dz  dee         dej        fd„Zˆ xZS )ÚApertusDecoderLayerrn   rw   c                 óÔ   •— t          ¦   «                              ||¦  «         t          |j        |j        ¬¦  «        | _        t          |j        |j        ¬¦  «        | _        | `| `d S )N)Úeps)	rK   ri   rp   r-   r7   Úattention_layernormÚfeedforward_layernormÚinput_layernormÚpost_attention_layernormr}   s      €rP   ri   zApertusDecoderLayer.__init__¸   se   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+Ý#1°&Ô2DÈ&ÔJ]Ð#^Ñ#^Ô#^ˆÔ Ý%3°FÔ4FÈFÔL_Ð%`Ñ%`Ô%`ˆÔ"àÐ ØÐ)Ð)Ð)rQ   NFr'   r(   Úposition_idsr    r8   r~   rN   r   c           
      óÎ   — |}|                       |¦  «        } | j        d||||||dœ|¤Ž\  }}	||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|S )N)r'   r(   r§   r    r8   r~   rJ   )r£   Ú	self_attnr¤   Úmlp)
rM   r'   r(   r§   r    r8   r~   rN   ÚresidualÚ_s
             rP   r›   zApertusDecoderLayer.forwardÀ   s¡   € ð !ˆØ×0Ò0°Ñ?Ô?ˆØ)˜4œ>ð 
Ø'Ø)Ø%Ø+ØØ 3ð
ð 
ð ð
ð 
Ñˆ�qð ! =Ñ0ˆð !ˆØ×2Ò2°=ÑAÔAˆØŸš Ñ/Ô/ˆØ  =Ñ0ˆØÐrQ   )NNNFN)rR   rS   rT   r   r[   ri   rœ   r�   Ú
LongTensorr   r_   rž   r   r   r›   rb   rc   s   @rP   r    r    ·   s÷   ø€ € € € € ð*˜}ð *¸ð *ð *ð *ð *ð *ð *ð /3Ø04Ø(,Ø!&ØHLðð à”|ðð œ tÑ+ðð Ô&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5¤<°´Ð#=Ô>ÀÑEðð Ð+Ô,ðð 
Œðð ð ð ð ð ð ð rQ   r    c                   ó   — e Zd ZdS )ÚApertusPreTrainedModelNrq   rJ   rQ   rP   r¯   r¯   ß   rr   rQ   r¯   c                   ó   — e Zd ZdS )ÚApertusModelNrq   rJ   rQ   rP   r±   r±   ã   rr   rQ   r±   c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚApertusForCausalLMc                 ó6   •—  t          ¦   «         j        di |¤ŽS )aŠ  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, ApertusForCausalLM

        >>> model = ApertusForCausalLM.from_pretrained("swiss-ai/Apertus-8B-Instruct-2509")
        >>> tokenizer = AutoTokenizer.from_pretrained("swiss-ai/Apertus-8B-Instruct-2509")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```rJ   )rK   r›   )rM   Úsuper_kwargsrO   s     €rP   r›   zApertusForCausalLM.forwardè   s!   ø€ ð. �u‰wŒwŒÐ.Ð. Ð.Ð.Ð.rQ   )rR   rS   rT   r›   rb   rc   s   @rP   r³   r³   ç   s8   ø€ € € € € ð/ð /ð /ð /ð /ð /ð /ð /ð /rQ   r³   c                   ó   — e Zd ZdS )ÚApertusForTokenClassificationNrq   rJ   rQ   rP   r·   r·     rr   rQ   r·   )r   r±   r³   r·   r¯   )1Úcollections.abcr   rœ   Úhuggingface_hub.dataclassesr   r   Úactivationsr   Úcache_utilsr   Úconfiguration_utilsr	   Úmodeling_rope_utilsr
   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   r   Úllama.modeling_llamar   r   r   r   r   r   r   r   r   r   Únemotron.modeling_nemotronr   Ú
get_loggerrR   Úloggerr   re   rp   rt   rv   r    r¯   r±   r³   r·   Ú__all__rJ   rQ   rP   ú<module>rÆ      s`  ðð %Ð $Ð $Ð $Ð $Ð $à €€€Ø .Ð .Ð .Ð .Ð .Ð .Ø Ð Ð Ð Ð Ð à "Ð "Ð "Ð "Ð "Ð "Ø  Ð  Ð  Ð  Ð  Ð  Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø &Ð &Ð &Ð &Ð &Ð &Ø @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð 5Ð 4Ð 4Ð 4Ð 4Ð 4ð 
ˆÔ	˜HÑ	%Ô	%€ð €Ð>Ð?Ñ?Ô?ØðB(ð B(ð B(ð B(ð B(Ð$ñ B(ô B(ñ „ñ @Ô?ðB(ðJ?ð ?ð ?ð ?ð ?�ñ ?ô ?ð ?ð	ð 	ð 	ð 	ð 	�\ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð1ñ 	ô 	ð 	ð.)ð .)ð .)ð .)ð .)�~ñ .)ô .)ð .)ðb%ð %ð %ð %ð %Ð+ñ %ô %ð %ðP	ð 	ð 	ð 	ð 	Ð1ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	�:ñ 	ô 	ð 	ð/ð /ð /ð /ð /Ð)ñ /ô /ð /ð6	ð 	ð 	ð 	ð 	Ð$?ñ 	ô 	ð 	ðð ð €€€rQ   