[TU]U5 [URURS9UlgN)eps)super__init__rintermediate_size rms_norm_eps ffn_sub_norm)selfr' __class__s r"r,BitNetMLP.__init__/s+  )&*B*BH[H[\r!c URURURURU55UR U5-55nU$)N) down_projr/act_fn gate_projup_proj)r0xr4s r"forwardBitNetMLP.forward3sFNN4#4#4T[[PQAR5SVZVbVbcdVe5e#fg r!)r/) rrrrrr,r9r __classcell__r1s@r"r%r%.s]|]r!r%cB^\rSrSrS\S\4U4SjjrSS\RS\ \R\R4S\ \RS\ \ S \ \RS \ \S \ \R\ \R\ \ \R44S jjrS rU=r$)BitNetAttention8r' layer_idxcj>[TU]X5 [URURS9Ulgr))r+r,r hidden_sizer. attn_sub_norm)r0r'r@r1s r"r,BitNetAttention.__init__9s+ +*6+=+=6CVCVWr! hidden_statesposition_embeddingsattention_maskpast_key_valuecache_positionkwargsreturnc VURSSn/UQSPURP7nURU5RU5R SS5n UR U5RU5R SS5n UR U5RU5R SS5n Uup[XX5upUb$XUS.nURXURU5up[nURRS:wa[URRnU"UU U U U4UR(dSO URUR S.UD6unnUR""/UQSP76R%5nUR'U5nUR)U5nUU4$)Nrr )sincosrIeagerg)dropoutscaling)shapehead_dimq_projview transposek_projv_projrupdater@rr'_attn_implementationr trainingattention_dropoutrRreshape contiguousrCo_proj)r0rErFrGrHrIrJ input_shape hidden_shape query_states key_states value_statesrOrN cache_kwargsattention_interface attn_output attn_weightss r"r9BitNetAttention.forward=s$))#2. 88b8$--8 {{=166|DNNqRST [[/44\BLLQPQR {{=166|DNNqRST &#7 RU#[  %#&nUL'5'<'r>8sX|XX+/59 +)||+)#5<<#=>+)!. +) ! +) !!1!12 +)-.+) u||Xell3XeELL>Q5RR S+)+)r!r>c\rSrSrSrg)BitNetDecoderLayerkrNrrr!r"rqrqkr#r!rqc\rSrSrSrg) BitNetModelorNrrr!r"rtrtor#r!rtc>^\rSrSrS/rSrSrS\4U4SjjrSr U=r $)BitNetForCausalLMszlm_head.weightNrKc $>[TU]"S0UD6$)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, transformers., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, transformers., config.vocab_size]`. Example: ```python >>> from transformers import AutoTokenizer, BitNetForCausalLM >>> model = BitNetForCausalLM.from_pretrained("microsoft/bitnet-b1.58-2B-4T") >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/bitnet-b1.58-2B-4T") >>> prompt = f'<|begin_of_text|>User: Hey, are you conscious? Can you talk to me?<|eot_id|>Assistant: ' >>> inputs = tokenizer(prompt, return_tensors="pt") >>> # Generate >>> generate_ids = model.generate(inputs.input_ids, max_length=100) >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] "User: Hey, are you conscious? Can you talk to me?Assistant: No, I'm not conscious. I'm an artificial intelligence designed to assist with information and tasks. How can I help you today?" ```r)r+r9)r0 super_kwargsr1s r"r9BitNetForCausalLM.forwardxs4w...r!r) rrrr_tied_weights_keys_tp_plan_pp_planrr9r r;r<s@r"rwrwss**+HH/ //r!rw)rwrtBitNetPreTrainedModel)'__doc__typingrrrl cache_utilsrmodeling_flash_attention_utilsrmodeling_outputsrmodeling_utilsr processing_utilsr utilsr gemma.modeling_gemmar llama.modeling_llamarrrrrrrconfiguration_bitnetr get_loggerrloggerrr%r>rqrtrw__all__rr!r"rs% B65&+/   H % L 0)n0)f *  * /(/D r!