
    <hl                        S r SSKJr  SSKJrJr  SSKrSSKrSSKJr  SSK	J
r
JrJr  SSKJr  SS	KJr  SS
KJr  SSKJr  SSKJr  SSKJrJrJrJrJr  SSKJr  SSKJ r   \RB                  " \"5      r#\\" SS9 " S S\5      5       5       r$\\" SS9 " S S\5      5       5       r% " S S\RL                  5      r'\ " S S\5      5       r(\" SS9 " S S\(5      5       r)\" S S9 " S! S"\(\5      5       r*/ S#Qr+g)$zPyTorch PaliGemmamodel.    )	dataclass)OptionalUnionN)nn   )CacheHybridCacheStaticCache)GenerationMixin)FlashAttentionKwargs)BaseModelOutputWithPast)PreTrainedModel)Unpack)ModelOutputTransformersKwargsauto_docstringcan_return_tuplelogging   )	AutoModel   )PaliGemmaConfigzN
    Base class for Paligemma outputs, with hidden states and attentions.
    )custom_introc                   B    \ rS rSr% SrSr\\R                     \	S'   Sr
g)PaligemmaModelOutputWithPast,   a  
past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
    Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
    `(batch_size, num_heads, sequence_length, embed_size_per_head)`)

    Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
    `past_key_values` input) to speed up sequential decoding.
image_hidden_states (`torch.FloatTensor`, *optional*):
    A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
    image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
Nimage_hidden_states )__name__
__module____qualname____firstlineno____doc__r   r   torchFloatTensor__annotations____static_attributes__r       h/var/www/html/shao/venv/lib/python3.13/site-packages/transformers/models/paligemma/modeling_paligemma.pyr   r   ,   s    
 8<%"3"34;r(   r   zU
    Base class for PaliGemma causal language model (or autoregressive) outputs.
    c                   &   \ rS rSr% SrSr\\R                     \	S'   Sr
\\R                     \	S'   Sr\\\\R                     \4      \	S'   Sr\\\R                        \	S'   Sr\\\R                        \	S'   Sr\\R                     \	S	'   S
rg)PaliGemmaCausalLMOutputWithPastB   a-  
loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
    Language modeling loss (for next-token prediction).
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.text_config.vocab_size)`):
    Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
    Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
    `(batch_size, num_heads, sequence_length, embed_size_per_head)`)

    Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
    `past_key_values` input) to speed up sequential decoding.
image_hidden_states (`torch.FloatTensor`, *optional*):
    A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
    image_hidden_states of the model produced by the vision encoder after projecting last hidden state.
Nlosslogitspast_key_valueshidden_states
attentionsr   r   )r   r    r!   r"   r#   r-   r   r$   r%   r&   r.   r/   r   listr   r0   tupler1   r   r'   r   r(   r)   r+   r+   B   s      )-D(5$$
%,*.FHU&&'.GKOXeD):):$;U$BCDK8<M8E%"3"345<59Ju001297;%"3"34;r(   r+   c                   6   ^  \ rS rSrS\4U 4S jjrS rSrU =r$ )PaliGemmaMultiModalProjectora   configc                    > [         TU ]  5         [        R                  " UR                  R
                  UR                  R                  SS9U l        g )NTbias)super__init__r   Linearvision_confighidden_sizeprojection_dimlinearselfr7   	__class__s     r)   r<   %PaliGemmaMultiModalProjector.__init__b   s;    ii 4 4 @ @&BVBVBeBelpqr(   c                 (    U R                  U5      nU$ NrA   )rC   image_featuresr0   s      r)   forward$PaliGemmaMultiModalProjector.forwardf   s    N3r(   rH   )	r   r    r!   r"   r   r<   rJ   r'   __classcell__rD   s   @r)   r5   r5   a   s    r r r(   r5   c                   L    \ rS rSr% \\S'   SrSrS/rSr	Sr
SrSrSrSrS rSrg	)
PaliGemmaPreTrainedModell   r7    Tr5   r/   c                 b   [        U R                  SU R                  R                  5       R                  5      n[	        U[
        R                  5      (       aW  UR                  R                  R                  SUS9  UR                  b%  UR                  R                  R                  5         g g g )Ninitializer_range        )meanstd)getattrr7   get_text_configrS   
isinstancer   r=   weightdatanormal_r:   zero_)rC   modulerV   s      r)   _init_weights&PaliGemmaPreTrainedModel._init_weightsz   s     dkk#68S8S8U8g8ghfbii((MM&&CS&9{{&  &&( ' )r(   r   N)r   r    r!   r"   r   r&   base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_can_compile_fullgraph_supports_flash_attn_supports_sdpa_supports_flex_attn_supports_attention_backendr_   r'   r   r(   r)   rO   rO   l   sE    &*#78"3!N"&)r(   rO   z{
    The Base Paligemma model which consists of a vision backbone and a language model withou language modeling head.,
    c            #       x  ^  \ rS rSrSS0rSrS\4U 4S jjrS rS r	S	 r
S
 r     S!S\\   4S jjrS\R                   4S jrS\R$                  S\R                   S\R                   4S jr\\             S"S\R$                  S\R                   S\\R,                     S\\R$                     S\\\\R                      \4      S\\R$                     S\\R$                     S\\R                      S\\R$                     S\\   S\\   S\\   S\\   S\\   S\\\4   4S jj5       5       rS rU =r $ )#PaliGemmaModel   zlanguage_model.modellanguage_modelFr7   c                   > [         TU ]  U5        [        R                  " UR                  S9U l        [        U5      U l        UR                  R                  U l	        [        R                  " UR                  S9nX l
        U R                  R                  b  U R                  R                  OSU l        U R                  5         g )N)r7   )r;   r<   r   from_configr>   vision_towerr5   multi_modal_projectortext_config
vocab_sizerm   r7   pad_token_id	post_init)rC   r7   rm   rD   s      r)   r<   PaliGemmaModel.__init__   s     %119M9MN%A&%I" ,,77"..f6H6HI,8<8P8P8\DKK44bdr(   c                 6    U R                   R                  5       $ rG   )rm   get_input_embeddingsrC   s    r)   ry   #PaliGemmaModel.get_input_embeddings   s    ""7799r(   c                 :    U R                   R                  U5        g rG   )rm   set_input_embeddingsrC   values     r)   r}   #PaliGemmaModel.set_input_embeddings   s    007r(   c                     Xl         g rG   rm   rC   decoders     r)   set_decoderPaliGemmaModel.set_decoder   s    %r(   c                     U R                   $ rG   r   rz   s    r)   get_decoderPaliGemmaModel.get_decoder   s    """r(   is_trainingc                    U R                   R                  R                  S:X  a  Ub  SU;   a  U$ g Ub  UOU R                  n[	        U[
        5      n[        R                  " U R                  5      R                  nUc  UnUR                  S S u  pU(       a  UR                  5       nO_[	        U[        5      (       a  UR                  5       nO9[	        U[        R                  5      (       a  UR                  S   O
US   U
-   S-   nUb  UR                  5       S:X  a  U$ [        R                  " X4XR                  UR                   S9nU
S:w  a(  U(       a  [        R"                  " USS	9nOSUS S 2S U
24'   U[        R$                  " XR                   S
9UR'                  SS5      :  -  nUS S S S 2S S 24   R)                  U	SSS5      nUb  UR+                  5       nUR                  S   nU(       ae  Uc  [-        S5      eUS S 2S S 2S S 2S U24   R/                  US S 2S S S S 24   R1                  UR                   5      S:H  S5      US S 2S S 2S S 2S U24'   US S 2S S 2S S 2S U24   US S 2S S S S 24   R1                  UR                   5      -   nUS:H  nUS S 2S S 2S S 2S U24   R/                  X5      US S 2S S 2S S 2S U24'   U$ )Nflash_attention_2rT   r   ro   r   r      
fill_valuedtypedevicediagonalr   z/Token type ids must be provided during training)r7   rs   _attn_implementationtrainingrY   r
   r$   finfor   minshapeget_max_cache_shaper	   Tensordimfullr   triuarangereshapeexpandclone
ValueErrormasked_fillto)rC   attention_masktoken_type_idsr/   cache_positioninput_tensorr   using_static_cache	min_dtypeinputs_lead_dimsequence_lengthtarget_lengthcausal_maskmask_lengthpadding_masks                  r)   _update_causal_mask"PaliGemmaModel._update_causal_mask   s    ;;""77;NN)c^.C%%%0%<k$--'EKK

+//	)L+7+=+=bq+A(+??AM55+??AM nell;; $$R(#A&81<  %.*<*<*>!*C!!jj,**]k]r]r
 a#jjqA36A///0u||M:O:OPSaSiSijlnoSppp!$a"23::?ArSUV%%++-K(..r2K !)$%VWW5@Aq,;,AV5W5c5c"1dD!#34778J8JKqPRS6Aq!\k\12
 'q!Q'<=qRVX\^_O_@`@c@cdodvdv@wwL'1,L1<Q1l{l=R1S1_1_2K1a+-. r(   pixel_valuesc                     U R                  U5      nUR                  nU R                  U5      nX@R                  R                  R
                  S-  -  nU$ )ae  
Obtains image last hidden states from the vision tower and apply multimodal projection.

Args:
    pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`)
       The tensors corresponding to the input images.
Returns:
    image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`).
g      ?)rq   last_hidden_staterr   r7   rs   r?   )rC   r   image_outputsselected_image_featurerI   s        r)   get_image_features!PaliGemmaModel.get_image_features   sU     )),7!.!@!@334JK';;+B+B+N+NPS+STr(   	input_idsinputs_embedsrI   c           	      J   Ucj  X R                  5       " [        R                  " U R                  R                  [        R
                  UR                  S95      :H  nUR                  S5      nOXR                  R                  :H  nUR                  5       nUR                  S5      R                  U5      R                  UR                  5      nUR                  S   UR                  S   -  nX$   R                  5       UR                  5       :w  a  [        SU SU 35      eU$ )z
Obtains multimodal placeholdr mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
equal to the length of multimodal features. If the lengths are different, an error is raised.
)r   r   ro   r   r   z6Image features and image tokens do not match: tokens: z, features )ry   r$   tensorr7   image_token_idlongr   allsum	unsqueeze	expand_asr   r   numelr   )rC   r   r   rI   special_image_maskn_image_tokensn_image_featuress          r)   get_placeholder_mask#PaliGemmaModel.get_placeholder_mask   s    !.2K2K2MT[[77uzzR_RfRfg3 " "4!7!7!;!*kk.H.H!H+//1/99"=GGVYYZgZnZno)//2^5I5I!5LL,2248L8L8NNHHXXcdtcuv  "!r(   r   position_idsr/   r   r   labels	use_cacheoutput_attentionsoutput_hidden_statesreturn_dictkwargsreturnc                     USL USL-  (       a  [        S5      eUb  UOU R                  R                  nUb  UOU R                  R                  nUb  UOU R                  R                  nUSL=(       a    U	SLnUbR  U R                  R
                  U R                  :  a.  XR                  R
                  :H  nUR                  5       nSUU'   OUnUc  U R                  5       " U5      nUcE  Ub  UR                  5       OSn[        R                  " UUUR                  S   -   UR                  S9nUc  UR                  S5      S-   nUbY  U R                  U5      nUR!                  UR                  UR"                  5      nU R%                  XUS9nUR'                  UU5      nU R)                  X6XWX5      nU R*                  " S
UUUUU
UUSUS.	UD6n[-        UR.                  UR0                  UR2                  UR4                  Ub  WS	9$ SS	9$ )  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`.

Example:

```python
>>> from PIL import Image
>>> import requests
>>> from transformers import AutoProcessor, PaliGemmaForConditionalGeneration

>>> model = PaliGemmaForConditionalGeneration.from_pretrained("google/paligemma2-3b-mix-224")
>>> processor = AutoProcessor.from_pretrained("google/paligemma2-3b-mix-224")

>>> prompt = "Where is the cat standing?"
>>> url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"
>>> image = Image.open(requests.get(url, stream=True).raw)

>>> inputs = processor(images=image, text=prompt,  return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(**inputs,)
>>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Where is the cat standing?\nsnow"
```Nz:You must specify exactly one of input_ids or inputs_embedsr   r   r   )r   rI   T)	r   r   r/   r   r   r   r   r   r   )r   r/   r0   r1   r   r   )r   r7   r   r   use_return_dictr   rt   r   ry   get_seq_lengthr$   r   r   r   r   r   r   r   r   masked_scatterr   rm   r   r   r/   r0   r1   )rC   r   r   r   r   r/   r   r   r   r   r   r   r   r   r   r   r   llm_input_idspast_seen_tokensrI   r   outputss                         r)   rJ   PaliGemmaModel.forward  sU   ^ -t";<YZZ1B1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++B]B]$D0GV45G  T[[%?%?4??%R!*kk.H.H!H%OO-M01M,-%M  557FM!CRC^==?de"\\ "2]5H5H5K"KTaThThN )33A6:L #!44\BN+..}/C/C]EXEXYN!%!:!:~ "; " *889K^\M..O]
 %% 
&%+'/!5)
 
 ,%77#33!//))2>2J
 	

 QU
 	
r(   )rm   rr   ru   rq   rt   )NNNNN)NNNNNNNNNNNNN)!r   r    r!   r"   _checkpoint_conversion_mappingaccepts_loss_kwargsr   r<   ry   r}   r   r   r   boolr   r$   r%   r   
LongTensorr   r   r   r   r   r2   r   r   r   r3   r   rJ   r'   rL   rM   s   @r)   rk   rk      s    '=>N%O"
 
:8&# &*B d^BHu/@/@  "))":?:K:K"]b]n]n"0  '+*.1537KO595959-1$(,0/3&*k
##k
 ''k
 !.	k

 u//0k
 "%U->->(?(F"GHk
 !!1!12k
 !!1!12k
   1 12k
 ))*k
 D>k
 $D>k
 'tnk
 d^k
 -.k
  
u22	3!k
  k
r(   rk   z|
    The Base Paligemma model which consists of a vision backbone and a language model without language modeling head.,
    c            %         ^  \ rS rSrSSSSS.rS/rS\4U 4S	 jjrS
 rS r	S r
S rS r\S 5       r\S 5       r\S 5       r\\              S*S\R(                  S\R*                  S\\R.                     S\\R(                     S\\\\R*                     \4      S\\R(                     S\\R(                     S\\R*                     S\\R(                     S\\   S\\   S\\   S\\   S\\\R.                  4   S \\   S!\\\ 4   4 S" jj5       5       r!          S+U 4S# jjr"\#S\R.                  S$\S%\S&\RH                  S\R.                  S'\4S( j5       r%S)r&U =r'$ ),!PaliGemmaForConditionalGenerationi  zmodel.language_modelzmodel.vision_towerzmodel.multi_modal_projectorlm_head)z^language_model.modelz^vision_towerz^multi_modal_projectorz^language_model.lm_headzlm_head.weightr7   c                    > [         TU ]  U5        [        U5      U l        [        R
                  " UR                  R                  UR                  R                  SS9U l	        U R                  5         g )NFr9   )r;   r<   rk   modelr   r=   rs   r?   rt   r   rv   rB   s     r)   r<   *PaliGemmaForConditionalGeneration.__init__  sS     #F+
yy!3!3!?!?ASASA^A^ejkr(   c                 6    U R                   R                  5       $ rG   )r   ry   rz   s    r)   ry   6PaliGemmaForConditionalGeneration.get_input_embeddings  s    zz..00r(   c                 :    U R                   R                  U5        g rG   )r   r}   r~   s     r)   r}   6PaliGemmaForConditionalGeneration.set_input_embeddings  s    

''.r(   c                 :    U R                   R                  U5        g rG   )r   r   r   s     r)   r   -PaliGemmaForConditionalGeneration.set_decoder  s    

w'r(   c                 6    U R                   R                  5       $ rG   )r   r   rz   s    r)   r   -PaliGemmaForConditionalGeneration.get_decoder  s    zz%%''r(   c                 8    U R                   R                  U5      $ rG   )r   r   )rC   r   s     r)   r   4PaliGemmaForConditionalGeneration.get_image_features  s    zz,,\::r(   c                 .    U R                   R                  $ rG   )r   rm   rz   s    r)   rm   0PaliGemmaForConditionalGeneration.language_model  s    zz(((r(   c                 .    U R                   R                  $ rG   )r   rq   rz   s    r)   rq   .PaliGemmaForConditionalGeneration.vision_tower  s    zz&&&r(   c                 .    U R                   R                  $ rG   )r   rr   rz   s    r)   rr   7PaliGemmaForConditionalGeneration.multi_modal_projector  s    zz///r(   r   r   r   r   r/   r   r   r   r   r   r   r   r   logits_to_keepr   r   c                 H   Ub  UOU R                   R                  nUb  UOU R                   R                  nUb  UOU R                   R                  nU R                  " SUUUUUUUU
U	UUSUS.UD6nUS   n[        U[        5      (       a  [        U* S5      OUnU R                  USS2USS24   5      nSnU	b3  U R                  " SUXR                   R                  R                  S.UD6n[        UUUR                  UR                  UR                  UR                   S9$ )r   NT)r   r   r   r   r   r/   r   r   r   r   r   r   r   r   )r.   r   rt   )r-   r.   r/   r0   r1   r   r   )r7   r   r   r   r   rY   intslicer   loss_functionrs   rt   r+   r/   r0   r1   r   )rC   r   r   r   r   r/   r   r   r   r   r   r   r   r   r   r   r   r0   slice_indicesr.   r-   s                        r)   rJ   )PaliGemmaForConditionalGeneration.forward  sP   ^ 2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++B]B]** 
%))%+'/!5)
 
"  
8B>SV8W8W~ot4]kmA}a,?@A%% f9P9P9[9[_eD /#33!//)) ' ; ;
 	
r(   c                 @  > [         TU ]  " U4UUUUUU	U
US.UD6nUR                  S5      b  US==   S-  ss'   US   S:X  a  XmS'   US L=(       a    US LnUS   S:X  a>  [        U[        5      (       a)  Ub  UOUnU R
                  R                  XxX$X5      nUUS'   U$ )N)r/   r   r   r   r   r   r   r   r   r   r   r   r   )r;   prepare_inputs_for_generationgetrY   r	   r   r   )rC   r   r/   r   r   r   r   r   r   r   r   r   r   model_inputsr   r   r   rD   s                    r)   r   ?PaliGemmaForConditionalGeneration.prepare_inputs_for_generation  s      w<
+')%)))
 
 N+7(A-( !!+7($D0GV45G!!j+&N&N,9,E=9L**88Q]K .9L)*r(   r   r   r   
batch_sizec                    U b  U R                  5       S:X  a  U nU$ [        R                  " U5      R                  n[        R                  " X4XUR
                  S9nUS:w  a  [        R                  " USS9nU[        R                  " X$R
                  S9UR                  SS5      :  -  nUSSSS2SS24   R                  USSS5      nU b  UR                  5       nU R                  S   n	USS2SS2SS2SU	24   U SS2SSSS24   R                  UR
                  5      -   n
U
S:H  n
USS2SS2SS2SU	24   R                  X5      USS2SS2SS2SU	24'   U$ )	a  
Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
`(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.

Args:
    attention_mask (`torch.Tensor`):
        A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape
        `(batch_size, 1, query_length, key_value_length)`.
    sequence_length (`int`):
        The sequence length being processed.
    target_length (`int`):
        The target length: when generating with static cache, the mask should be as long as the static cache,
        to account for the 0 padding, the part of the cache that is not filled yet.
    dtype (`torch.dtype`):
        The dtype to use for the 4D attention mask.
    cache_position (`torch.Tensor`):
        Indices depicting the position of the input sequence tokens in the sequence.
    batch_size (`torch.Tensor`):
        Batch size.
Nr   r   r   r   r   ro   r   )r   r$   r   r   r   r   r   r   r   r   r   r   r   r   )r   r   r   r   r   r  r   r   r   r   r   s              r)   5_prepare_4d_causal_attention_mask_with_cache_positionWPaliGemmaForConditionalGeneration._prepare_4d_causal_attention_mask_with_cache_position=  s}   > %.*<*<*>!*C(K* ' E*..I** 0Y\j\q\qK !##jjqA5<<>S>STWeWmWmnprsWtttK%dD!Q&67>>z1bRTUK))//1,2226*1aL[L+@ANSTVZ\`bcScDdDgDg&&E    ,q05@Aq,;,AV5W5c5c 6Aq!\k\12 r(   )r   r   )NNNNNNNNNNNNNr   )
NNNNNNNTNN)(r   r    r!   r"   r   _tied_weights_keysr   r<   ry   r}   r   r   r   propertyrm   rq   rr   r   r   r$   r   r%   r   r   r   r2   r   r   r   r   r   r3   r+   rJ   r   staticmethodr   r  r'   rL   rM   s   @r)   r   r     s    "8-"?#,	&" ++ 1/((; ) ) ' ' 0 0  '+*.1537KO595959-1$(,0/3&*34V
##V
 ''V
 !.	V

 u//0V
 "%U->->(?(F"GHV
 !!1!12V
 !!1!12V
   1 12V
 ))*V
 D>V
 $D>V
 'tnV
 d^V
 c5<</0V
  +,!V
" 
u55	6#V
  V
v ,\ 444 4 {{	4
 4 4 4r(   r   )r   rO   rk   ),r#   dataclassesr   typingr   r   r$   torch.utils.checkpointr   cache_utilsr   r	   r
   
generationr   modeling_flash_attention_utilsr   modeling_outputsr   modeling_utilsr   processing_utilsr   utilsr   r   r   r   r   autor   configuration_paligemmar   
get_loggerr   loggerr   r+   Moduler5   rO   rk   r   __all__r   r(   r)   <module>r     s6    ! "    : : ) B 7 - &   4 
		H	% 
<#: < <  
<k < <2299  ) ) )0 
x
- x

x
v 
i(@/ i
iX ^r(   