
    Ph                        d Z ddlmZ ddlmZmZmZmZ ddlZddl	m
c mZ ddlm
Z
 ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZmZmZ ddlmZ ddl m!Z!m"Z"m#Z#m$Z$ ddl%m&Z& ddl'm(Z(m)Z) ddl*m+Z+ ddl,m-Z- ddl.m/Z/m0Z0  e$jb                  e2      Z3e e"d       G d de                    Z4e e"d       G d de                    Z5	 	 	 	 d@dZ6g fdZ7 G d d e
jp                        Z9 G d! d"e
jt                        Z; G d# d$e
jx                        Z= G d% d&ej                  jx                        Z>d' Z?dAd(Z@ G d) d*e
jx                        ZA	 dBd+e
jx                  d,ej                  d-ej                  d.ej                  d/eej                     d0eCd1eCfd2ZD G d3 d4e
jx                        ZE G d5 d6e      ZF G d7 d8e      ZGe" G d9 d:e             ZHe" G d; d<eH             ZI G d= d>eHe      ZJg d?ZKy)CzPyTorch Idefics model.    )	dataclass)AnyCallableOptionalUnionN)nn   )ACT2FN)CacheDynamicCache)GenerationMixin)create_causal_mask)GradientCheckpointingLayer)ModelOutput)ALL_ATTENTION_FUNCTIONSPretrainedConfigPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuplelogging)deprecate_kwarg)OutputRecordercheck_model_inputs   )IdeficsConfig)IdeficsPerceiverResampler)IdeficsVisionEmbeddingsIdeficsVisionTransformerz{
    Base class for Idefics model's outputs that may also contain a past key/values (to speed up sequential decoding).
    )custom_introc                       e Zd ZU dZdZeej                     ed<   dZ	ee
   ed<   dZeeej                        ed<   dZeeej                        ed<   dZeeej                        ed<   y)IdeficsBaseModelOutputWithPasta  
    last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
        Sequence of hidden-states at the output of the last layer of the model.

        If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
        hidden_size)` is output.
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks and optionally if
        `config.is_encoder_decoder=True` in the cross-attention blocks) that can be used (see `past_key_values`
        input) to speed up sequential decoding.
    image_hidden_states (`tuple(torch.FloatTensor)`, *optional*):
        Tuple of `torch.FloatTensor` (one for the output of the image embeddings, `(batch_size, num_images,
        sequence_length, hidden_size)`.

        image_hidden_states of the model produced by the vision encoder, and optionally by the perceiver
    Nlast_hidden_statepast_key_valueshidden_states
attentionsimage_hidden_states)__name__
__module____qualname____doc__r$   r   torchFloatTensor__annotations__r%   r   r&   tupler'   r(        f/var/www/html/saasai/venv/lib/python3.12/site-packages/transformers/models/idefics/modeling_idefics.pyr#   r#   0   s|    & 6:x 1 129'+OXe_+8<M8E%"3"345<59Ju00129>B%(9(9":;Br2   r#   zS
    Base class for Idefics causal language model (or autoregressive) outputs.
    c                       e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   dZ
ee   ed<   dZeeej                        ed<   dZeeej                        ed<   dZeeej                        ed<   y)	IdeficsCausalLMOutputWithPastae  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`tuple(torch.FloatTensor)`, *optional*):
        Tuple of `torch.FloatTensor` (one for the output of the image embeddings, `(batch_size, num_images,
        sequence_length, hidden_size)`.

        image_hidden_states of the model produced by the vision encoder, and optionally by the perceiver
    Nlosslogitsr%   r&   r'   r(   )r)   r*   r+   r,   r6   r   r-   r.   r/   r7   r%   r   r&   r0   r'   r(   r1   r2   r3   r5   r5   Q   s    " )-D(5$$
%,*.FHU&&'.'+OXe_+8<M8E%"3"345<59Ju00129>B%(9(9":;Br2   r5   c                    t        j                  | j                  d         j                  dd      j	                  d|      j                  d      j                  | j                        }| j                  d|      } |j                  d      |d<   |j                  d      |d<   |j                  d      |d<   |j                  d      |d<   d|v r|d   }|j                  d|      |d<   ||j                  d|      |d	<   |d   |d   j                  d|      |d<   |d   |d   j                  d|      |d<   | |fS |d   |d   j                  d|      |d<   | |fS |d   |d   j                  d|      |d<   | |fS )
Nr   r   pixel_valuesimage_encoder_embeddingsperceiver_embeddingsimage_attention_masktoken_type_idsattention_mask)	r-   arangeshapeviewrepeattodeviceindex_selectget)	input_idsexpand_sizeis_encoder_decoderr?   encoder_outputsmodel_kwargsexpanded_return_idxr>   s           r3   expand_inputs_for_generationrN   q   s    	Y__Q'(--b!4;;A{KPPQSTWWXaXhXhi  &&q*=>I#/#3#3N#CL /;/?/?@Z/[L+,+7+;+;<R+SL'(+7+;+;<R+SL'(<'%&67)7)D)DQH[)\%&!)7)D)DQH[)\%&*+7/;<R/S/`/`"0
+, N#/'3N'C'P'PQRTg'h^$ l"" 
0	1	=3?@Z3[3h3h"4
/0 l"" 
,	-	9/;<R/S/`/`"0
+, l""r2   c                 2   t         j                  t         j                  t         j                  d}|D cg c]  }||   	 }}| j	                         D ];  |r&t        fd|D              rj                  d       +j                  d       = | S c c}w )N)	LayerNormLinear	Embeddingc              3   6   K   | ]  }t        |        y wN)
isinstance).0tmodules     r3   	<genexpr>zfreeze_model.<locals>.<genexpr>   s     $]D\qZ%:D\s   TF)r   rP   rQ   rR   modulesanyrequires_grad_)modelmodule_exceptionsmappingmmodule_exceptions_mappedrX   s        @r3   freeze_modelrb      s    \\))\\G
 5FF4Eq
4EF--/$]D\$]!]!!$'!!%(	 "
 L  Gs   Bc                   N     e Zd ZdZ	 	 	 	 ddee   ddf fdZd ZdefdZ	 xZ
S )	IdeficsDecoupledEmbeddinga  
    Implements a decoupling of parameters to allow freezing (or not) a subset of the embeddings. In practise, the
    regular `weight` can be trained or frozen (i.e. `partially_freeze=True`), and if `num_additional_embeddings` > 0,
    then it will create `num_additional_embeddings` additional parameters that are always trained. If
    `num_additional_embeddings=0`, then the module defaults back to the regular behavior of `nn.Embedding`.
    Npartially_freezereturnc           	      B   |||kD  rt        d| d|       t        	|   d|||||d| || _        || _        || _        || _        |r| j                  j                  d       | j
                  dkD  r)t        j                  | j
                  |||      | _        yy)	a)  
        Args:
            num_embeddings (`int`):
                Size of the dictionary of embeddings
            num_additional_embeddings (`int`):
                Number of additional embeddings. Only useful when you `partially_freeze=True`.
            embedding_dim (`int`):
                The size of each embedding vector
            partially_freeze: (`bool`, *optional*, defaults to `False`):
                If `True`, the regular `weight` will be frozen. `additional_weight` is never frozen.
            padding_idx (`int`, *optional*):
                The padding index (needs to be less than num_embeddings)

        Note: there are a lot of other parameters to initialize a standard `nn.Embedding` such as `padding_idx`,
        `max_norm` or `norm_type`. We are not supporting these.
        Nz/padding_idx must be within num_embeddings. Got z and )num_embeddingsembedding_dimrE   dtypepadding_idxFr   )rh   ri   rE   rj   r1   )
ValueErrorsuper__init__rh   rk   num_additional_embeddingsre   weightr\   r   rR   additional_embedding)
selfrh   ro   ri   re   rE   rj   rk   kwargs	__class__s
            r3   rn   z"IdeficsDecoupledEmbedding.__init__   s    6 "{^'CN{m[`ao`pqrr 	
)'#	
 	
 -&)B& 0KK&&u-))A-(*#==+	)D% .r2   c                 b   | j                   dk(  r t        j                  || j                        S |j	                         }t        j                  || j                  k\        }||   }| j                  || j                  z
        }d||<   t        j                  || j                        }|||<   |S )a  
        we have 2 embeddings, with different indices - one pretrained self.weight and another
        self.additional_embedding.weight that is being trained.

        in order to make a lookup of the input ids, we:
        1. find out the indices of the entries belonging to the 2nd embedding
        2. extract those values while subtracting the size of the first embedding (num_embeddings), since the 2nd
           embedding starts from 0 and not num_embeddings
        3. perform the 2nd embedding lookup
        4. now we handle the 1st embedding, we overwrite indices belonging to the 2nd embedding with a padding index
        5. perform the 1st embedding lookup
        6. now we overwrite the values in the 1st embedding lookup with the values of the 2nd embedding lookup

        note: for the 1st embedding lookup we could have looked up only the low indices and not do the padding, but
        then we have to create a new tensor and populate it with 2 tensors that are spread out across various indices -
        i.e. not a simple concat - I haven't benchmarked the complex case if it's any faster, given that seqlens are
        usually relatively short it's probably not faster or if faster not by much - but might be a good idea to
        measure.

        r   )	ro   F	embeddingrp   cloner-   whererh   rq   )rr   rH   additional_vocab_indicesinput_ids_additional_vocabadditional_embeddingsfull_vectors         r3   forwardz!IdeficsDecoupledEmbedding.forward   s    * ))Q.;;y$++66 OO%	#(;;yD<O<O/O#P %./G%H" $ 9 9:TW[WjWj:j k /0	*+kk)T[[9 1F,-r2   c                 n    d| j                    d| j                   d| j                   d| j                   S )Nznum_embeddings=z, num_additional_embeddings=z, embedding_dim=, partially_freeze=)rh   ro   ri   re   rr   s    r3   
extra_reprz$IdeficsDecoupledEmbedding.extra_repr  sq     !4!4 55QRVRpRpQq  rB  CG  CU  CU  BV  Vi  jn  j  j  i@  A  	Ar2   )FNNN)r)   r*   r+   r,   r   boolrn   r~   strr   __classcell__rt   s   @r3   rd   rd      sH     ,13
 #4.3 
3j%NAC Ar2   rd   c                        e Zd ZdZ	 	 	 	 	 ddedededededdf fd	Zd
ej                  dej                  fdZ	de
fdZ xZS )IdeficsDecoupledLineara  
    Implements a decoupling of parameters to allow freezing (or not) a subset of the parameters. In practise, the
    regular `weight` can be trained or frozen (i.e. `partially_freeze=True`), and if `out_additional_features` > 0,
    then it will create `out_additional_features * in_features` additional parameters that are always trained. If
    `out_additional_features=0`, then the module defaults back to the regular behavior of `nn.Linear`.
    Nin_featuresout_featuresout_additional_featuresbiasre   rf   c                 "   t         |   |||||       || _        || _        || _        || _        |r8| j                  j                  d       |r| j                  j                  d       |dkD  r t        j                  |||||      | _        yy)aG  
        out_additional_features: int. Number of additional trainable dimensions. Only makes sense when
        `partially_freeze=True`. partially_freeze: bool. If True, the regular `weight` will be frozen and extra
        parameters (if any) will be trainable. If False, default to the regular behavior of nn.Linear.
        Fr   )r   r   r   rE   rj   N)rm   rn   r   re   r   r   rp   r\   r   r   rQ   additional_fc)	rr   r   r   r   r   re   rE   rj   rt   s	           r3   rn   zIdeficsDecoupledLinear.__init__  s     	lD&%H'>$ 0&(KK&&u-		((/"Q&!#'4"D 'r2   inputc                     t        j                  || j                  | j                        }| j                  dkD  r)| j                  |      }t        j                  ||fd      }|S )Nr   r9   )rv   linearrp   r   r   r   r-   cat)rr   r   outputadditional_featuress       r3   r~   zIdeficsDecoupledLinear.forwardC  sV    %dii8''!+"&"4"4U";YY(;<bAFr2   c           
          d| j                    d| j                   d| j                   d| j                  du d| j                   
S )z=Overwriting `nn.Linear.extra_repr` to include new parameters.zin_features=z, out_features=z, out_additional_features=z, bias=Nr   r   r   r   r   re   r   s    r3   r   z!IdeficsDecoupledLinear.extra_reprL  s    d../t?P?P>QQklp  mI  mI  lJ  JQ  RV  R[  R[  cg  Rg  Qh  h{  |@  |Q  |Q  {R  S  	Sr2   )r   TTNN)r)   r*   r+   r,   intr   rn   r-   Tensorr~   r   r   r   r   s   @r3   r   r     s     ()!%"" " "%	"
 " " 
"HU\\ ell SC Sr2   r   c                   ,     e Zd Zd fd	Zd Zd Z xZS )IdeficsRMSNormc                     t         |           t        j                  t	        j
                  |            | _        || _        y)z=
        IdeficsRMSNorm is equivalent to T5LayerNorm
        N)rm   rn   r   	Parameterr-   onesrp   variance_epsilon)rr   hidden_sizeepsrt   s      r3   rn   zIdeficsRMSNorm.__init__S  s1     	ll5::k#:; #r2   c                    |j                  t        j                        j                  d      j	                  dd      }|t        j
                  || j                  z         z  }| j                  j                  t        j                  t        j                  fv r%|j                  | j                  j                        }| j                  |z  S )N   r9   T)keepdim)rD   r-   float32powmeanrsqrtr   rp   rj   float16bfloat16)rr   r&   variances      r3   r~   zIdeficsRMSNorm.forward[  s     ##EMM266q9>>r4>P%Ht?T?T4T(UU ;; ??),,T[[->->?M{{]**r2   c                 ^    t        | j                  j                         d| j                   S )Nz, eps=)r0   rp   rA   r   r   s    r3   r   zIdeficsRMSNorm.extra_repre  s*    ))*+6$2G2G1HIIr2   )gư>)r)   r*   r+   rn   r~   r   r   r   s   @r3   r   r   R  s    $+Jr2   r   c                   .     e Zd Zd fd	Zd ZddZ xZS )IdeficsEmbeddingc                    t         |           || _        || _        || _        d| j                  t        j                  d| j                  dt
        j                        j                  |t
        j                        | j                  z  z  z  }| j                  d|d       | j                  || j                  j                  t        j                         	       y )
N      ?r   r   rj   rE   rj   inv_freqF
persistentseq_lenrE   rj   )rm   rn   dimmax_position_embeddingsbaser-   r@   int64rD   floatregister_buffer_set_cos_sin_cacher   rE   get_default_dtype)rr   r   r   r   rE   r   rt   s         r3   rn   zIdeficsEmbedding.__init__k  s    '>$	IIQ!5;;?BB&X]XcXcBdgkgogooq
 	ZeD 	+DMM4H4HPUPgPgPi 	  	
r2   c                    || _         t        j                  | j                   |t        j                        j	                  | j
                        }t        j                  d|| j
                        }t        j                  ||fd      }| j                  d|j                         j                  |      d       | j                  d|j                         j                  |      d       y )	Nr   zi,j->ijr9   r   
cos_cachedFr   
sin_cached)max_seq_len_cachedr-   r@   r   type_asr   einsumr   r   cosrD   sin)rr   r   rE   rj   rW   freqsembs          r3   r   z#IdeficsEmbedding._set_cos_sin_cache|  s    ")LL00u{{S[[\`\i\ijY4==9iiB/\3779<<+>5Q\3779<<+>5Qr2   c                    || j                   kD  r(| j                  ||j                  |j                         | j                  d | j                  |j                        | j                  d | j                  |j                        fS )Nr   r   )r   r   rE   rj   r   rD   r   )rr   xr   s      r3   r~   zIdeficsEmbedding.forward  sy    T,,,##GAHHAGG#T OOHW%((qww(7OOHW%((qww(7
 	
r2   )i   i'  NrT   )r)   r*   r+   rn   r   r~   r   r   s   @r3   r   r   j  s    
"R
r2   r   c                     | dd| j                   d   dz  f   }| d| j                   d   dz  df   }t        j                  | |fd      S )z*Rotates half the hidden dims of the input..Nr9   r   r   )rA   r-   r   )r   x1x2s      r3   rotate_halfr     sZ    	
3"!''"+"""	#B	
3q ""	#B99rc2YB''r2   c                     ||   j                  |      }||   j                  |      }| |z  t        |       |z  z   }||z  t        |      |z  z   }||fS )an  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`):
            The position indices of the tokens corresponding to the query and key tensors. For example, this can be
            used to pass offsetted position ids when working with a KV-cache.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )	unsqueezer   )qkr   r   position_idsunsqueeze_dimq_embedk_embeds           r3   apply_rotary_pos_embr     sg    * l

%
%m
4C
l

%
%m
4C3w;q>C/0G3w;q>C/0GGr2   c                   2     e Zd Zdededef fdZd Z xZS )
IdeficsMLPr   intermediate_size
hidden_actc                     t         |           t        j                  ||d      | _        t        j                  ||d      | _        t        j                  ||d      | _        t        |   | _        y )NFr   )	rm   rn   r   rQ   	gate_proj	down_projup_projr
   act_fn)rr   r   r   r   rt   s       r3   rn   zIdeficsMLP.__init__  s[     	;0AN#4kNyy.?eLZ(r2   c                     | j                  | j                  | j                  |            | j                  |      z        S rT   )r   r   r   r   )rr   r   s     r3   r~   zIdeficsMLP.forward  s0    ~~dkk$..*;<t||ANOOr2   )r)   r*   r+   r   r   rn   r~   r   r   s   @r3   r   r     s*    
)
) 
) 	
)Pr2   r   rX   querykeyvaluer?   scalingdropoutc                    t        j                  ||j                  dd            |z  }|||z   }t        j                  j                  |dt         j                        j                  |j                        }t        j                  j                  ||| j                        }t        j                  ||      }	|	j                  dd      j                         }	|	|fS )Nr9   )r   rj   ptrainingr   r   )r-   matmul	transposer   
functionalsoftmaxr   rD   rj   r   r   
contiguous)
rX   r   r   r   r?   r   r   rs   attn_weightsattn_outputs
             r3   eager_attention_forwardr     s     <<s}}R'<=GL!#n4==((2U]](SVVW\WbWbcL==((6??([L,,|U3K''1-88:K$$r2   c                       e Zd ZdZ	 	 	 	 	 ddededededee   dedee   f fd	Z	d
e
j                  dedefdZ eddd      	 	 	 	 	 dde
j                  dee
j                     dee
j                     dee
j                     dee   dee
j                     dee   dee
j                  e
j                  f   fd       Z xZS )IdeficsAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr   	num_headsr   is_cross_attentionconfigqk_layer_norms	layer_idxc                    t         	|           || _        || _        || _        ||z  | _        || _        d| _        | j
                  dz  | _        || _	        |-t        j                  d| j                  j                   d       | j
                  |z  | j                  k7  rt        d| j                   d| d      || _        t!        t"        j$                  d      st        d	      | j                  rt!        |j&                  d
      s| j                  n|j&                  j(                  }t#        j*                  | j                  || j
                  z  d      | _        t#        j*                  ||| j
                  z  d      | _        t#        j*                  ||| j
                  z  d      | _        nt#        j*                  | j                  || j
                  z  d      | _        t#        j*                  | j                  || j
                  z  d      | _        t#        j*                  | j                  || j
                  z  d      | _        t#        j*                  || j
                  z  |d      | _        t5        | j
                        | _        || _        | j8                  rMt;        | j
                  |j<                        | _        t;        | j
                  |j<                        | _         y y )NTg      zInstantiating z without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.z?hidden_size must be divisible by num_heads (got `hidden_size`: z and `num_heads`: z).scaled_dot_product_attentionz)this model requires pytorch 2.0 or higher	embed_dimFr   r   )!rm   rn   r   r   r   head_dimr   	is_causalr   r   loggerwarning_oncert   r)   rl   r   hasattrr   r   vision_configr   rQ   q_projk_projv_projo_projr   
rotary_embr   r   rms_norm_epsq_layer_normk_layer_norm)
rr   r   r   r   r   r   r   r   kv_input_dimrt   s
            r3   rn   zIdeficsAttention.__init__  s    	&"#y0}}d*" !8!8 9 :, , MMI%$*:*::QRVRbRbQc$YKr3 
 #5r}}&DEHII""(/0D0Dk(R  X^XlXlXvXv  ))  DMM)DK
 ))L)dmm2KRWXDK))DMM)DK ))  DMM)DK
 ))  DMM)DK
 ))  DMM)DK
 ii%

 +4==9, .t}}&BUBU VD .t}}&BUBU VD r2   tensorr   bszc                     |j                  ||| j                  | j                        j                  dd      j	                         S )Nr   r   )rB   r   r  r   r   )rr   r  r   r  s       r3   _shapezIdeficsAttention._shape3  s7    {{3GQQRSUVWbbddr2   past_key_valuer%   4.58new_nameversionr&   key_value_statesr?   r   cache_positionrs   rf   c                    | j                   xs |d u}|j                         \  }	}
}| j                  |      j                  |	|
| j                  | j
                        j                  dd      }|s| j                  |      j                  |	|
| j                  | j
                        j                  dd      }| j                  |      j                  |	|
| j                  | j
                        j                  dd      }n|j                         \  }}}| j                  |      j                  |	|| j                  | j
                        j                  dd      }| j                  |      j                  |	|| j                  | j
                        j                  dd      }|j                  d   }|||d   z  }|s2| j                  |t        ||
            \  }}t        |||||      \  }}|%d|i}|j                  ||| j                  |      \  }}| j                  r"| j!                  |      }| j#                  |      }t$        }| j&                  j(                  dk7  rt*        | j&                  j(                     } || ||||f| j,                  sdn| j.                  | j0                  d	|\  }}|j3                  |	|
d
      j5                         }| j7                  |      }||fS )Nr   r   r   r   )r   r  eager        )r   r   r9   )r   sizer  rB   r   r  r   r  r	  rA   r  maxr   updater   r   r  r  r   r   _attn_implementationr   r   r   r   reshaper   r
  )rr   r&   r  r?   r   r%   r  rs   r   r  q_len_query_states
key_statesvalue_stateskv_len
kv_seq_lenr   r   cache_kwargsattention_interfacer   r   s                          r3   r~   zIdeficsAttention.forward6  s    "44T8HPT8T%**,UA{{=166sE4>>SWS`S`akklmopq!]388eT^^UYUbUbcmmnoqrsJ;;}5::3t~~W[WdWdeoopqstuL+002LAvq%56;;CY]YfYfgqqrsuvwJ,-223PTP]P]^hhijlmn   %%b)
&.++J!|SU=STHC';L*VY[^`l'm$L* &,n=L'6'='=j,X\XfXfht'u$J,,\:L**:6J(?;;++w6"9$++:Z:Z"[$7	%
  $}}C$,,LL	%
 	%
!\ "))#ub9DDFkk+.L((r2   )r  FNFNNNNNN)r)   r*   r+   r,   r   r   r   r   r   rn   r-   r   r  r   
LongTensorr   r   r   r0   r~   r   r   s   @r3   r   r     sd   G #(-1$#'OWOW OW 	OW
 !OW )*OW OW C=OWbeU\\ eC ec e %0A6R 481537+/59?)||?) #5<<0?) !.	?)
 u//0?) "%?) !!1!12?) +,?) 
u||U\\)	*?) S?)r2   r   c                       e Zd Zddedee   f fdZ eddd      e	 	 	 	 dde	j                  d	ee	j                     d
ee	j                     dee   dee	j                     dee   de	j                  fd              Z xZS )IdeficsDecoderLayerr   r   c                    t         |           |j                  | _        t        | j                  |j                  |j
                  ||      | _        t        | j                  |j                  |j                        | _
        t        |j                  |j                        | _        t        |j                  |j                        | _        |j
                  | _        y )N)r   r   r   r   r   r   r   r   r   )rm   rn   r   r   num_attention_headsr   	self_attnr   r   r   mlpr   r  input_layernormpost_attention_layernormrr   r   r   rt   s      r3   rn   zIdeficsDecoderLayer.__init__{  s    !--)((00NN
 (($66((

  .f.@.@fFYFYZ(6v7I7IvObOb(c%~~r2   r  r%   r  r  r&   r?   r   r  rs   rf   c           	         |}| j                  |      } | j                  d|||||d|\  }}t        j                  j	                  || j                  | j
                        }||z   }|}| j                  |      }| j                  |      }t        j                  j	                  || j                  | j
                        }||z   }|S )N)r&   r?   r   r%   r  r   r1   )r5  r3  r   r   r   r   r6  r4  )	rr   r&   r?   r   r%   r  rs   residualr$  s	            r3   r~   zIdeficsDecoderLayer.forward  s     !,,]; *4>> 
')%+)
 
q --mt||VZVcVc-d =0 !55mD/--mt||VZVcVc-d =0r2   rT   )NNNN)r)   r*   r+   r   r   r   rn   r   r   r-   r   r-  r   r   r   r.   r~   r   r   s   @r3   r/  r/  z  s    &} &# && %0A6R 2637+/59 ||  !.  u//0	 
 "%  !!1!12  +,  
		   S r2   r/  c                   &    e Zd Zddedee   f fdZ eddd      e	 	 	 	 	 dde	j                  d	ee	j                     d
ee	j                     dee	j                     dee	j                     dee   dee   de	j                  fd              Z xZS )IdeficsGatedCrossAttentionLayerr   r   c           	      	   t         |           |j                  | _        t        | j                  |j                  d|j
                  ||j                  |      | _        t        | j                  |j                  |j                        | _        t        |j                  |j                        | _        t        |j                  |j                        | _        |j
                  | _        t#        j$                         | _        t#        j$                         | _        |j*                  dk(  r|j,                  dk(  rtt#        j.                  t1        j2                  dd| j                              | _        t#        j.                  t1        j2                  dd| j                              | _        n|j,                  dk(  r\t#        j.                  t1        j2                  d            | _        t#        j.                  t1        j2                  d            | _        nt9        d	|j,                   d
      |j*                  dk(  r|j,                  dk(  rtt#        j.                  t1        j:                  dd| j                              | _        t#        j.                  t1        j:                  dd| j                              | _        n|j,                  dk(  r\t#        j.                  t1        j:                  d            | _        t#        j.                  t1        j:                  d            | _        n|t9        d	|j,                   d
      |j*                  dv r;|j,                  dk(  rt#        j.                  t1        j<                  d|j>                  dd| j                  f            | _        t#        j.                  t1        j<                  d|j>                  dd| j                  f            | _        n|j,                  dk(  rut#        j.                  t1        j<                  d|j>                  d            | _        t#        j.                  t1        j<                  d|j>                  d            | _        n2t9        d	|j,                   d
      tA        d|j*                   d      tC        | d      rtC        | d      st9        d      y )NT)r   r   r   r   r   r   r   r1  r   zerosvectorr   r   z Unknown value for `alpha_type` ()r   >   normalrandomgaussianr  )r   stdr  zAlpha initialization scheme z not yet implemented!alpha_cross_attnalpha_densez+Alpha parameters not initialized correctly!)"rm   rn   r   r   r2  r   r   
cross_attnr   r   r   r4  r   r  r5  r6  r   r   Tanhact_cross_attn	act_densealpha_initializer
alpha_typer   r-   r=  rD  rE  rl   r   r@  alphas_initializer_rangeNotImplementedErrorr  r7  s      r3   rn   z(IdeficsGatedCrossAttentionLayer.__init__  s   !--*((00#NN!00
 (($66((

  .f.@.@fFYFYZ(6v7I7IvObOb(c%nn ggi##w.  H,(*U[[AtGWGW5X(Y%#%<<Aq$BRBR0S#T ""g-(*U[[^(D%#%<<A#?  #CFDUDUCVVW!XYY%%/  H,(*UZZ1dFVFV5W(X%#%<<

1aAQAQ0R#S ""g-(*UZZ](C%#%<<

1#>  #CFDUDUCVVW!XYY%%)II  H,(*LLcv/N/NVWYZ\`\l\lUmn)% $&<<LLcv/N/NVWYZ\`\l\lUmn$  ""g-(*LLcv/N/NVWY)% $&<<#6KjKjrs0u#v  #CFDUDUCVVW!XYY &(DVE]E]D^^s&tuu01gdM6RJKK 7Sr2   r  r%   r  r  r&   r?   r(   r=   cross_attention_gaters   rf   c                    |t        d      |t        d      |t        d      |}| j                  |      } | j                  d	|||d|\  }}	t        j
                  j                  || j                  | j                        }|j                  |dk(  dddddf   d      }|| j                  | j                        |z  z   }|}| j                  |      }| j                  |      }t        j
                  j                  || j                  | j                        }|| j                  | j                        |z  z   }|S )
a  
        image_hidden_states (`torch.FloatTensor`):
            Input to the layer of shape `(batch, seq_len, embed_dim)`
        image_attention_mask (`torch.FloatTensor`, *optional*):
            image attention mask of size
            `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
        cross_attention_gate (`torch.FloatTensor`, *optional*):
            gate of size `(batch, seq_len)` used to zero-out cross-attention output for tokens attending no images.
        Nzt`image_hidden_states` is required for Idefics cross attention module which are visual features to be conditioned on.z`cross_attention_gate` is required for Idefics cross attention module to zero-out the cross-attention hidden_states attending to no images.zMPast key value states are not implemented for Idefics cross attention module.)r&   r  r?   r   r   r  r1   )rl   rM  r5  rF  r   r   r   r   r   masked_fillrH  rD  r6  r4  rI  rE  )
rr   r&   r?   r(   r=   rN  r%   rs   r9  r$  s
             r3   r~   z'IdeficsGatedCrossAttentionLayer.forward  sm   * &# 
  ' ^  &%&uvv ,,]; +4?? 
'0/
 	
q --mt{{UYUbUb-c%113G13LaQRTXj2Y[^_ 4#6#6t7L7L#MP]#]] !55mD/--mt{{UYUbUb-c 4>>$2B2B#Cm#SSr2   rT   r,  )r)   r*   r+   r   r   r   rn   r   r   r-   r   r   r   r   r.   r~   r   r   s   @r3   r;  r;    s    @L} @L# @LD %0A6R 266:7;7;+/8||8 !.8 &ell3	8
 'u||48 'u||48 "%8 +,8 
		8  S8r2   r;  c                   Z    e Zd ZU eed<   dZdZddgZdZdZ	dZ
dZe eedd	      d
Zd Zy)IdeficsPreTrainedModelr   r]   Tr/  r;  Fr   r3  )index
layer_name)r&   r'   c                    | j                   j                  }t        |t        j                  t        j
                  f      rY|j                  j                  j                  d|       |j                  %|j                  j                  j                          y y t        |t        j                        rf|j                  j                  j                  d|       |j                  2|j                  j                  |j                     j                          y y t        |t        j                        rJ|j                  j                  j                  d       |j                  j                  j                          y t        |t              r&|j                  j                  j                  d       y t        |t               r%|j"                  j                  j                          y t        |t$              rV| j                   j&                  dk(  rI|j(                  j                  j                          |j*                  j                  j                          y | j                   j&                  dk(  rK|j(                  j                  j                  d       |j*                  j                  j                  d       y | j                   j&                  dv rw|j(                  j                  j                  d| j                   j,                         |j*                  j                  j                  d| j                   j,                         y y t        |t.              r%|j0                  j                  j                          y y )Nr  )r   rC  r   r=  r   >   r@  rA  rB  )r   initializer_rangerU   r   rQ   Conv2drp   datanormal_r   zero_rR   rk   rP   fill_r   r   class_embeddingr;  rJ  rD  rE  rL  r   latents)rr   rX   rC  s      r3   _init_weightsz$IdeficsPreTrainedModel._init_weightsD  sp    kk++fryy"))45MM&&CS&9{{&  &&( '-MM&&CS&9!!-""6#5#56<<> .-MM$$S)KK""$/MM$$S) 78""''//1 ?@{{,,7'',,224""''--/..&8'',,2237""''--c2..2RR'',,44#4;;CgCg4h""''//Sdkk>b>b/c S  9:NN'') ;r2   N)r)   r*   r+   r   r/   base_model_prefixsupports_gradient_checkpointing_no_split_modules_supports_sdpa_supports_flash_attn_can_compile_fullgraph_supports_attention_backendr/  r   r   _can_record_outputsr^  r1   r2   r3   rR  rR  3  sV    &*#.0QRN ""& -$%5Q;W
*r2   rR  c            !           e Zd ZdZdef fdZddZg fdZg fdZe	e
	 	 	 	 	 	 	 	 	 	 	 	 ddeej                     deej                     d	eej                     d
ee   deej                      deej                      deej                      deej                      deej                     dee   dee   deej                     dee   deeef   fd              Z xZS )IdeficsModelz
    Transformer decoder consisting of `config.num_hidden_layers` layers. Each layer is a [`IdeficsDecoderLayer`]

    Args:
        config: IdeficsConfig
    r   c           	         t         |   |       || _        |j                  | _        |j
                  | _        t        |j
                  |j                  |j                  |j                  | j                        | _
        |j                  j                  | _        |j                  | _        |j                  | j                  _        t        |j                        | _        |j                   r]|j"                  }t%        ||j                  j&                  |j(                  |j*                  |j,                  |j.                        | _        t3        j4                  t7        |j8                        D cg c]  }t;        ||       c}      | _        |j>                  | _        |j8                  | j>                  z  }t3        j4                  t7        |      D cg c]  }tA        ||       c}      | _!        d| _"        tG        |j                  |jH                        | _%        | jM                          | jO                  |       y c c}w c c}w )N)rh   ro   ri   re   rk   )r   Fr   )(rm   rn   r   pad_token_idrk   
vocab_sizerd   additional_vocab_sizer   freeze_text_layersembed_tokensr  
image_sizer!  r    vision_modeluse_resamplerperceiver_configr   r   resampler_depthresampler_n_headsresampler_head_dimresampler_n_latentsperceiver_resamplerr   
ModuleListrangenum_hidden_layersr/  layerscross_layer_intervalr;  gated_cross_attn_layersgradient_checkpointingr   r  norm	post_initfreeze_relevant_params)rr   r   rr  inum_cross_layersrt   s        r3   rn   zIdeficsModel.__init__o  s    !.. ++5!,,&,&B&B ,,#66((
 !..99#11282M2M/4V5I5IJ %66'@$$.. 00 22 33 44(D$ mm?DVE]E]?^_?^! 15?^_
 %+$?$?!!33t7P7PP')}}KPQaKbcKba,VqAKbc(
$ ',#"6#5#56;N;NO	 	##F+ ` ds   1IIc                     || j                   }|j                  r| j                  |j                         |j                  r"t	        | j
                  |j                         y y N)r^   )r   rm  freeze_text_module_exceptionsfreeze_vision_layersrb   rp  freeze_vision_module_exceptions)rr   r   s     r3   r  z#IdeficsModel.freeze_relevant_params  sQ    >[[F$$##F$H$HI&&**f>d>de 'r2   c                 X    | j                   | j                  fD ]  }t        ||        y r  )r{  r  rb   )rr   r^   rX   s      r3   rm  zIdeficsModel.freeze_text_layers  s$    {{DII.F3DE /r2   c                 2    t        | j                  |       y r  )rb   rp  )rr   r^   s     r3   r  z!IdeficsModel.freeze_vision_layers  s    T&&:KLr2   rH   r?   r   r%   inputs_embedsr:   r;   r<   r=   	use_cacheinterpolate_pos_encodingr  rs   rf   c           	         ||j                   n|j                   }|du |duz  rt        d      || j                  |      }|
r|t        | j                        }|j
                  \  }}}||j                         nd}||z   }|2t        j                  |||j
                  d   z   |j                         }|F|D|j                         j                  d      dz
  }|j                  |dk(  d       |dd| df   }n||j                  d      }t        d |||fD              d	k7  rt        d
      |~|j                  | j                  |      }|j
                  dd	 \  }} |j!                         j"                  ||z  g|j
                  d	d  }| j%                  ||      j&                  }nJ|H|j)                         \  }}}}|j                  | j                  |      }|j#                  ||z  ||      }| j                  j*                  rN|4| j-                        }|j)                  d      |j)                  d	      }}n|j)                         \  }}}}|}n0|#j)                  d      |j)                  d	      }}nt        d      |j#                  ||z  |      }|	j)                  d      }|	j                  d      }	|	j/                  ddd|      }	|	j#                  ||||z        }	|C|j)                         \  }}}||f}|	t        j0                  ||      }	| j3                  |	      }	nd}	|	dk(  j5                  d      j                  | j                        j7                  d      j                  |      }|2t        j0                  ||ft        j8                  |j                         }t;        | j                  |||||      }|}t=        | j>                        D ]P  \  }} || j@                  z  dk(  r+| jB                  || j@                  z     }! |!|||f|	|dd|} | |f||||d|}R | jE                  |      }|j#                  ||||      }tG        |||      S )ab  
        image_encoder_embeddings (`torch.FloatTensor`, *optional*):
            The output of the image encoder.
        perceiver_embeddings (`torch.FloatTensor`, *optional*):
            The output of the perceiver resampler.
        image_attention_mask (`torch.LongTensor`, *optional*):
            The attention mask for the image encoder.
        Nz:You must specify exactly one of input_ids or inputs_embeds)r   r   r   )rE   r9   c              3   $   K   | ]  }|d u  
 y wrT   r1   )rV   r   s     r3   rY   z'IdeficsModel.forward.<locals>.<genexpr>  s     a"`QqDy"`s   r   z_Exactly 1 of pixel_values, image_encoder_embeddings or perceiver_embeddings has to be not-None.)rj   rE   )r:   r  zBIf `perceiver_embeddings` are passed, use_resampler should be Truer  r   r   )r   input_embedsr?   r  r%   r   )r=   rN  r%   )r?   r   r%   r  )r$   r(   r%   )$rE   rl   rn  r   r   rA   get_seq_lengthr-   r@   longcumsummasked_fill_r   sumrD   rj   r   rB   rp  r$   r  rq  rw  rC   r   invert_attention_maskr[   squeezer   r   	enumerater{  r|  r}  r  r#   )"rr   rH   r?   r   r%   r  r:   r;   r<   r=   r  r  r  rs   rE   
batch_size
seq_lengthr$  past_key_values_lengthseq_length_with_past
num_imagesr(   image_seq_lenimage_hidden_sizetext_seq_lenimage_batch_sizeimage_sequence_lengthimage_hidden_shaperN  causal_maskr&   idxdecoder_layercross_attn_blocks"                                     r3   r~   zIdeficsModel.forward  s.   4 &/%:!!@T@T-t";<YZZ  --i8M0*$++>O$1$7$7!
JETE`!?!?!Afg),BB!"\\&(>ATATUVAW(W`m`t`tN %,*>)..077;a?L%%n&91='J;<8L!)33A6La<1IK_"`aaeffq  %'??F?KL%1%7%7%;"J
9<22499*z:QkT`TfTfghgiTjkL #'"3"3)D\ #4 #   &1G_GdGdGfDJ
M3D":"="=DJJW]"="^"5":"::
;RTact"u;;$$#+'+'?'?@S'T$3G3L3LQ3OQeQjQjklQm0K_KdKdKfH
J7H"6!)/B/G/G/JL_LdLdefLg,Mabb166z:P]C]_pq ,0033==bA3::1aMR388\S]`mSmn*9L9Q9Q9S63Q"24I!J#+',zz2DV'T$#'#=#=>R#S #'  $83#>"C"C"C"K!O!OVZV`V`!O!a j jop j quu 

 !"ZZ12%**]MaMaN );;&))+%
 &"+DKK"8CT...!3#'#?#?tG`G`@`#a  0!'! *>)=$(! ! **) /- M #9. 		-0166z:}^op-+ 3+
 	
r2   rT   )NNNNNNNNNNFN)r)   r*   r+   r,   r   rn   r  rm  r  r   r   r   r-   r-  r   r   r.   r   r   r   r   r0   r#   r~   r   r   s   @r3   rh  rh  f  s   0,} 0,df 46 F 68 M  151537+/5948@D<@7;$(3859^
E,,-^
 !.^
 u//0	^

 "%^
   1 12^
 u001^
 #+5+<+<"=^
 'u'8'89^
 'u||4^
 D>^
 #+4.^
 !!1!12^
 +,^
 
u44	5^
  ^
r2   rh  c            #           e Zd ZddgZd fd	Zd Zee	 	 	 	 	 	 	 	 	 	 	 	 	 ddee	j                     dee	j                     dee	j                     dee   d	ee	j                     d
ee	j                     dee	j                     dee	j                     dee	j                     dee	j                     dee   dee   dee	j                     dee   deeef   fd              Z	 	 	 	 	 	 	 	 	 d fd	Z	 ddedeeef   dedeeef   f fdZ xZS )IdeficsForVisionText2Textzmodel.embed_tokens.weightzlm_head.weightc                     t         |   |       t        |      | _        t	        |j
                  |j                  |j                  d|j                        | _	        | j                          y )NFr   )rm   rn   rh  r]   r   r   rk  rl  freeze_lm_headlm_headr  )rr   r   rp  rt   s      r3   rn   z"IdeficsForVisionText2Text.__init__X  s[     !&)
-****$*$@$@#22
 	r2   c                    | j                         }| j                         }t        | j                  dd      r`|j                  |_        |j
                  dkD  r@|j                  |j
                  k(  sJ |j                  j                  |j                  _        t        |d      rJt        |d      r=|j                  |_        t        |d      rt        |d      r|j
                  |_        yyyyy)	z
        Overwrite `transformers.modeling_utils.PreTrainedModel.tie_weights` to handle the case of
        IdeficsDecoupledLinear and IdeficsDecoupledEmbedding.
        tie_word_embeddingsTr   r   rh   r   ro   N)get_output_embeddingsget_input_embeddingsgetattrr   rp   ro   r   rq   r   r  rh   r   )rr   output_embeddingsinput_embeddingss      r3   tie_weightsz%IdeficsForVisionText2Text.tie_weightsg  s    
 !6684464;; 5t<'7'>'>$99A=(@@DTDnDnnnn9I9^9^9e9e!//6$n5'BRTd:e-=-L-L*(*CD "=J =M<f<f!9JD ;f5r2   rH   r?   r   r%   r  r:   r;   r<   r=   labelsr  r  r  rs   rf   c                 <    | j                   d|||||||||	||d|d|}|d   }| j                  |      }d}|
* | j                  d||
| j                  j                  d|}t        |||j                  |j                  |j                  |j                        S )aC  
        image_encoder_embeddings (`torch.FloatTensor`, *optional*):
            The output of the image encoder.
        perceiver_embeddings (`torch.FloatTensor`, *optional*):
            The output of the perceiver resampler.
        image_attention_mask (`torch.LongTensor`, *optional*):
            The attention mask for the image encoder.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoProcessor, IdeficsForVisionText2Text

        >>> model = IdeficsForVisionText2Text.from_pretrained("HuggingFaceM4/idefics-9b")
        >>> processor = AutoProcessor.from_pretrained("HuggingFaceM4/idefics-9b")

        >>> dogs_image_url_1 = "https://huggingface.co/datasets/hf-internal-testing/fixtures_nlvr2/raw/main/image1.jpeg"
        >>> dogs_image_url_2 = "https://huggingface.co/datasets/hf-internal-testing/fixtures_nlvr2/raw/main/image2.jpeg"

        >>> prompts = [
        ...     [
        ...         "User:",
        ...         dogs_image_url_1,
        ...         "Describe this image.\nAssistant: An image of two dogs.\n",
        ...         "User:",
        ...         dogs_image_url_2,
        ...         "Describe this image.\nAssistant:",
        ...     ]
        ... ]
        >>> inputs = processor(prompts, return_tensors="pt")
        >>> generate_ids = model.generate(**inputs, max_new_tokens=6)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True)
        ```T)rH   r?   r   r%   r  r:   r;   r<   r=   r  r  return_dictr  r   N)r7   r  rk  )r6   r7   r%   r&   r'   r(   r1   )
r]   r  loss_functionr   rk  r5   r%   r&   r'   r(   )rr   rH   r?   r   r%   r  r:   r;   r<   r=   r  r  r  r  rs   outputsr&   r7   r6   s                      r3   r~   z!IdeficsForVisionText2Text.forward|  s    r $** 
)%+'%%=!5!5%=)
 
"  
m,%4%%pVFt{{OeOepiopD,#33!//)) ' ; ;
 	
r2   c                    i }|"| j                   j                  r||d<   n||d<   n||d<   |j                  dd      |d<   t        |   |f||||||
|	d||}|	#|!|d   j
                  d   }|	d d | d f   |d	<   |S )
Nr<   r;   r:   r  F)r%   r?   r  r  r   r  r=   rH   r   r=   )r   rq  poprm   prepare_inputs_for_generationrA   )rr   rH   r?   r   r  r%   r  r:   r(   r=   r  rs   images_kwargsmodel_inputsr  rt   s                  r3   r  z7IdeficsForVisionText2Text.prepare_inputs_for_generation  s      *{{((8K45<O89,8M.)4:JJ?Y[`4a01w<
+)')%!5
 
 
  +0E%k288;J3GJ;<3XL/0r2   r  rL   rJ   c                     t        |   |||fi |}d|v rT|d   }|d d dd d f   j                  d      }|j                  dd      r||d<   nt	        j
                  ||gd      |d<   |j                  |d<   |S )Nr=   r9   r   r  Tr   r(   )rm   #_update_model_kwargs_for_generationr   rG   r-   r   r(   )rr   r  rL   rJ   rs   r=   	last_maskrt   s          r3   r  z=IdeficsForVisionText2Text._update_model_kwargs_for_generation  s     wB
 	
 "\1#/0F#G ,QAX6@@CIT27@347<yyBVXaAbhi7j34 /6.I.I*+r2   rT   )NNNNNNNNNNNFN)	NNNNNNNNN)F)r)   r*   r+   _tied_weights_keysrn   r  r   r   r   r-   r-  r   r   r.   r   r   r   r   r0   r5   r~   r  r   dictr   r   r  r   r   s   @r3   r  r  U  s   57GHg*  151537+/5948@D<@7;-1$(3859V
E,,-V
 !.V
 u//0	V

 "%V
   1 12V
 u001V
 #+5+<+<"=V
 'u'8'89V
 'u||4V
 ))*V
 D>V
 #+4.V
 !!1!12V
 +,V
  
u33	4!V
  V
v  !+b $)	 38n !	 
c3h r2   r  )r  rh  rR  )r   FNN)r   )r  )Lr,   dataclassesr   typingr   r   r   r   r-   torch.nn.functionalr   r   rv   activationsr
   cache_utilsr   r   
generationr   masking_utilsr   modeling_layersr   modeling_outputsr   modeling_utilsr   r   r   processing_utilsr   utilsr   r   r   r   utils.deprecationr   utils.genericr   r   configuration_ideficsr   	perceiverr   visionr   r    
get_loggerr)   r  r#   r5   rN   rb   rR   rd   rQ   r   Moduler   r   r   r   r   r   r   r   r   r/  r;  rR  rh  r  __all__r1   r2   r3   <module>r     s8  (  ! 1 1     ! . ) / 9 + X X & R R 0 ? 0 0 E 
		H	% 
C[ C C6 
CK C C8 *#Z +- fA fAR8SRYY 8SxJRYY J0$
uxx $
N(:P P2 %II%<<% 
% <<	%
 U\\*% % %0W)ryy W)v64 6r}&@ }@ /*_ /* /*d k
) k
 k
\F 6 FR Rr2   