
    h.                       d dl Z d dlmZ d dlmZ d dlmZmZmZ d dl	Z
d dlZd dlmc mZ d dlZd dlmZ d dlmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZmZmZ ddlmZ ddl m!Z!m"Z" ddl#m$Z$m%Z%m&Z& ddl'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1 ddl2m3Z3 ddl4m5Z5m6Z6 ddl7m8Z8 ddl9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@ ddlAmBZBmCZCmDZD ddlEmFZF ddlGmHZH ddlImJZJmKZKmLZLmMZMmNZN ddlOmPZPmQZQ ddlRmSZS ddlTmUZU ddlVmWZWmXZXmYZY  e?       rd dlZZZ e@j                  e\      Z] G d deU      Z^ G d d eH      Z_ G d! d"e      Z`e< G d# d$e6             Zae e<d%&       G d' d(e3                    Zb G d) d*eP      Zc G d+ d,eQ      Zd G d- d.eY      Ze G d/ d0ej                        Zg G d1 d2ej                        Zh G d3 d4eX      Zi G d5 d6eW      Zj G d7 d8eF      Zk G d9 d:ej                        Zl G d; d<eN      Zm G d= d>eM      Zn G d? d@eK      Zo G dA dBeL      Zp G dC dDej                        Zq G dE dFej                        Zr G dG dHej                        Zs G dI dJej                        Zt G dK dLeJ      Zu G dM dNej                        Zv G dO dPej                        Zw e<dQ&       G dR dSea             Zx G dT dUeae      Zy G dV dWe      Zzg dXZ{y)Y    N)Iterable)	dataclass)CallableOptionalUnion)nn)BlipImageProcessor   )ACT2FN)Cache)PretrainedConfig)%ClassifierFreeGuidanceLogitsProcessorGenerationMixinGenerationModeLogitsProcessorList)GenerateDecoderOnlyOutput)BatchFeatureget_size_dict)convert_to_rgbresizeto_channel_dimension_format)
ChannelDimension
ImageInputPILImageResamplingget_image_sizeinfer_channel_dimension_formatis_scaled_imagemake_flat_list_of_imagesto_numpy_arrayvalid_imagesvalidate_preprocess_arguments)ModelOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)
TensorTypeTransformersKwargsauto_docstringcan_return_tuplefilter_out_non_signature_kwargsis_vision_availablelogging   )CONFIG_MAPPING
AutoConfig	AutoModel)Blip2VisionModel)ChameleonVQVAEConfig)ChameleonVQVAEChameleonVQVAEEncoderAttnBlock#ChameleonVQVAEEncoderConvDownsample ChameleonVQVAEEncoderResnetBlockChameleonVQVAEVectorQuantizer)IdeficsBaseModelOutputWithPastIdeficsCausalLMOutputWithPast)eager_attention_forward)SiglipVisionConfig)SiglipEncoderSiglipEncoderLayerSiglipVisionEmbeddingsc                   P     e Zd ZdZdZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d fd	Z xZS )JanusVisionConfiga
  
    This is the configuration class to store the configuration of a [`JanusVisionModel`]. It is used to instantiate a
    `JanusVisionModel` according to the specified arguments, defining the model architecture.

    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information.
    Args:
        hidden_size (`int`, *optional*, defaults to 1024):
            Dimensionality of the encoder layers and the pooler layer.
        num_hidden_layers (`int`, *optional*, defaults to 24):
            Number of hidden layers in the Transformer encoder.
        num_attention_heads (`int`, *optional*, defaults to 16):
            Number of attention heads for each attention layer in the Transformer encoder.
        num_channels (`int`, *optional*, defaults to 3):
            The number of input channels.
        patch_size (`int`, *optional*, defaults to 16):
            The size (resolution) of each patch.
        image_size (`int`, *optional*, defaults to 384):
            The size (resolution) of each image.
        attention_dropout (`float`, *optional*, defaults to 0.0):
            Dropout probability for attention weights.
        layer_norm_eps (`float`, *optional*, defaults to 1e-06):
            The epsilon used by the layer normalization layers.
        hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
            `"relu"`, `"selu"`, and `"gelu_new"` are supported.
        mlp_ratio (`float`, *optional*, defaults to 4.0):
            Ratio of MLP hidden dimensionality to embedding dimensionality.
        attention_bias (`bool`, *optional*, defaults to `True`):
            Whether to add a bias to the queries, keys, and values in the attention layers.
        hidden_dropout_rate (`float`, *optional*, defaults to 0.0):
            The dropout probability for fully connected layers in the encoder.
        projection_dim (`int`, *optional*, defaults to 2048):
            Dimensionality of the MLP projection head.
        projection_dropout (`float`, *optional*, defaults to 0.0):
            Dropout probability for the projection layer.
        use_qk_norm (`bool`, *optional*, defaults to `False`):
            Whether to normalize the query and key matrices.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated normal initializer for initializing all weight matrices.
        depth (`int`, *optional*, defaults to 2):
            Number of hidden layers in the aligner module.
        num_image_tokens (`int`, *optional*, defaults to 576):
            Number of image tokens.
    janus_vision_modelvision_configc                     t        |   d|||||||||	d	| | `|
| _        || _        || _        || _        || _        || _        || _	        || _
        || _        y )N)	hidden_sizenum_hidden_layersnum_attention_headsnum_channels
patch_size
image_sizeattention_dropoutlayer_norm_eps
hidden_act )super__init__intermediate_size	mlp_ratioattention_biashidden_dropout_rateprojection_dimprojection_dropoutuse_qk_norminitializer_rangedepthnum_image_tokens)selfrD   rE   rF   rG   rH   rI   rJ   rK   rL   rQ   rR   rS   rT   rU   rV   rW   rX   rY   kwargs	__class__s                       f/var/www/html/aiagenthome/venv/lib/python3.12/site-packages/transformers/models/janus/modular_janus.pyrO   zJanusVisionConfig.__init__   s    , 	 	
#/ 3%!!/)!	
 	
 "",#6 ,"4&!2
 0    )i         r
   r`   i          ư>gelug      @Tra      ra   F{Gz?r-   i@  )__name__
__module____qualname____doc__
model_typebase_config_keyrO   __classcell__r\   s   @r]   r@   r@   T   sW    ,\ &J%O ',1 ,1r^   r@   c                   |     e Zd ZdZddddddddg d	d
dddd
ddfdededededededededee   dedef fdZ xZ	S )JanusVQVAEConfiga:
  
    This is the configuration class to store the configuration of a [`JanusVQVAEModel`]. It is used to instantiate a
    `JanusVQVAEModel` according to the specified arguments, defining the model architecture.
    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information. Instantiating a
    configuration with the defaults will yield a similar configuration to the VQModel of the
    [deepseek-community/Janus-Pro-1B](https://huggingface.co/deepseek-community/Janus-Pro-1B).

    Args:
        embed_dim (`int`, *optional*, defaults to 8):
            Dimensionality of each embedding vector.
        num_embeddings (`int`, *optional*, defaults to 16384):
            Number of codebook embeddings.
        double_latent (`bool`, *optional*, defaults to `False`):
            Whether to use double z channels.
        latent_channels (`int`, *optional*, defaults to 256):
            Number of channels for the latent space.
        num_patches (`int`, *optional*, defaults to 32):
            Num of patches the input images can be divided into.
        in_channels (`int`, *optional*, defaults to 3):
            Number of input channels.
        out_channels (`int`, *optional*, defaults to 3):
            Number of out channels.
        base_channels (`int`, *optional*, defaults to 128):
            Base channel count.
        channel_multiplier (`list[int]`, *optional*, defaults to `[1, 1, 2, 2, 4]`):
            Channel multipliers for each resolution.
        num_res_blocks (`int`, *optional*, defaults to 2):
            Number of residual blocks.
        dropout (`float`, *optional*, defaults to 0.0):
            Dropout rate.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
        projection_dim (`int`, *optional*, defaults to 2048):
            Dimensionality of the MLP projection head.
        num_hidden_layers (`int`, *optional*, defaults to 2):
            Number of hidden layers in VAVAE MLP Connecter module.
        hidden_act (`str` or `Callable`, *optional*, defaults to `"gelu"`):
            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
            `"relu"`, `"silu"` and `"gelu_new"` are supported.
        image_token_embed_dim (`int`, *optional*, defaults to 2048):
            Dimension of image embeddings. It should be same as the dimensionality of text embeddings.
       i @  F       r
      )   rt   r-   r-      r-   ra   re   rd   rc   	embed_dimnum_embeddingsdouble_latentlatent_channelsnum_patchesin_channelsout_channelsbase_channelschannel_multipliernum_res_blocksdropoutc                     t        |   d|||||||	|
||d
| || _        || _        || _        || _        || _        || _        | `| `	| `
y )N)
rv   rw   rx   ry   r{   r}   r~   r   r   rW   rM   )rN   rO   rz   r|   rT   rE   rL   image_token_embed_dim
resolutionattn_resolutions	attn_type)rZ   rv   rw   rx   ry   rz   r{   r|   r}   r~   r   r   rW   rT   rE   rL   r   r[   r\   s                     r]   rO   zJanusVQVAEConfig.__init__   s    ( 	 	
)'+#'1)/	
 	
 '(,!2$%:"O!Nr^   )
rf   rg   rh   ri   intboollistfloatrO   rl   rm   s   @r]   ro   ro      s    *\ ##" (7"#** * 	*
 * * * * * !I* * * *r^   ro   c                   <     e Zd ZdZdZeeedZ	 	 	 	 d fd	Z	 xZ
S )JanusConfiga;  
    This is the configuration class to store the configuration of a [`JanusModel`]. It is used to instantiate an
    Janus model according to the specified arguments, defining the model architecture. Instantiating a configuration
    with the defaults will yield a similar configuration to that of the Janus-1B or Janus-7B models.

    e.g. [deepseek-community/Janus-Pro-1B](https://huggingface.co/deepseek-community/Janus-Pro-1B) or
    [deepseek-community/Janus-Pro-7B](https://huggingface.co/deepseek-community/Janus-Pro-7B)

    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information.

    Args:
        text_config (`Union[AutoConfig, dict]`, *optional*, defaults to `LlamaConfig`):
            The config object or dictionary of the text backbone.
        vision_config (`Union[AutoConfig, dict]`,  *optional*, defaults to `JanusVisionConfig`):
            The config object or dictionary of the vision backbone.
        vq_config (`Union[AutoConfig, dict]`,  *optional*, defaults to `JanusVQVAEConfig`):
            The config object or dictionary of the VQVAE backbone.
        image_token_id (`int`, *optional*, defaults to 100581):
            Token index of a placeholder image token.

    Example:

    ```python
    >>> from transformers import JanusForConditionalGeneration, JanusConfig, JanusVisionConfig, JanusVQVAEConfig, LlamaConfig

    >>> # Initializing a Janus vision config
    >>> vision_config = JanusVisionConfig()

    >>> # Initializing a Llama config
    >>> text_config = LlamaConfig()

    >>> # Initializing a VQ config
    >>> vq_config = JanusVQVAEConfig()

    >>> # Initializing a Janus Pro 1B style configuration
    >>> configuration = JanusConfig(vision_config=vision_config, text_config=text_config, vq_config=vq_config)

    >>> # Initializing a model from the Janus Pro 1B style configuration
    >>> model = JanusForConditionalGeneration(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```janus)text_configrB   	vq_configc                    t        |t              r,|j                  dd      |d<   t        |d      d	i || _        nY|(t
        j                  d       t        d          | _        n/t        |t              r|| _        nt        dt        |             |%t
        j                  d       t               | _        nPt        |t              rt        d	i || _        n/t        |t              r|| _        nt        dt        |             |%t
        j                  d       t               | _        nPt        |t              rt        d	i || _        n/t        |t              r|| _        nt        dt        |             | j                  j                  | _        | j                  j                  | j                  j                   z  | j                  _        || _        t'        | P  d	i | y )
Nrj   llamaz7`text_config` is None. Initializing with default valueszTInvalid type for `text_config`. Must be either `dict` or `LlamaConfig`. Type found: zK`vision_config` is None. Initializing with default JanusVisionConfig valuesz\Invalid type for `vision_config`. Must be either `dict` or `JanusVisionConfig`. Type found: zF`vq_config` is None. Initializing with default JanusVQVAEConfig valueszWInvalid type for `vq_config`. Must be either `dict` or `JanusVQVAEConfig`. Type found: rM   )
isinstancedictgetr.   r   loggerinfor   
ValueErrortyper@   rB   ro   r   rW   rI   rH   rz   image_token_idrN   rO   )rZ   r   rB   r   r   r[   r\   s         r]   rO   zJanusConfig.__init__D  s    k4((3g(NK%-k,.GHW;WD KKQR-g68D%56*D  $[ 124 
  KKef!2!4Dt,!2!C]!CD'89!.D  $] 346 
 KK`a-/DN	4(-:	:DN	#34&DN  $Y02 
 "&!3!3!E!E%)%7%7%B%BdFXFXFcFc%c","6"r^   )NNNi )rf   rg   rh   ri   rj   r/   r@   ro   sub_configsrO   rl   rm   s   @r]   r   r     s8    +Z J!*%K 6# 6#r^   r   c                   @    e Zd ZU eed<   dZdZddgZddgZdZ	dZ
dZdZy	)
JanusPreTrainedModelconfigmodelTLlamaDecoderLayerJanusVisionEncoderLayerpast_key_valuescausal_maskFN)rf   rg   rh   r   __annotations__base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa_can_compile_fullgraph!_supports_param_buffer_assignmentrM   r^   r]   r   r   }  sB    &*#,.GH#4m"DN!(-%r^   r   z9
    Base class for Janus VQ-VAE mode model outputs.
    )custom_introc                   b    e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   y)JanusVQVAEOutputz
    decoded_pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
        Reconstructed pixel values after encoding and decoding the input.
    embedding_loss (`torch.FloatTensor`):
        Embedding loss.
    Ndecoded_pixel_valuesembedding_loss)
rf   rg   rh   ri   r   r   torchFloatTensorr   r   rM   r^   r]   r   r     s4     9=(5#4#45<26NHU../6r^   r   c                       e Zd Zy)JanusBaseModelOutputWithPastNrf   rg   rh   rM   r^   r]   r   r         r^   r   c                       e Zd Zy)JanusCausalLMOutputWithPastNr   rM   r^   r]   r   r     r   r^   r   c                   J    e Zd Zddej                  dedej                  fdZy)JanusVisionEmbeddingspixel_valuesinterpolate_pos_encodingreturnc                 X   |j                   \  }}}}| j                  j                  j                  }| j                  |j	                  |            }|j                  d      j                  dd      }|r| j                  |||      }	n| j                  | j                        }	||	z   }|S )Ndtyper-   rt   )
shapepatch_embeddingweightr   toflatten	transposer   position_embeddingposition_ids)
rZ   r   r   _heightwidthtarget_dtypepatch_embeds
embeddings
pos_embedss
             r]   forwardzJanusVisionEmbeddings.forward  s    *001fe++2288++LOO,O,OP!))!,66q!<
#66z65QJ001B1BCJ*,
r^   N)F)rf   rg   rh   r   Tensorr   r   rM   r^   r]   r   r     s'    ELL D ]b]i]i r^   r   c                   t     e Zd ZdZdef fdZ	 ddej                  deej                     de	e
   fdZ xZS )	JanusVisionAttentionz(Attention Class for Janus Vision Encoderr   c                 F   t         |           || _        |j                  | _        |j
                  | _        | j                  | j                  z  | _        | j                  | j                  z  | j                  k7  r&t        d| j                   d| j                   d      | j                  dz  | _	        |j                  | _
        |j                  }|j                  }d| _        d| _        t        j                   | j                  | j                  | j                  z  |j"                        | _        t        j                   | j                  | j                  | j                  z  |j"                        | _        t        j                   | j                  | j                  | j                  z  |j"                        | _        t        j                   | j                  | j                        | _        |dkD  rt        j,                  |      nt        j.                         | _        |rt        j0                  | j                        nt        j.                         | _        |r%t        j0                  | j                        | _        y t        j.                         | _        y )	Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).g      Frt   biasr   )rN   rO   r   rD   rv   rF   	num_headshead_dimr   scalerJ   rU   rV   	is_causalnum_key_value_groupsr   LinearrR   q_projk_projv_projprojection_layerDropoutIdentity	LayerNormq_normk_norm)rZ   r   proj_dropoutqk_normr\   s       r]   rO   zJanusVisionAttention.__init__  s   ++33$..8==4>>)T^^;MdnnM] ^NN#2'  ]]D(
!'!9!900$$ %&!ii0NU[UjUjkii0NU[UjUjkii0NU[UjUjk "		$..$.. I>JQ>N"**\":TVT_T_Ta6=bll4>>22;;=6=bll4>>22;;=r^   hidden_statesattention_maskr[   c                 >   |j                         \  }}}| j                  |      }| j                  |      }| j                  |      }	|j	                  d| j
                  | j                        }| j                  |      }|j	                  d| j
                  | j                        }| j                  |      }|j	                  ||| j
                  | j                        j                  dd      }|j	                  ||| j
                  | j                        j                  dd      }|	j                  ||| j
                  | j                        j                  dd      }	t        }
| j                  j                  dk7  rt        | j                  j                     }
 |
| |||	|f| j                  sdn| j                   | j"                  | j$                  d|\  }}|j	                  ||| j&                        }| j)                  |      }| j+                  |      }||fS )Nrt   r-   eagerra   )r   scalingr   )sizer   r   r   reshaper   r   r   r   r   viewr:   r   _attn_implementationr#   trainingrJ   r   r   rv   r   rU   )rZ   r   r   r[   
batch_sizeseq_lenr   query_states
key_statesvalue_statesattention_interfaceattn_outputattn_weightsoutputs                 r]   r   zJanusVisionAttention.forward  s    "/!3!3!5
GQ{{=1[[/
{{=1#++BN{{<0''DNNDMMJ
[[,
#++JQUQ^Q^_iijkmno''
GT^^T]][eefgijk
#((Wdnndmm\ffghjkl(?;;++w6"9$++:Z:Z"[$7
%
  $}}C$2H2HJJnn
%
 
%
!\ "))*gt~~N&&{3((0|##r^   N)rf   rg   rh   ri   r@   rO   r   r   r   r%   r'   r   rl   rm   s   @r]   r   r     sO    2Q0 Q@ 26)$||)$ !.)$ +,	)$r^   r   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )JanusVisionMLPr   c                    t         |           || _        t        |j                  |j
                  z        | _        t        |j                     | _	        t        j                  |j                  | j                        | _        t        j                  | j                  |j                        | _        t        j                  |j                        | _        t        j                  |j                        | _        y r   )rN   rO   r   r   rD   rQ   rP   r   rL   activation_fnr   r   fc1fc2r   rS   dropout1dropout2rZ   r   r\   s     r]   rO   zJanusVisionMLP.__init__  s    !$V%7%7&:J:J%J!K#F$5$5699V//1G1GH99T33V5G5GH

6#=#=>

6#=#=>r^   r   r   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }|S r   )r   r   r  r  r  rZ   r   s     r]   r   zJanusVisionMLP.forward  sP    /**=9m4/m4r^   )	rf   rg   rh   r@   rO   r   r   r   rl   rm   s   @r]   r   r     s+    ?0 ?U\\ ell r^   r   c                   $     e Zd Zdef fdZ xZS )r   r   c                 T   t         |   |       || _        |j                  | _        t        |      | _        t        j                  | j                  |j                        | _
        t        j                  | j                  |j                        | _        t        |      | _        y )N)eps)rN   rO   r   rD   rv   r   	self_attnr   r   rK   layer_norm1layer_norm2r   mlpr  s     r]   rO   z JanusVisionEncoderLayer.__init__  sv     ++-f5<<F<Q<QR<<F<Q<QR!&)r^   rf   rg   rh   r@   rO   rl   rm   s   @r]   r   r     s    *0 * *r^   r   c                   $     e Zd Zdef fdZ xZS )JanusVisionEncoderr   c                     t         |   |       t        j                  t	        |j
                        D cg c]  }t        |       c}      | _        y c c}w r   )rN   rO   r   
ModuleListrangerE   r   layersrZ   r   r   r\   s      r]   rO   zJanusVisionEncoder.__init__"  sF     mmeTZTlTlNm$nNm%<V%DNm$no$ns   Ar  rm   s   @r]   r  r  !  s    p0 p pr^   r  c                   $     e Zd Zdef fdZ xZS )JanusVisionModelr   c                 D    t         |   |       t        |      | _        y r   )rN   rO   r  encoderr  s     r]   rO   zJanusVisionModel.__init__(  s     )&1r^   r  rm   s   @r]   r  r  '  s    20 2 2r^   r  c                   *     e Zd Zdef fdZd Z xZS )JanusVisionAlignerMLPr   c           	         t         |           t        j                  |j                  |j
                        | _        t        j                  t        d|j                        D cg c],  }t        j                  |j
                  |j
                        . c}      | _
        t        |j                     | _        y c c}w Nrt   )rN   rO   r   r   rD   rT   r   r  r  rX   hidden_layersr   rL   r   r  s      r]   rO   zJanusVisionAlignerMLP.__init__.  s    99V//1F1FG]]NSTUW]WcWcNdeNdRYYv,,f.C.CDNde
 $F$5$56 f   &1B<c                 |    | j                  |      }| j                  D ]  }| j                  |      } ||      } |S r   r   r  r   rZ   r   layers      r]   r   zJanusVisionAlignerMLP.forward7  B    /''E ..}=M!-0M ( r^   )rf   rg   rh   r@   rO   r   rl   rm   s   @r]   r  r  -  s    70 7r^   r  c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZ	S )JanusVQVAEVectorQuantizerr   c                 N    t         |   |       |j                  gdz  | _        y )Nr-   )rN   rO   rz   quant_state_dimsr  s     r]   rO   z"JanusVQVAEVectorQuantizer.__init__@  s&     !'!3!3 4q 8r^   image_tokensr   c                 B   |j                   d   }| j                  j                  j                   d   }| j                  |      }t        j                  |dd      }|j                  |g| j                  |      }|j                  dddd      j                         }|S )Nr   r   r-   )pdimr
   rt   )	r   	embeddingr   F	normalizer   r(  permute
contiguous)rZ   r)  r   emb_dimhidden_state_quants        r]   get_codebook_entryz,JanusVQVAEVectorQuantizer.get_codebook_entryD  s    !''*
~~,,2226 "^^L9[[);qbI 044j5b4CXCX5bZa5bc/771aCNNP!!r^   )
rf   rg   rh   ro   rO   r   
LongTensorr   r4  rl   rm   s   @r]   r&  r&  ?  s/    9/ 9"u/?/? "EDUDU "r^   r&  c                       e Zd Zy)JanusVQVAEResnetBlockNr   rM   r^   r]   r7  r7  T  r   r^   r7  c                       e Zd Zy)JanusVQVAEAttnBlockNr   rM   r^   r]   r9  r9  X  r   r^   r9  c                       e Zd Zy)JanusVQVAEConvDownsampleNr   rM   r^   r]   r;  r;  \  r   r^   r;  c                   $     e Zd Z fdZd Z xZS )JanusVQVAEConvUpsamplec                 t    t         |           t        j                  j	                  ||ddd      | _        y )Nr
   rt   kernel_sizestridepadding)rN   rO   r   r   Conv2dconv)rZ   r{   r\   s     r]   rO   zJanusVQVAEConvUpsample.__init__a  s.    HHOOK!TU_`Oa	r^   c                 X    t        j                  |dd      }| j                  |      }|S )Ng       @nearest)scale_factormode)r.  interpolaterD  r  s     r]   r   zJanusVQVAEConvUpsample.forwarde  s(    m#IV		-0r^   )rf   rg   rh   rO   r   rl   rm   s   @r]   r=  r=  `  s    br^   r=  c                   `     e Zd Zdedef fdZdej                  dej                  fdZ xZ	S )JanusVQVAEMidBlockr   channelsc                     t         |           t        |||      | _        t	        |      | _        t        |||      | _        y )Nr   r{   r|   )rN   rO   r7  block_1r9  attn_1block_2)rZ   r   rL  r\   s      r]   rO   zJanusVQVAEMidBlock.__init__l  sF    , !

 *(3, !
r^   r   r   c                 l    | j                  |      }| j                  |      }| j                  |      }|S r   )rO  rP  rQ  r  s     r]   r   zJanusVQVAEMidBlock.forwardz  s2    ]3M2]3r^   )
rf   rg   rh   ro   r   rO   r   r   r   rl   rm   s   @r]   rK  rK  k  s2    
/ 
3 
U\\ ell r^   rK  c                   >     e Zd Z fdZdej
                  fdZ xZS )JanusVQVAEEncoderc           	         t         |           t        |j                        | _        |j
                  | _        |j                  }|j                  }|j                  }|j                  }|j                  }t        j                  j                  ||ddd      | _        dt        |      z   }|| _        t        j                          | _        t%        | j                        D ]   }t        j                          }	t        j                          }
|||   z  }|||   z  }t%        | j
                        D ]N  }|	j'                  t)        |||             |}|| j                  dz
  k(  s5|
j'                  t+        |             P t        j,                         }|	|_        |
|_        || j                  dz
  k7  rt3        |      |_        | j"                  j'                  |        t7        |      | _        t        j                  j;                  d|dd	      | _        t        j                  j                  ||rd
|z  n|ddd      | _        y )Nr
   rt   r?  )rt   rN  rr   rb   T
num_groupsrG   r	  affiner-   ) rN   rO   lenr~   num_resolutionsr   r}   r{   rx   ry   r   r   rC  conv_intuplein_channel_multiplierr  downr  appendr7  r9  Moduleblockattnr;  
downsamplerK  mid	GroupNormnorm_outconv_out)rZ   r   r}   r{   rx   ry   r~   r]  i_levelra  rb  block_in	block_outi_blockr^  r\   s                  r]   rO   zJanusVQVAEEncoder.__init__  s   "6#<#<=$33,,((,, 00#66xx{MqYZdef $u-?'@ @%:"MMO	T112GMMOE==?D$'<W'EEH%(:7(CCI !4!45)%$,%. %d22Q66KK 3H => 6 99;DDJDI$..22":8"DIIT"- 30 &fh7**bxUYbf*g#0Ao ( 
r^   r   c                    | j                  |      g}t        | j                        D ]  }t        | j                        D ]  } | j                  |   j
                  |   |d         }t        | j                  |   j                        dkD  r" | j                  |   j                  |   |      }|j                  |        || j                  dz
  k7  s|j                  | j                  |   j                  |d                 |d   }| j                  |      }| j                  |      }|t        j                  |      z  }| j                  |      }|S )Nr   r   rt   )r[  r  rZ  r   r^  ra  rY  rb  r_  rc  rd  rf  r   sigmoidrg  )rZ   r   r   rh  rk  hidden_statelast_hidden_states          r]   r   zJanusVQVAEEncoder.forward  sH   l34T112G !4!45@tyy177@!"%  tyy)../!3#C499W#5#:#:7#CL#QL$$\2 6 $..22$$TYYw%7%B%B=QSCT%UV 3 *"- HH%67 !MM*;<U]]+<== MM*;<  r^   )rf   rg   rh   rO   r   r5  r   rl   rm   s   @r]   rT  rT    s    1
f!E$4$4 !r^   rT  c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )JanusVQVAEDecoderc           	      v   t         |           t        |j                        | _        |j
                  | _        |j                  }|j                  }|j                  }||j                  | j                  dz
     z  }t        j                  j                  ||ddd      | _        t        ||      | _        t        j                         | _        t#        t%        | j                              D ]  }t        j                         }t        j                         }||j                  |   z  }	t%        | j
                  dz         D ]N  }
|j'                  t)        |||	             |	}|| j                  dz
  k(  s5|j'                  t+        |             P t        j,                         }||_        ||_        |dk7  rt3        |      |_        | j                   j'                  |        t        j                  j7                  d|dd	      | _        t        j                  j                  ||ddd      | _        y )
Nrt   r
   r?  rN  r   rr   rb   TrV  )rN   rO   rY  r~   rZ  r   r}   ry   r|   r   r   rC  r[  rK  rd  r  upreversedr  r_  r7  r9  r`  ra  rb  r=  upsamplere  rf  rg  )rZ   r   r}   ry   r|   ri  rh  ra  rb  rj  rk  rs  r\   s               r]   rO   zJanusVQVAEDecoder.__init__  s   "6#<#<=$33,, 00** !6#<#<T=Q=QTU=U#VV xxaXYcde &fh7 --/d&:&: ;<GMMOE==?D%(A(A'(JJI !4!4q!89)%$,%. %d22Q66KK 3H => : BBHBG!|4X>GGNN2) =. **bxUYbf*g,AVWabcr^   rn  r   c                 b   | j                  |      }| j                  |      }t        | j                        D ]  }t        | j                  dz         D ]l  } | j
                  |   j                  |   |      }t        | j
                  |   j                        dkD  sK | j
                  |   j                  |   |      }n || j                  dz
  k7  s| j
                  |   j                  |      } | j                  |      }|t        j                  |      z  }| j                  |      }|S )Nrt   r   )r[  rd  r  rZ  r   rs  ra  rY  rb  ru  rf  r   rm  rg  )rZ   rn  rh  rk  s       r]   r   zJanusVQVAEDecoder.forward  s   ||L1 xx- T112G !4!4q!89>twww/55g>|Ltwww',,-1#A4777#3#8#8#A,#OL : $..22#www/88F 3 }}\2l33}}\2r^   )rf   rg   rh   rO   r   r   r   rl   rm   s   @r]   rq  rq    s)    ,d\E$5$5 %:K:K r^   rq  c                        e Zd Zg dZdZdef fdZdej                  dej                  fdZ
eedej                  deej                  ej                  f   fd              Z xZS )	
JanusVQVAE)r9  r7  r&  r   r   c                 r    t         |   |       t        |      | _        d| _        | j                          y )NF)rN   rO   rq  decodergradient_checkpointing	post_initr  s     r]   rO   zJanusVQVAE.__init__  s0     (0&+# 	r^   r)  r   c                    |j                   d   | j                  j                  d   | j                  j                  d   z  k7  rMt        d| j                  j                  d   | j                  j                  d   z   d|j                    d      | j                  j	                  |      }| j                  |      }| j                  |      }|S )aG  
        Decodes quantized token IDs into pixel values.
        Args:
            image_tokens (torch.LongTensor): Batch of token IDs.
        Returns:
            pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
                Pixel values decoded from the token IDs.
        rt   r   z4Expected `image_tokens` to have shape `(batch_size, z)`, but got shape `z`.)r   quantizer(  r   r4  post_quant_convrz  )rZ   r)  codebook_entryr   r   s        r]   decodezJanusVQVAE.decode"  s     a DMM$B$B1$EHfHfghHi$iiFt}}GeGefgGhkokxkx  lJ  lJ  KL  lM  HM  GN N""."4"4!5R9  99,G,,^<||M2r^   c                     |j                   d   }| j                  |      \  }}}| j                  |j                  |d            }t	        ||      S )Nr   r   )r   encoder  r   r   )rZ   r   r   quantr   indicesr   s          r]   r   zJanusVQVAE.forward5  sQ     "''*
)-\)B&~w#{{7<<
B+GH 4nEEr^   )rf   rg   rh   r   main_input_namero   rO   r   r5  r   r  r)   r(   r\  r   rl   rm   s   @r]   rx  rx    s    
 %O/ 5#3#3 8I8I & F''F 
u  %"3"33	4F  Fr^   rx  c                   *     e Zd Zdef fdZd Z xZS )JanusVQVAEAlignerMLPr   c           	         t         |           t        j                  |j                  |j
                        | _        t        j                  t        d|j                        D cg c],  }t        j                  |j
                  |j
                        . c}      | _
        t        |j                     | _        y c c}w r  )rN   rO   r   r   rv   rT   r   r  r  rE   r  r   rL   r   r  s      r]   rO   zJanusVQVAEAlignerMLP.__init__C  s    99V--v/D/DE]]NSTUW]WoWoNpqNpRYYv,,f.C.CDNpq
 $F$5$56 rr  c                 |    | j                  |      }| j                  D ]  }| j                  |      } ||      } |S r   r!  r"  s      r]   r   zJanusVQVAEAlignerMLP.forwardL  r$  r^   )rf   rg   rh   ro   rO   r   rl   rm   s   @r]   r  r  B  s    7/ 7r^   r  c                   `     e Zd ZdZdef fdZdej                  dej                  fdZ	 xZ
S )JanusVQVAEHeadzOHead used for sampling tokens in image generation, replacing the usual lm head.r   c                    t         |           t        j                  |j                  |j
                        | _        t        |j                     | _	        t        j                  |j
                  |j                        | _        y r   )rN   rO   r   r   r   rT   proj_outr   rL   r   rw   vision_headr  s     r]   rO   zJanusVQVAEHead.__init__W  s^    		&">">@U@UV#F$5$5699V%:%:F<Q<QRr^   r   r   c                 l    | j                  |      }| j                  |      }| j                  |      }|S r   )r  r   r  r  s     r]   r   zJanusVQVAEHead.forward]  s6    m4**=9((7r^   )rf   rg   rh   ri   ro   rO   r   r   tensorr   rl   rm   s   @r]   r  r  T  s0    YS/ SU\\ ell r^   r  zl
    The Janus model which consists of a siglip vision backbone, a Llama language model and a VQ model.
    c                       e Zd Zdef fdZd Zd Zd Zdej                  dej                  dej                  fd	Zee	 	 	 	 	 	 	 	 	 ddeej                     d
eej                     deej                     deej                     dee   deej                     deej                     dee   deeej                  f   fd              Z xZS )
JanusModelr   c                    t         |   |       || _        t        j	                  |j
                        | _        t        | j                  j                        | _        t        j	                  |j                        | _        t        j                  | j                  j                  j                  | j                  j                  j                        | _        t#        | j                  j                        | _        t'        | j                  j                        | _        t+        j,                  |j.                        | _        d| _        | j5                          y )N)r   F)rN   rO   r   r  _from_configrB   vision_modelr  alignerrx  r   vqmodelr   	Embeddingrw   rv   generation_embeddingsr  generation_alignerr  generation_headr0   from_configr   language_modelr{  r|  r  s     r]   rO   zJanusModel.__init__j  s     ,99&:N:NO,T->->-E-EF!..v/?/?@ &(\\$,,2E2E2T2TVZVbVbViViVsVs%t""6t||7J7J"K-dll.A.AB'336;M;MN&+#r^   c                 6    | j                   j                         S r   )r  get_input_embeddingsrZ   s    r]   r  zJanusModel.get_input_embeddings  s    ""7799r^   c                 :    | j                   j                  |       y r   )r  set_input_embeddingsrZ   values     r]   r  zJanusModel.set_input_embeddings  s    007r^   c                 ^    | j                  |      }| j                  |j                        }|S r   )r  r  ro  )rZ   r   image_embedss      r]   get_image_featureszJanusModel.get_image_features  s,    ((6||L$B$BCr^   	input_idsinputs_embedsimage_featuresc                 P   |m| | j                         t        j                  | j                  j                  t        j
                  |j                              k(  }|j                  d      }n|| j                  j                  k(  }|j                         }|j                  d      j                  |      j                  |j                        }||   j                         |j                         k7  r0|j                  d   |j                  d   z  }t        d| d|       |S )z
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        r   devicer   r   rt   z6Image features and image tokens do not match: tokens: z, features )r  r   r  r   r   longr  allsum	unsqueeze	expand_asr   numelr   r   )rZ   r  r  r  special_image_maskn_image_tokensn_image_featuress          r]   get_placeholder_maskzJanusModel.get_placeholder_mask  s    !.2M$2K2K2MT[[77uzzR_RfRfg3 " "4!7!7!;!*dkk.H.H!H+//1/99"=GGVYYZgZnZno+,2248L8L8NN-33A69M9Ma9PPHHXXcdtcuv  "!r^   r   r   r   r   cache_position	use_cachelogits_to_keepc
                    |d u |d uz  rt        d      | | j                         |      }||| j                  |      }|j                  d|j                  d         }|j                  |j                  |j                        }| j                  |||      }|j                  ||      } | j                  d|||||||	d|
}t        |j                  |j                  |j                  |j                  |      S d       S )NzaYou cannot specify both input_ids and inputs_embeds at the same time, and must specify either oner   )r  r  )r  r   r   r   r  r  r  )ro  r   r   
attentionsimage_hidden_statesrM   )r   r  r  r   r   r   r  r   r  masked_scatterr  r   ro  r   r   r  )rZ   r  r   r   r   r   r  r  r  r  r[   r  r  image_attention_mask	lm_outputs                  r]   r   zJanusModel.forward  sH    -t";<s   7D557	BM#22<@L)11"m6I6I"6MNN+..}/C/C]EXEXYN#'#<#<~ $= $  *889M~^M'D'' 	
')%+))	
 	
	 ,'99%55#11 ++0<0H
 	

 OS
 	
r^   )	NNNNNNNNr   )rf   rg   rh   r   rO   r  r  r  r   r5  r   r  r)   r(   r   r   r   r   r   r   r   rl   rm   s   @r]   r  r  d  s4   { *:8
"))":?:K:K"]b]n]n"0  15481537+/5959$(34.
E,,-.
 u001.
 !.	.

 u//0.
 "%.
 !!1!12.
   1 12.
 D>.
 c5<</0.
  .
r^   r  c                   h    e Zd ZddgZdZdef fdZd Zd Zde	j                  d	e	j                  fd
Zee	 	 	 	 	 	 	 	 	 	 ddee	j                     dee	j                      dee	j                     dee	j                     dee   dee	j                     dee	j                      dee	j                     dee   deee	j                  f   dee   fd              Z	 	 	 	 	 	 d fd	Zde	j                  fdZe	j4                  	 	 	 ddee	j                     dee	j                     dee   f fd       Z xZS )JanusForConditionalGenerationz(model.language_model.embed_tokens.weightzlm_head.weightTr   c                     t         |   |       || _        t        |      | _        t        j                  |j                  j                  |j                  j                  d      | _
        | j                          y )NFr   )rN   rO   r   r  r   r   r   r   rD   
vocab_sizelm_headr|  r  s     r]   rO   z&JanusForConditionalGeneration.__init__  s\     '
yy!3!3!?!?ASASA^A^ejk 	r^   c                 J    | j                   j                  j                         S r   )r   r  r  r  s    r]   r  z2JanusForConditionalGeneration.get_input_embeddings  s    zz((==??r^   c                 N    | j                   j                  j                  |       y r   )r   r  r  r  s     r]   r  z2JanusForConditionalGeneration.set_input_embeddings  s    

!!66u=r^   inputsr   c                 r    | j                   j                  |      }| j                   j                  |      }|S r   )r   r  r  )rZ   r  rn  s      r]   'prepare_embeddings_for_image_generationzEJanusForConditionalGeneration.prepare_embeddings_for_image_generation  s0    zz77?zz44\Br^   r  r   r   r   r   r  r  labelsr  r  r[   c                     | j                   d|||||||	|d|}|j                  }t        |
t              rt	        |
 d      n|
}| j                  |dd|ddf         }d}|4 | j                  d||| j                  j                  j                  d|}t        |||j                  |j                  |j                  |j                        S )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        )r  r   r   r   r   r  r  r  N)logitsr  r  )lossr  r   r   r  r  rM   )r   ro  r   r   slicer  loss_functionr   r   r  r   r   r   r  r  )rZ   r  r   r   r   r   r  r  r  r  r  r[   outputsr   slice_indicesr  r  s                    r]   r   z%JanusForConditionalGeneration.forward  s    , $** 

%)%+')

 

  118B>SV8W~ot4]kmA}a,?@A%4%% f9P9P9[9[_eD +#33!//)) ' ; ;
 	
r^   c           	      N    t        
|   |f|||||d|}	|d   dk(  r||	d<   |	S )N)r   r  r   r  r  r   r   )rN   prepare_inputs_for_generation)rZ   r  r   r   r   r  r  r  r[   model_inputsr\   s             r]   r  z;JanusForConditionalGeneration.prepare_inputs_for_generation"  sT     w<
+')))
 
 !!+7L(r^   r)  c                 x    | j                   j                  j                  |      }|j                  dddd      }|S )a,  
        Decodes generated image tokens from language model to continuous pixel values
        with VQGAN module via upsampling.
        Args:
            image_tokens (`torch.LongTensor` of shape `(batch_size, num_of_tokens)`):
                The tensors corresponding to the input images.
        r   r-   r
   rt   )r   r  r  r0  )rZ   r)  decoded_images      r]   decode_image_tokensz1JanusForConditionalGeneration.decode_image_tokens@  s:     

**11,?%--aAq9r^   logits_processorc           	         |j                  d| j                        }t        j                  |      }|j                  dd      }|dk(  rt	        %|   d|||d d|S  |j                  di |}|j                         t        j                  t        j                  fvrt        d      |j                          | j                  |j                                ||n	t               }d|d<   |j                  t         j#                  d       d	|_        |j                  |d
<   | j%                  ||j&                  |      \  }}	}|j(                  |j*                  }}
t-        |j.                        dk7  rt        d|j.                   d      |d u}| j1                  |||j*                         |j                  r:|j                  dkD  r+|j3                  t5        |j                               d |_        | j7                  ||j.                  d   |d ||      } | j8                  d|||j:                  d|\  }}| j<                  j>                  j@                  jB                  }|j.                  \  }}|jE                  dd      }|j                  dd       }|jE                  dd      }||d<   ||d d d f   |j&                  k7  ||d d d f   |jF                  d   k7  z  }||d d d f   jI                  ||jJ                          | jM                         |      }| jO                  |||      }|jQ                  dd       @| jS                  |jT                  xs d|dz  tW        |jX                  ||z         |      |d<   t[        j\                  ||f|
|      }|j^                  }|j`                  }|jb                  }|jd                  }|jf                  }|r|rdnd }|r|rdnd }|r|rdnd }|r|rdnd }ti        |      D ]x  } | jj                  d||d|}|d   jm                  |j*                        |d<   |d   jm                  |j*                        |d<    | j<                  jn                  di |||d}| jq                  ||      }|jr                  d d dd d f   ju                         } | j<                  jw                  |       }! |||!      }"|jx                  r>t[        jz                  |"d      }#t[        j|                  |#d      j                  d      }$nt[        j                  |"d      }$|$|d d |f<   t[        j                  |$|$g      }$|$j                  d      }$| j                  |$      }{ |r@|r|!fz  }|r| j                         fz  }|r|j                  z  }|r|j                  z  }|rt        |!|||j                        S |S ) Ngeneration_configgeneration_modetext)r  r   r  guidance_scalezGot incompatible mode for Image Generation, should be one of greedy or sampling. Ensure that beam search is de-activated by setting `num_beams=1`.Tr  zU`guidance_scale` is required for CFG but not provided. Setting to default value of 5.   r  r-   z;Expected input ids of shape (batch_size, seq_len), but got z3Passing `inputs embeds` is not supported currently.)r  rt   )r  input_ids_seq_lengthencoder_input_idsprefix_allowed_tokens_fnr  r  )r  r   expand_sizer   boi_token_idr   static)cache_implementationr   max_cache_lenmodel_kwargsr  rM   )r  r  r  )output_attentionsoutput_hidden_statesr   )r,  )num_samples)	sequencesscoresr  r  r   r   )Ipopr  copydeepcopyrN   generateupdateget_generation_moder   SAMPLEGREEDY_SEARCHr   validate_validate_model_kwargsr   r  r   warning_prepare_model_inputsbos_token_idr   r  rY  r   _prepare_special_tokensr_  r   _get_logits_processor_expand_inputs_for_generationnum_return_sequencesr   r  r   rY   repeatgeneration_kwargsmasked_fill_pad_token_idr  _get_initial_cache_positionr   
_get_cacher  max
max_lengthr   zerosr  r  output_scoresoutput_logitsreturn_dict_in_generater  r  r   r  #_update_model_kwargs_for_generationro  cloner  	do_samplesoftmaxmultinomialsqueezeargmaxcatr  r  r   r  r   r   r   )&rZ   r  r   r  r[   r  r  r  r  model_input_namer   r  kwargs_has_attention_maskrY   r   r   input_tokensmaskr  generated_tokensr  r  r	  r
  r  
raw_scores
raw_logitsdecoder_hidden_statesdecoder_attentionsir  r  rn  r  next_token_scoresprobs
next_tokenr\   s&                                        r]   r  z&JanusForConditionalGeneration.generateL  su    #JJ':D<R<RS MM*;< !**%6?f$7# -"3#	
   0(//9&9 002>;P;PR`RnRn:ooT  	""$##L$5$5$78 0@/K+QdQf %)[!++3NNrs/0,):)I)I%& 594N4N%22L5
1	#\ ")9)9vy1$MiooM^EF  %3$$>!$$%68QZcZjZj$k ++0A0P0PST0T##$IJ[JjJj$kl/3,  55/!*!3'%)- 6 
 #E$"D"D #
))>>#
 	#
	<  ::2299JJ'oo
G ''1-%))*:DA'..q!4)7%& Z[!^,0A0N0NNa(,=,O,OP^,__
 	Z[!^$11$8I8V8VW3113LA77V-t4<.2oo%6%K%K%Wx%>!"3">">@PSZ@Z[) /> /L*+ !;;
4D'EU[ab .??0EE)77)77"3"K"K3RD
3RD
'>CW^b$;@QRX\'(A=4== +|GSL .::J-K-N-N}OcOc-dL)*-9:J-K-N-N}OcOc-dL)*/djj// "3%9G  CCG\ZL"44QAX>DDFL ZZ//=F 0F C !**&7R@"..u!DLLRP
"\\*;D
%/QT" J
#;<J#--b1J HHTMG )J #vi'
|11355
 "g&8&88"#%)>)>>%",*!-3 ' 7 7  $#r^   )
NNNNNNNNNr   )NNNNNN)NNN)rf   rg   rh   _tied_weights_keysr   r   rO   r  r  r   r   r  r)   r(   r   r5  r   r   r   r   r   r%   r'   r   r  r  no_gradr   r  rl   rm   s   @r]   r  r    s   DFVW!{ @>ell u|| 
  15481537+/5959-1$(341
E,,-1
 u0011
 !.	1

 u//01
 "%1
 !!1!121
   1 121
 ))*1
 D>1
 c5<</01
 +,1
  1
l <
 
 ]] *.59:>	|$&|$ !!1!12|$ ##67	|$ |$r^   r  c            #       &    e Zd ZdZdddej
                  dddddddfdedeee	e
f      de
d	ed
edee
ef   dedeeeee   f      deeeee   f      dee   dee   f fdZ	 	 	 ddej                   dee
ee
e
e
f   f   deee	ef      deee	ef      dej                   f
dZej
                  ddfdej                   deee	e
f   e
f   d	edeee	ef      deee	ef      dej                   fdZ e       ddddddddddddej,                  dfdedee   deee	e
f      d	ee   d
ee   dee   dee   deeeee   f      deeeee   f      deee	ef      dee   deee
ee
e
e
f   f      dee   dedeee	ef      dej4                  j4                  f d       Z	 	 	 	 	 	 	 dded
ee   dee   dee   deee      deee      dee	   dee	   fdZ	 d dej                   deeee   f   deeee   f   deee	ef      dej                   f
dZ xZS )!JanusImageProcessora  
    Constructs a JANUS image processor.

    Args:
        do_resize (`bool`, *optional*, defaults to `True`):
            Whether to resize the image's (height, width) dimensions to the specified `size`. Can be overridden by the
            `do_resize` parameter in the `preprocess` method.
        size (`dict`, *optional*, defaults to `{"height": 384, "width": 384}`):
            Size of the output image after resizing. Can be overridden by the `size` parameter in the `preprocess`
            method.
        min_size (`int`, *optional*, defaults to 14):
            The minimum allowed size for the resized image. Ensures that neither the height nor width
            falls below this value after resizing.
        resample (`PILImageResampling`, *optional*, defaults to `Resampling.BICUBIC`):
            Resampling filter to use if resizing the image. Only has an effect if `do_resize` is set to `True`. Can be
            overridden by the `resample` parameter in the `preprocess` method.
        do_rescale (`bool`, *optional*, defaults to `True`):
            Whether to rescale the image by the specified scale `rescale_factor`. Can be overridden by the
            `do_rescale` parameter in the `preprocess` method.
        rescale_factor (`int` or `float`, *optional*, defaults to `1/255`):
            Scale factor to use if rescaling the image. Only has an effect if `do_rescale` is set to `True`. Can be
            overridden by the `rescale_factor` parameter in the `preprocess` method.
        do_normalize (`bool`, *optional*, defaults to `True`):
            Whether to normalize the image. Can be overridden by the `do_normalize` parameter in the `preprocess`
            method. Can be overridden by the `do_normalize` parameter in the `preprocess` method.
        image_mean (`float` or `list[float]`, *optional*, defaults to `IMAGENET_STANDARD_MEAN`):
            Mean to use if normalizing the image. This is a float or list of floats the length of the number of
            channels in the image. Can be overridden by the `image_mean` parameter in the `preprocess` method. Can be
            overridden by the `image_mean` parameter in the `preprocess` method.
        image_std (`float` or `list[float]`, *optional*, defaults to `IMAGENET_STANDARD_STD`):
            Standard deviation to use if normalizing the image. This is a float or list of floats the length of the
            number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` method.
            Can be overridden by the `image_std` parameter in the `preprocess` method.
        do_convert_rgb (`bool`, *optional*, defaults to `True`):
            Whether to convert the image to RGB.
        do_pad (`bool`, *optional*, defaults to `True`):
            Whether to pad the image to square or not.
    TN   gp?	do_resizer   min_sizeresample
do_rescalerescale_factordo_normalize
image_mean	image_stddo_convert_rgbdo_padc                     t        |   di | || _        || _        |d| _        y t        d |D              | _        y )N)   r1  r1  c              3   8   K   | ]  }t        |d z          yw)   N)r   ).0xs     r]   	<genexpr>z/JanusImageProcessor.__init__.<locals>.<genexpr>J  s     )K
1#a#g,
s   rM   )rN   rO   r/  r'  background_colorr\  )rZ   r&  r   r'  r(  r)  r*  r+  r,  r-  r.  r/  r[   r\   s                r]   rO   zJanusImageProcessor.__init__4  sD     	"6" $3D!$))K
)K$KD!r^   imager7  data_formatinput_data_formatr   c                 N   t        ||      \  }}|t        j                  k(  r|j                  d   n|j                  d   }||k(  r|t	        |||      }|S |}|S t        ||      }t        |t              r|g}nt        |      |k7  rt        d| d      |t        j                  k(  r~t        j                  |||f|j                        }	t        |      D ]  \  }
}||	|
ddddf<    ||kD  r||z
  dz  }||	dd|||z   ddf<   |	S ||z
  dz  }||	dddd|||z   f<   |	S t        j                  |||f|j                        }	t        |      D ]  \  }
}||	dddd|
f<    ||kD  r||z
  dz  }||	|||z   ddddf<   |	S ||z
  dz  }||	dd|||z   ddf<   |	S )a}  
        Pads an image to a square based on the longest edge.

        Args:
            image (`np.ndarray`):
                The image to pad.
            background_color (`int` or `tuple[int, int, int]`, *optional*, defaults to 0):
                The color to use for the padding. Can be an integer for single channel or a
                tuple of integers representing for multi-channel images. If passed as integer
                in multi-channel mode, it will default to `0` in subsequent channels.
            data_format (`str` or `ChannelDimension`, *optional*):
                The channel dimension format for the output image. Can be one of:
                    - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
                    - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
                If unset, will use same as the input image.
            input_data_format (`str` or `ChannelDimension`, *optional*):
                The channel dimension format for the input image. Can be one of:
                    - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
                    - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.

        Returns:
            `np.ndarray`: The padded image.
        r   r   Nz(background_color must have no more than z) elements to match the number of channelsr   r-   )r   r   FIRSTr   r   r  r   r   rY  r   npr  r   	enumerate)rZ   r8  r7  r9  r:  r   r   rG   max_dimresultr  colorstarts                r]   pad_to_squarez!JanusImageProcessor.pad_to_squareL  s!   < 'u.?@):>N>T>T)Tu{{1~Z_ZeZefhZiU? * ,E;@QR 
 L  
 Lfe$ &, 01!"l2:<.Hqr   0 6 66XX|Wg>ekkRF%&675"'q!Qw 8v~ 6)a/7<q%%&.0!34  !5Q.6;q!UUU]223  XXw>ekkRF%&675"'q!Qw 8v~ 6)a/7<uuv~-q!34
  !5Q.6;q%%%-/23r^   c                 r   |t        |      }t        ||      \  }}t        ||      }	t        |d      }|d   |d   k7  rt	        d|d    d|d          |d   }||	z  }
t        t        ||
z        | j                        t        t        ||
z        | j                        g}t        |f||||d|}|S )an  
        Resize an image to dynamically calculated size.

        Args:
            image (`np.ndarray`):
                Image to resize.
            size (`dict[str, int]` or `int`):
                The size to resize the image to. If a dictionary, it should have the keys `"height"` and `"width"`.
            resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BICUBIC`):
                `PILImageResampling` filter to use when resizing the image e.g. `PILImageResampling.BICUBIC`.
            data_format (`ChannelDimension` or `str`, *optional*):
                The channel dimension format for the output image. If unset, the channel dimension format of the input
                image is used. Can be one of:
                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
                - `None`: will be inferred from input
            input_data_format (`ChannelDimension` or `str`, *optional*):
                The channel dimension format for the input image. If unset, the channel dimension format is inferred
                from the input image. Can be one of:
                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
                - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.

        Returns:
            `np.ndarray`: The resized image.
        Tdefault_to_squarer   r   z5Output height and width must be the same. Got height=z and width=)r   r(  r9  r:  )r   r   r  r   r   r   r'  r   )rZ   r8  r   r(  r9  r:  r[   r   r   max_sizedeltaoutput_size_nonpaddeds               r]   r   zJanusImageProcessor.resize  s    F $ >u E&u.?@vu%TT:>T']*GXGWWbcghocpbqr  H~x FUN#T]]3EEM"DMM2!

 
&#/
 
 r^   imagesreturn_tensorsc           
         ||n| j                   }||n| j                  }||n| j                  }||n| j                  }||n| j                  }||n| j
                  }|	|	n| j                  }	||n| j                  }||n| j                  }||n| j                  }||n| j                  }t        |d      }| j                  |      }t        |      }t        |      st        d      t!        |||||	|||       |r|D cg c]  }t#        |       }}|D cg c]  }t%        |       }}|r#t'        |d         rt(        j+                  d       |t-        |d         }|r"|D cg c]  }| j/                  ||||       }}|r!|D cg c]  }| j1                  |||       }}|r!|D cg c]  }| j3                  |||	       }}|r"|D cg c]  }| j5                  |||	|
       }}|D cg c]  }t7        |||       }}t9        d|i|
      }|S c c}w c c}w c c}w c c}w c c}w c c}w c c}w )a`  
        Preprocess an image or batch of images.

        Args:
            images (`ImageInput`):
                Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If
                passing in images with pixel values between 0 and 1, set `do_rescale=False`.
            do_resize (`bool`, *optional*, defaults to `self.do_resize`):
                Whether to resize the image.
            size (`dict[str, int]`, *optional*, defaults to `self.size`):
                Controls the size of the image after `resize`. The shortest edge of the image is resized to
                `size["shortest_edge"]` whilst preserving the aspect ratio. If the longest edge of this resized image
                is > `int(size["shortest_edge"] * (1333 / 800))`, then the image is resized again to make the longest
                edge equal to `int(size["shortest_edge"] * (1333 / 800))`.
            resample (`PILImageResampling`, *optional*, defaults to `self.resample`):
                Resampling filter to use if resizing the image. Only has an effect if `do_resize` is set to `True`.
            do_rescale (`bool`, *optional*, defaults to `self.do_rescale`):
                Whether to rescale the image values between [0 - 1].
            rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`):
                Rescale factor to rescale the image by if `do_rescale` is set to `True`.
            do_normalize (`bool`, *optional*, defaults to `self.do_normalize`):
                Whether to normalize the image.
            image_mean (`float` or `list[float]`, *optional*, defaults to `self.image_mean`):
                Image mean to normalize the image by if `do_normalize` is set to `True`.
            image_std (`float` or `list[float]`, *optional*, defaults to `self.image_std`):
                Image standard deviation to normalize the image by if `do_normalize` is set to `True`.
            do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`):
                Whether to convert the image to RGB.
            background_color (`tuple[int, int, int]`):
                The background color to use for the padding.
            do_pad (`bool`, *optional*, defaults to `self.do_pad`):
                Whether to pad the image to square or not.
            return_tensors (`str` or `TensorType`, *optional*):
                The type of tensors to return. Can be one of:
                    - Unset: Return a list of `np.ndarray`.
                    - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`.
                    - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`.
                    - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`.
                    - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`.
            data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`):
                The channel dimension format for the output image. Can be one of:
                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
                - Unset: Use the channel dimension format of the input image.
            input_data_format (`ChannelDimension` or `str`, *optional*):
                The channel dimension format for the input image. If unset, the channel dimension format is inferred
                from the input image. Can be one of:
                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
                - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.
        FrE  zkInvalid image type. Must be of type PIL.Image.Image, numpy.ndarray, torch.Tensor, tf.Tensor or jax.ndarray.)r)  r*  r+  r,  r-  r&  r   r(  r   zIt looks like you are trying to rescale already rescaled images. If the input images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again.)r8  r   r(  r:  )r8  r7  r:  )r8  r   r:  r8  meanstdr:  input_channel_dimr   datatensor_type)r&  r(  r)  r*  r+  r,  r-  r.  r/  r7  r   r   fetch_imagesr   r    r   r!   r   r   r   r   warning_oncer   r   rC  rescaler/  r   r   )rZ   rJ  r&  r   r(  r)  r*  r+  r,  r-  rK  r.  r7  r/  r9  r:  r8  encoded_outputss                     r]   
preprocesszJanusImageProcessor.preprocess  s   L "+!6IDNN	'38#-#9Zt
+9+E4K^K^'3'?|TEVEV#-#9Zt
!*!6IDNN	+9+E4K^K^!-4;;/?/K+QUQfQf'tTYYTU;""6*)&1F#: 
 	&!)%!		
 9?@nU+F@ 6<<VE.'V</&)4s
 $ >vay I $#E %dXYjk#  
  $ $E ""%5&7 # 
 $    $#E 5Rcd#  
  $#E U^op#   ou
ntej'{N_`nt 	 
 '^V,DR`ae A =

s*   	H$!H)0H.H37H8H=<Ic	                    ||n| j                   }|d| j                  z  n|}||n| j                  }||n| j                  }||n| j                  }t        |      }t        |d   t        j                  j                        rt        |      dkD  r|S |d   S |t        |d         }g }	|D ]  }
t        |
      }
|r| j                  |
|||      }
|rC| j                  |
||      }
|
j                  dd      j                  t         j"                        }
|rB|r@|dk(  r;t%        |
t&        j(                  |	      }
t        j                  j+                  |
      }
|	j-                  |
        d
|	i}|dk7  r|nd}t/        ||      S )znApplies post-processing to the decoded image tokens by reversing transformations applied during preprocessing.Ng      ?r   rt   )r8  r,  r-  r:  )r   r:  r3  zPIL.Image.ImagerP  r   rR  )r)  r*  r+  r,  r-  r   r   PILImagerY  r   r   unnormalizerW  clipastyper=  uint8r   r   LAST	fromarrayr_  r   )rZ   rJ  r)  r*  r+  r,  r-  r:  rK  r   r8  rS  s               r]   postprocesszJanusImageProcessor.postprocesss  s    $.#9Zt
6D6Lt222R`'3'?|TEVEV#-#9Zt
!*!6IDNN	)&1fQi1 [1_6;&);$ >vay IE"5)E((J)_p )  U.Tef

1c*11"((;
~AR/R3E;K;P;Pduv		++E2&! $ -+9=N+NTX>BBr^   c                    d}t        |t              r(t        |      |k7  r t        d| dt        |             |g|z  }t        |t              r(t        |      |k7  r t        d| dt        |             |g|z  }t	        d t        ||      D              }t	        d |D              }| j                  ||||      }|S )a~  
        Unnormalizes `image` using the mean and standard deviation specified by `mean` and `std`.
        image = (image * image_std) + image_mean
        Args:
            image (`torch.Tensor` of shape `(batch_size, num_channels, image_size, image_size)` or `(num_channels, image_size, image_size)`):
                Batch of pixel values to postprocess.
            image_mean (`float` or `Iterable[float]`):
                The mean to use for unnormalization.
            image_std (`float` or `Iterable[float]`):
                The standard deviation to use for unnormalization.
            input_data_format (`ChannelDimension` or `str`, *optional*):
                The channel dimension format for the input image. If unset, the channel dimension format is inferred
                from the input image. Can be one of:
                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
                - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.
        r
   zmean must have z$ elements if it is an iterable, got zstd must have c              3   .   K   | ]  \  }}| |z    y wr   rM   )r4  rN  rO  s      r]   r6  z2JanusImageProcessor.unnormalize.<locals>.<genexpr>  s     W<VytSus{<Vs   c              3   &   K   | ]	  }d |z    yw)rt   NrM   )r4  rO  s     r]   r6  z2JanusImageProcessor.unnormalize.<locals>.<genexpr>  s     ;#a#gs   rM  )r   r   rY  r   r\  zipr/  )rZ   r8  r,  r-  r:  rG   rev_image_meanrev_image_stds           r]   r]  zJanusImageProcessor.unnormalize  s    0 j(+:,. ?<.@dehisetdu!vww$4Ji*9~- >,?cdghqdrcs!tuu"l2IWC
I<VWW;;;n-Sd  
 r^   )r   NN)NNNNNNNr   ) rf   rg   rh   ri   r   BICUBICr   r   r   strr   r   r   r   rO   r=  ndarrayr\  r   rC  r   r*   r<  r   r&   r[  r\  rY  rc  r   r]  rl   rm   s   @r]   r$  r$    s   %R )-'9'A'A,3!:>9=)-!%LL tCH~&L 	L
 %L L c5j)L L U5$u+#567L E%e"456L !L L6 >?>BDHHzzH  U3S=%9 9:H eC)9$9:;	H
 $E#/?*?$@AH 
H\ (:'A'A>BDH?zz? DcNC'(? %	?
 eC)9$9:;? $E#/?*?$@A? 
?B %& %))-15%)*.'+:>9=;?)-GK!%(8(>(>DH!YY D>Y tCH~&	Y
 -.Y TNY !Y tnY U5$u+#567Y E%e"456Y !sJ!78Y !Y #5eCcM.B)B#CDY Y &Y  $E#/?*?$@A!Y" 
#Y 'Y| &**.'+,0+/+/(,1C1C TN1C !	1C
 tn1C T%[)1C DK(1C $C=1C !1Cp EI+zz+ %%01+ /0	+
 $E#/?*?$@A+ 
+r^   r$  )	r$  r   r  r  rx  r  ro   r@   r   )|r  collections.abcr   dataclassesr   typingr   r   r   numpyr=  r   torch.nn.functionalr   
functionalr.  torch.utils.checkpoint.transformers.models.blip.image_processing_blipr	   activationsr   cache_utilsr   configuration_utilsr   
generationr   r   r   r   generation.utilsr   image_processing_utilsr   r   image_transformsr   r   r   image_utilsr   r   r   r   r   r   r   r   r    r!   modeling_outputsr"   modeling_utilsr#   r$   processing_utilsr%   utilsr&   r'   r(   r)   r*   r+   r,   autor.   r/   r0   blip_2.modeling_blip_2r1   !chameleon.configuration_chameleonr2   chameleon.modeling_chameleonr3   r4   r5   r6   r7   idefics.modeling_ideficsr8   r9   llama.modeling_llamar:   siglip.configuration_siglipr;   siglip.modeling_siglipr<   r=   r>   r[  
get_loggerrf   r   r@   ro   r   r   r   r   r   r   r`  r   r   r   r  r  r  r&  r7  r9  r;  r=  rK  rT  rq  rx  r  r  r  r  r$  __all__rM   r^   r]   <module>r     s     $ ! , ,       M !   3 u u 9 A S S   , F &   9 8 5 D  e : < ^ ^ 			H	%
^1* ^1BW+ Wtk#" k#\ 
.? 
. 
. 
	7{ 	7 	7	#A 		"? 	2 "I$299 I$XRYY (*0 *p p2' 2BII $" = "*	< 		8 		B 	RYY  ,J!		 J!ZA		 AH-F -F`299 $RYY   
i
% i

i
Xt$$8/ t$n	E, EP
r^   