§
    ‚Štj˜ä  ã                   óX  — d dl mZ d dlmZ d dlZd dlmc mZ d dlmZ ddl	m
Z ddlmZ ddlmZ dd	lmZmZmZmZ dd
lmZ ddlmZ ddlmZmZmZ ddlmZmZ ddl m!Z! ddl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z( ddl)m*Z* ddl+m,Z, ddl-m.Z. ddl/m0Z0m1Z1m2Z2  e&j3        e4¦  «        Z5e$ G d„ de¦  «        ¦   «         Z6 e$d¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z7 e$d¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z8 e$d¬¦  «        e G d „ d!e¦  «        ¦   «         ¦   «         Z9 G d"„ d#ej:        ¦  «        Z;d$ej<        d%e=d&ej<        fd'„Z>	 d]d)ej:        d*ej<        d+ej<        d,ej<        d-ej<        dz  d.e?d/e?d0e!e#         fd1„Z@ G d2„ d3ej:        ¦  «        ZA G d4„ d5ej:        ¦  «        ZB G d6„ d7e¦  «        ZC G d8„ d9ej:        ¦  «        ZDe$ G d:„ d;e6¦  «        ¦   «         ZE G d<„ d=ej:        ¦  «        ZF G d>„ d?ej:        ¦  «        ZG G d@„ dAej:        ¦  «        ZH G dB„ dCej:        ¦  «        ZI G dD„ dEej:        ¦  «        ZJ G dF„ dGej:        ¦  «        ZK G dH„ dIej:        ¦  «        ZL G dJ„ dKej:        ¦  «        ZM G dL„ dMej:        ¦  «        ZNe$e G dN„ dOe¦  «        ¦   «         ¦   «         ZO e$dP¬¦  «         G dQ„ dRe6¦  «        ¦   «         ZP G dS„ dTej:        ¦  «        ZQ G dU„ dVej:        ¦  «        ZR e$dW¬¦  «         G dX„ dYe6¦  «        ¦   «         ZS G dZ„ d[e6e¦  «        ZTg d\¢ZUdS )^é    )ÚCallable)Ú	dataclassN)Únné   )Úinitialization)ÚACT2FN)ÚCache)Ú%ClassifierFreeGuidanceLogitsProcessorÚGenerationMixinÚGenerationModeÚLogitsProcessorList)ÚGenerateDecoderOnlyOutput)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithPoolingÚModelOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚloggingÚtorch_compilable_checkÚ	torch_int)Úmerge_with_config_defaults)Úcapture_outputsé   )Ú	AutoModelé   )ÚJanusConfigÚJanusVisionConfigÚJanusVQVAEConfigc                   óR   ‡ — e Zd ZU eed<   dZdZdZddgZddgZ	dZ
dZdZˆ fd	„Zˆ xZS )
ÚJanusPreTrainedModelÚconfigÚmodel©ÚimageÚtextTÚLlamaDecoderLayerÚJanusVisionEncoderLayerÚpast_key_valuesÚcausal_maskc                 ó  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rQt	          j        |j        t          j        |j        j	        d         ¦  «         
                    d¦  «        ¦  «         d S d S )Néÿÿÿÿ©r    r0   )ÚsuperÚ_init_weightsÚ
isinstanceÚJanusVisionEmbeddingsÚinitÚcopy_Úposition_idsÚtorchÚarangeÚshapeÚexpand)ÚselfÚmoduleÚ	__class__s     €úf/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/janus/modeling_janus.pyr3   z"JanusPreTrainedModel._init_weights<   s{   ø€ Ý‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ3Ñ4Ô4ð 	iÝŒJ�vÔ*­E¬L¸Ô9LÔ9RÐSUÔ9VÑ,WÔ,W×,^Ò,^Ð_fÑ,gÔ,gÑhÔhÐhÐhÐhð	ið 	ió    )Ú__name__Ú
__module__Ú__qualname__r!   Ú__annotations__Úbase_model_prefixÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_can_compile_fullgraphr3   Ú__classcell__©r?   s   @r@   r%   r%   /   s‰   ø€ € € € € € àÐÐÑØÐØ(ÐØ&*Ð#Ø,Ð.GÐHÐØ#4°mÐ"DÐØÐØ€Nà!Ððið ið ið ið ið ið ið ið irA   r%   z9
    Base class for Janus VQ-VAE mode model outputs.
    )Úcustom_introc                   óP   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dS )ÚJanusVQVAEOutputzÿ
    decoded_pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
        Reconstructed pixel values after encoding and decoding the input.
    embedding_loss (`torch.FloatTensor`):
        Embedding loss.
    NÚdecoded_pixel_valuesÚembedding_loss)	rB   rC   rD   Ú__doc__rS   r9   ÚFloatTensorrE   rT   © rA   r@   rR   rR   B   sO   € € € € € € ðð ð 6:Ð˜%Ô+¨dÑ2Ð9Ð9Ñ9Ø/3€N�EÔ%¨Ñ,Ð3Ð3Ñ3Ð3Ð3rA   rR   zy
    Base class for Janus model's outputs that may also contain a past key/values (to speed up sequential decoding).
    c                   óÄ   — e Zd ZU dZdZej        dz  ed<   dZe	dz  ed<   dZ
eej                 dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dS )ÚJanusBaseModelOutputWithPasta…  
    last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
        Sequence of hidden-states at the output of the last layer of the model.

        If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
        hidden_size)` is output.
    image_hidden_states (`tuple(torch.FloatTensor)`, *optional*):
        Tuple of `torch.FloatTensor` (one for the output of the image embeddings, `(batch_size, num_images,
        sequence_length, hidden_size)`.

        image_hidden_states of the model produced by the vision encoder, and optionally by the perceiver
    NÚlast_hidden_stater-   Úhidden_statesÚ
attentionsÚimage_hidden_states)rB   rC   rD   rU   rZ   r9   rV   rE   r-   r	   r[   Útupler\   r]   rW   rA   r@   rY   rY   T   s£   € € € € € € ðð ð 37Ð�uÔ(¨4Ñ/Ð6Ð6Ñ6Ø$(€O�U˜T‘\Ð(Ð(Ñ(Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ø;?Ð˜˜uÔ0Ô1°DÑ8Ð?Ð?Ñ?Ð?Ð?rA   rY   zQ
    Base class for Janus causal language model (or autoregressive) outputs.
    c                   óâ   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dS )	ÚJanusCausalLMOutputWithPastae  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`tuple(torch.FloatTensor)`, *optional*):
        Tuple of `torch.FloatTensor` (one for the output of the image embeddings, `(batch_size, num_images,
        sequence_length, hidden_size)`.

        image_hidden_states of the model produced by the vision encoder, and optionally by the perceiver
    NÚlossÚlogitsr-   r[   r\   r]   )rB   rC   rD   rU   ra   r9   rV   rE   rb   r-   r	   r[   r^   r\   r]   rW   rA   r@   r`   r`   o   sº   € € € € € € ðð ð" &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø$(€O�U˜T‘\Ð(Ð(Ñ(Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ø;?Ð˜˜uÔ0Ô1°DÑ8Ð?Ð?Ñ?Ð?Ð?rA   r`   c                   óz   ‡ — e Zd Zdefˆ fd„Zdej        dededej        fd„Zdd	ej        d
e	dej        fd„Z
ˆ xZS )r5   r&   c                 ó  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        |j        | _        t          j        |j	        | j        | j        | j        d¬¦  «        | _
        | j        | j        z  dz  | _        | j        | _        t          j        | j        | j        ¦  «        | _        |                      dt!          j        | j        ¦  «                             d¦  «        d¬¦  «         d S )NÚvalid)Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚpaddingr   r8   r1   F)Ú
persistent)r2   Ú__init__r&   Úhidden_sizeÚ	embed_dimÚ
image_sizeÚ
patch_sizer   ÚConv2dÚnum_channelsÚpatch_embeddingÚnum_patchesÚnum_positionsÚ	EmbeddingÚposition_embeddingÚregister_bufferr9   r:   r<   ©r=   r&   r?   s     €r@   rl   zJanusVisionEmbeddings.__init__�   sé   ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ+ˆŒØ Ô+ˆŒØ Ô+ˆŒå!œyØÔ+ØœØœØ”?Øð 
ñ  
ô  
ˆÔð !œO¨t¬Ñ>À1ÑDˆÔØ!Ô-ˆÔÝ"$¤,¨tÔ/AÀ4Ä>Ñ"RÔ"RˆÔØ×Ò˜^­U¬\¸$Ô:LÑ-MÔ-M×-TÒ-TÐU\Ñ-]Ô-]ÐjoÐÑpÔpÐpÐpÐprA   Ú
embeddingsÚheightÚwidthÚreturnc                 ó~  — |j         d         }| j        j        j         d         }t          j                             ¦   «         s&||k    r ||k    r|                      | j        ¦  «        S | j        j                             d¦  «        }|j         d         }|| j        z  }|| j        z  }	t          |dz  ¦  «        }
| 
                    d|
|
|¦  «        }|                     dddd¦  «        }t          j                             |||	fdd¬	¦  «        }|                     dddd¦  «                             dd|¦  «        }|S )
a  
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing and no class embeddings.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r    r   r0   g      à?r   r   ÚbicubicF)ÚsizeÚmodeÚalign_corners)r;   rw   Úweightr9   ÚjitÚ
is_tracingr8   Ú	unsqueezerp   r   ÚreshapeÚpermuter   Ú
functionalÚinterpolateÚview)r=   rz   r{   r|   rt   ru   Úpatch_pos_embedÚdimÚ
new_heightÚ	new_widthÚsqrt_num_positionss              r@   Úinterpolate_pos_encodingz.JanusVisionEmbeddings.interpolate_pos_encoding¤   sL  € ð !Ô& qÔ)ˆØÔ/Ô6Ô<¸QÔ?ˆõ Œy×#Ò#Ñ%Ô%ð 	>¨+¸Ò*FÐ*FÈ6ÐUZÊ?È?Ø×*Ò*¨4Ô+<Ñ=Ô=Ð=àÔ1Ô8×BÒBÀ1ÑEÔEˆàÔ˜rÔ"ˆà˜tœÑ.ˆ
Ø˜Tœ_Ñ,ˆ	å& }°cÑ'9Ñ:Ô:ÐØ)×1Ò1°!Ð5GÐI[Ð]`ÑaÔaˆØ)×1Ò1°!°Q¸¸1Ñ=Ô=ˆåœ-×3Ò3ØØ˜iÐ(ØØð	 4ñ 
ô 
ˆð *×1Ò1°!°Q¸¸1Ñ=Ô=×BÒBÀ1ÀbÈ#ÑNÔNˆØÐrA   FÚpixel_valuesr‘   c                 óV  — |j         \  }}}}| j        j        j        }|                      |                     |¬¦  «        ¦  «        }|                     d¦  «                             dd¦  «        }|r|                      |||¦  «        }	n|                      | j	        ¦  «        }	||	z   }|S )N)Údtyper   r    )
r;   rs   rƒ   r”   ÚtoÚflattenÚ	transposer‘   rw   r8   )
r=   r’   r‘   Ú_r{   r|   Útarget_dtypeÚpatch_embedsrz   Ú
pos_embedss
             r@   ÚforwardzJanusVisionEmbeddings.forwardÊ   s­   € Ø*Ô0Ñˆˆ1ˆf�eØÔ+Ô2Ô8ˆØ×+Ò+¨L¯OªOÀ,¨OÑ,OÔ,OÑPÔPˆØ!×)Ò)¨!Ñ,Ô,×6Ò6°q¸!Ñ<Ô<ˆ
à#ð 	DØ×6Ò6°zÀ6È5ÑQÔQˆJˆJà×0Ò0°Ô1BÑCÔCˆJà *Ñ,ˆ
àÐrA   )F)rB   rC   rD   r"   rl   r9   ÚTensorÚintr‘   Úboolrœ   rN   rO   s   @r@   r5   r5   �   s¸   ø€ € € € € ðqÐ0ð qð qð qð qð qð qð($°5´<ð $Èð $ÐUXð $Ð]bÔ]ið $ð $ð $ð $ðLð  E¤Lð ÈDð Ð]bÔ]ið ð ð ð ð ð ð ð rA   r5   r[   Ún_repr}   c                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r    N)r;   r<   r‡   )r[   r    ÚbatchÚnum_key_value_headsÚslenÚhead_dims         r@   Ú	repeat_kvr¦   Ú   s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTrA   ç        r>   ÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 ó  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t
          j                             |
dt          j        ¬¦  «         	                    |j
        ¦  «        }
t
          j                             |
|| j        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )Nr   r   r0   )r�   r”   )ÚpÚtrainingr    )r¦   Únum_key_value_groupsr9   Úmatmulr—   r   r‰   ÚsoftmaxÚfloat32r•   r”   r­   r±   Ú
contiguous)r>   r¨   r©   rª   r«   r¬   r­   r®   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r@   Úeager_attention_forwardr»   æ   sé   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$rA   c                   óf   ‡ — e Zd ZdZdefˆ fd„Z	 d	dej        dej        dz  dee	         fd„Z
ˆ xZS )
ÚJanusVisionAttentionz(Attention Class for Janus Vision Encoderr&   c                 ó6  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        | j        | j        z  | _        | j        | j        z  | j        k    r t          d| j        › d| j        › d�¦  «        ‚| j        dz  | _	        |j
        | _
        |j        }|j        }d| _        d| _        t          j        | j        | j        | j        z  |j        ¬¦  «        | _        t          j        | j        | j        | j        z  |j        ¬¦  «        | _        t          j        | j        | j        | j        z  |j        ¬¦  «        | _        t          j        | j        | j        ¦  «        | _        |dk    rt          j        |¦  «        nt          j        ¦   «         | _        |rt          j        | j        ¦  «        nt          j        ¦   «         | _        |rt          j        | j        ¦  «        nt          j        ¦   «         | _        d S )	Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).ç      à¿Fr    ©Úbiasr   )r2   rl   r&   rm   rn   Únum_attention_headsÚ	num_headsr¥   Ú
ValueErrorÚscaleÚattention_dropoutÚprojection_dropoutÚuse_qk_normÚ	is_causalr²   r   ÚLinearÚattention_biasÚq_projÚk_projÚv_projÚprojection_layerÚDropoutÚIdentityÚ	LayerNormÚq_normÚk_norm)r=   r&   Úproj_dropoutÚqk_normr?   s       €r@   rl   zJanusVisionAttention.__init__  sÑ  ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ+ˆŒØÔ3ˆŒØœ¨$¬.Ñ8ˆŒØŒ=˜4œ>Ñ)¨T¬^Ò;Ð;Ýð'ÈdÌnð 'ð 'Ø”Nð'ð 'ð 'ñô ð ð ”] DÑ(ˆŒ
Ø!'Ô!9ˆÔØÔ0ˆØÔ$ˆØˆŒð %&ˆÔ!å”i ¤°´ÀÄÑ0NÐU[ÔUjÐkÑkÔkˆŒÝ”i ¤°´ÀÄÑ0NÐU[ÔUjÐkÑkÔkˆŒÝ”i ¤°´ÀÄÑ0NÐU[ÔUjÐkÑkÔkˆŒÝ "¤	¨$¬.¸$¼.Ñ IÔ IˆÔØ>JÈQÒ>NÐ>N¥"¤*¨\Ñ":Ô":Ð":ÕTVÔT_ÑTaÔTaˆÔà6=ÐP•b”l 4¤>Ñ2Ô2Ð2Å2Ä;Á=Ä=ˆŒØ6=ÐP•b”l 4¤>Ñ2Ô2Ð2Å2Ä;Á=Ä=ˆŒˆˆrA   Nr[   r«   r®   c                 ó   — |                      ¦   «         \  }}}|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }	|                     d| j        | j        ¦  «        }|                      |¦  «        }|                     d| j        | j        ¦  «        }|                      |¦  «        }|                     ||| j        | j        ¦  «         	                    dd¦  «        }|                     ||| j        | j        ¦  «         	                    dd¦  «        }|	 
                    ||| j        | j        ¦  «         	                    dd¦  «        }	t          j        | j        j        t          ¦  «        }
 |
| |||	|f| j        sdn| j        | j        | j        dœ|¤Ž\  }}|                     ||| j        ¦  «        }|                      |¦  «        }|                      |¦  «        }||fS )Nr0   r    r   r§   )r­   r¬   rÉ   )r€   rÌ   rÍ   rÎ   r‡   rÃ   r¥   rÓ   rÔ   r—   r‹   r   Úget_interfacer&   Ú_attn_implementationr»   r±   rÆ   rÅ   rÉ   rn   rÏ   rÇ   )r=   r[   r«   r®   Ú
batch_sizeÚseq_lenr˜   Úquery_statesr·   r¸   Úattention_interfacerº   r¹   Úoutputs                 r@   rœ   zJanusVisionAttention.forward  sù  € ð "/×!3Ò!3Ñ!5Ô!5Ñˆ
�G˜Qà—{’{ =Ñ1Ô1ˆØ—[’[ Ñ/Ô/ˆ
Ø—{’{ =Ñ1Ô1ˆà#×+Ò+¨B°´ÀÄÑNÔNˆØ—{’{ <Ñ0Ô0ˆà×'Ò'¨¨D¬N¸D¼MÑJÔJˆ
Ø—[’[ Ñ,Ô,ˆ
à#×+Ò+¨J¸ÀÄÐQUÔQ^Ñ_Ô_×iÒiÐjkÐmnÑoÔoˆØ×'Ò'¨
°G¸T¼^ÈTÌ]Ñ[Ô[×eÒeÐfgÐijÑkÔkˆ
Ø#×(Ò(¨°W¸d¼nÈdÌmÑ\Ô\×fÒfÐghÐjkÑlÔlˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð  $œ}ÐH�C�C°$Ô2HØ”JØ”nð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð "×)Ò)¨*°g¸t¼~ÑNÔNˆà×&Ò& {Ñ3Ô3ˆØ×(Ò(¨Ñ0Ô0ˆØ�|Ð#Ð#rA   ©N)rB   rC   rD   rU   r"   rl   r9   r�   r   r   rœ   rN   rO   s   @r@   r½   r½   ÿ   sœ   ø€ € € € € Ø2Ð2ðQÐ0ð Qð Qð Qð Qð Qð Qð@ /3ð)$ð )$à”|ð)$ð œ tÑ+ð)$ð Ð+Ô,ð	)$ð )$ð )$ð )$ð )$ð )$ð )$ð )$rA   r½   c                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )ÚJanusVisionMLPr&   c                 óÎ  •— t          ¦   «                              ¦   «          || _        t          |j        |j        z  ¦  «        | _        t          |j                 | _	        t          j        |j        | j        ¦  «        | _        t          j        | j        |j        ¦  «        | _        t          j        |j        ¦  «        | _        t          j        |j        ¦  «        | _        d S rß   )r2   rl   r&   rž   rm   Ú	mlp_ratioÚintermediate_sizer   Ú
hidden_actÚactivation_fnr   rÊ   Úfc1Úfc2rÐ   Úhidden_dropout_rateÚdropout1Údropout2ry   s     €r@   rl   zJanusVisionMLP.__init__L  s©   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ!$ VÔ%7¸&Ô:JÑ%JÑ!KÔ!KˆÔÝ# FÔ$5Ô6ˆÔÝ”9˜VÔ/°Ô1GÑHÔHˆŒÝ”9˜TÔ3°VÔ5GÑHÔHˆŒÝœ
 6Ô#=Ñ>Ô>ˆŒÝœ
 6Ô#=Ñ>Ô>ˆŒˆˆrA   r[   r}   c                 óØ   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S rß   )rç   ræ   rê   rè   rë   ©r=   r[   s     r@   rœ   zJanusVisionMLP.forwardV  s_   € ØŸš Ñ/Ô/ˆØ×*Ò*¨=Ñ9Ô9ˆØŸš mÑ4Ô4ˆØŸš Ñ/Ô/ˆØŸš mÑ4Ô4ˆØÐrA   )	rB   rC   rD   r"   rl   r9   r�   rœ   rN   rO   s   @r@   rá   rá   K  sk   ø€ € € € € ð?Ð0ð ?ð ?ð ?ð ?ð ?ð ?ð U¤\ð °e´lð ð ð ð ð ð ð ð rA   rá   c            	       óv   ‡ — e Zd Zdefˆ fd„Zedej        dej        dee	         dej
        fd„¦   «         Zˆ xZS )r,   r&   c                 óR  •— t          ¦   «                              ¦   «          |j        | _        t	          j        | j        |j        ¬¦  «        | _        t          |¦  «        | _	        t	          j        | j        |j        ¬¦  «        | _
        t          |¦  «        | _        || _        d S ©N)Úeps)r2   rl   rm   rn   r   rÒ   Úlayer_norm_epsÚlayer_norm1r½   Ú	self_attnÚlayer_norm2rá   Úmlpr&   ry   s     €r@   rl   z JanusVisionEncoderLayer.__init__`  s„   ø€ Ý‰Œ×ÒÑÔÐØÔ+ˆŒÝœ<¨¬¸FÔ<QÐRÑRÔRˆÔÝ-¨fÑ5Ô5ˆŒÝœ<¨¬¸FÔ<QÐRÑRÔRˆÔÝ! &Ñ)Ô)ˆŒØˆŒˆˆrA   r[   r«   r®   r}   c                 óÆ   — |}|                       |¦  «        } | j        d||dœ|¤Ž\  }}||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|S )N)r[   r«   rW   )ró   rô   rõ   rö   )r=   r[   r«   r®   Úresidualr˜   s         r@   rœ   zJanusVisionEncoderLayer.forwardi  s“   € ð !ˆà×(Ò(¨Ñ7Ô7ˆØ)˜4œ>ð 
Ø'Ø)ð
ð 
ð ð
ð 
Ñˆ�qð
 ! =Ñ0ˆà ˆØ×(Ò(¨Ñ7Ô7ˆØŸš Ñ/Ô/ˆØ  =Ñ0ˆàÐrA   )rB   rC   rD   r"   rl   r   r9   r�   r   r   rV   rœ   rN   rO   s   @r@   r,   r,   _  s˜   ø€ € € € € ðÐ0ð ð ð ð ð ð ð ðà”|ðð œðð Ð+Ô,ð	ð
 
Ô	ðð ð ñ „^ðð ð ð ð rA   r,   c                   ól   ‡ — e Zd ZdZdefˆ fd„Ze	 d	dej        dz  de	e
         defd„¦   «         Zˆ xZS )
ÚJanusVisionEncoderz»
    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
    [`JanusVisionEncoderLayer`].

    Args:
        config: JanusVisionConfig
    r&   c                 óÔ   •‡— t          ¦   «                              ¦   «          ‰| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d| _        d S )Nc                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS rW   )r,   ©Ú.0r˜   r&   s     €r@   ú
<listcomp>z/JanusVisionEncoder.__init__.<locals>.<listcomp>Ž  s"   ø€ Ð$nÐ$nÐ$nÈÕ%<¸VÑ%DÔ%DÐ$nÐ$nÐ$nrA   F)	r2   rl   r&   r   Ú
ModuleListÚrangeÚnum_hidden_layersÚlayersÚgradient_checkpointingry   s    `€r@   rl   zJanusVisionEncoder.__init__‹  sa   øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”mÐ$nÐ$nÐ$nÐ$nÍeÐTZÔTlÑNmÔNmÐ$nÑ$nÔ$nÑoÔoˆŒØ&+ˆÔ#Ð#Ð#rA   Nr«   r®   r}   c                 óN   — |}| j         D ]} |||fi |¤Ž}Œt          |¬¦  «        S )N)rZ   )r  r   )r=   Úinputs_embedsr«   r®   r[   Úencoder_layers         r@   rœ   zJanusVisionEncoder.forward’  sU   € ð &ˆØ!œ[ð 	ð 	ˆMØ)˜MØØðð ð ðð ˆMˆMõ °Ð?Ñ?Ô?Ð?rA   rß   )rB   rC   rD   rU   r"   rl   r   r9   r�   r   r   r   rœ   rN   rO   s   @r@   rú   rú   ‚  s°   ø€ € € € € ðð ð,Ð0ð ,ð ,ð ,ð ,ð ,ð ,ð ð /3ð@ð @ð œ tÑ+ð@ð Ð+Ô,ð	@ð
 
ð@ð @ð @ñ „^ð@ð @ð @ð @ð @rA   rú   c                   óÌ   ‡ — e Zd ZU dZdZeed<   eedœZ	defˆ fd„Z
e ed¬¦  «        e	 	 ddej        dz  d	ed
ee         deez  fd„¦   «         ¦   «         ¦   «         Zd„ Zˆ xZS )ÚJanusVisionModelr’   )r)   r&   ©r[   r\   c                 ó  •— t          ¦   «                              |¦  «         || _        |j        }t	          |¦  «        | _        t          |¦  «        | _        t          j	        ||j
        ¬¦  «        | _        |                      ¦   «          d S rð   )r2   rl   r&   rm   r5   rz   rú   Úencoderr   rÒ   rò   Úpost_layernormÚ	post_init)r=   r&   rn   r?   s      €r@   rl   zJanusVisionModel.__init__®  sx   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒØÔ&ˆ	å/°Ñ7Ô7ˆŒÝ)¨&Ñ1Ô1ˆŒÝ œl¨9¸&Ô:OÐPÑPÔPˆÔà�ŠÑÔÐÐÐrA   F)Útie_last_hidden_statesNr‘   r®   r}   c                 ó  — |€t          d¦  «        ‚|                      ||¬¦  «        } | j        dd|i|¤Ž}|j        }|                      |¦  «        }|d d …dd d …f         }|                      |¦  «        }t          ||¬¦  «        S )Nz You have to specify pixel_values)r‘   r  r   )rZ   Úpooler_outputrW   )rÄ   rz   r  rZ   r  r   )r=   r’   r‘   r®   r[   Úencoder_outputsrZ   Úpooled_outputs           r@   rœ   zJanusVisionModel.forward¹  s¿   € ð ÐÝÐ?Ñ@Ô@Ð@àŸš¨ÐOg˜ÑhÔhˆà+7¨4¬<ð ,
ð ,
Ø'ð,
àð,
ð ,
ˆð
 ,Ô=ÐØ ×/Ò/Ð0AÑBÔBÐà)¨!¨!¨!¨Q°°°¨'Ô2ˆØ×+Ò+¨MÑ:Ô:ˆå)Ø/Ø'ð
ñ 
ô 
ð 	
rA   c                 ó   — | j         S rß   )rz   )r=   s    r@   Úget_input_embeddingsz%JanusVisionModel.get_input_embeddings×  s
   € ØŒÐrA   ©NF)rB   rC   rD   Úmain_input_namerG   r"   rE   r,   r½   Ú_can_record_outputsrl   r   r   r   r9   rV   rŸ   r   r   r^   r   rœ   r  rN   rO   s   @r@   r	  r	  ¤  s
  ø€ € € € € € à$€OØ!ÐØÐÐÑà0Ø*ðð Ðð
	Ð0ð 	ð 	ð 	ð 	ð 	ð 	ð  Ø€_¨EÐ2Ñ2Ô2Øð 26Ø).ð
ð 
àÔ'¨$Ñ.ð
ð #'ð
ð Ð+Ô,ð	
ð
 
Ð+Ñ	+ð
ð 
ð 
ñ „^ñ 3Ô2ñ  Ôð
ð6ð ð ð ð ð ð rA   r	  c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚJanusVisionAlignerMLPr&   c                 ó0  •‡— t          ¦   «                              ¦   «          t          j        ‰j        ‰j        ¦  «        | _        t          j        ˆfd„t          d‰j	        ¦  «        D ¦   «         ¦  «        | _
        t          ‰j                 | _        d S )Nc                 óN   •— g | ]!}t          j        ‰j        ‰j        ¦  «        ‘Œ"S rW   ©r   rÊ   Úprojection_dimrý   s     €r@   rÿ   z2JanusVisionAlignerMLP.__init__.<locals>.<listcomp>á  s+   ø€ ÐeÐeÐeÈ�RŒY�vÔ,¨fÔ.CÑDÔDÐeÐeÐerA   r    )r2   rl   r   rÊ   rm   r  rç   r   r  ÚdepthÚhidden_layersr   rå   ræ   ry   s    `€r@   rl   zJanusVisionAlignerMLP.__init__Ü  s…   øø€ Ý‰Œ×ÒÑÔÐå”9˜VÔ/°Ô1FÑGÔGˆŒÝœ]ØeÐeÐeÐeÍeÐTUÐW]ÔWcÑNdÔNdÐeÑeÔeñ
ô 
ˆÔõ $ FÔ$5Ô6ˆÔÐÐrA   c                 ó„   — |                       |¦  «        }| j        D ]"}|                      |¦  «        } ||¦  «        }Œ#|S rß   ©rç   r   ræ   ©r=   r[   Úlayers      r@   rœ   zJanusVisionAlignerMLP.forwardå  óO   € ØŸš Ñ/Ô/ˆØÔ'ð 	1ð 	1ˆEØ ×.Ò.¨}Ñ=Ô=ˆMØ!˜E -Ñ0Ô0ˆMˆMØÐrA   )rB   rC   rD   r"   rl   rœ   rN   rO   s   @r@   r  r  Û  sT   ø€ € € € € ð7Ð0ð 7ð 7ð 7ð 7ð 7ð 7ðð ð ð ð ð ð rA   r  c                   ób   ‡ — e Zd ZdZdefˆ fd„Zdej        fd„Zdej	        dej
        fd„Zˆ xZS )	ÚJanusVQVAEVectorQuantizeraâ  
    A module for vector quantization using learned embedding vectors.

    This module implements the quantization process similar to te one described in
    the VQ-VAE (Vector Quantized Variational AutoEncoder) paper. It quantizes continuous
    input vectors into discrete codebook vectors, which are learned during training.
    Current implementation improves over previous ones by avoiding costly matrix multiplications
    and allowing for post-hoc remapping of indices.
    r&   c                 ó  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        t          |dd¦  «        | _        t          j        | j        | j        ¦  «        | _	        |j
        gdz  | _        d S )NÚbetag      Ð?r   )r2   rl   Únum_embeddingsrn   Úembedding_dimÚgetattrr)  r   rv   Ú	embeddingrt   Úquant_state_dimsry   s     €r@   rl   z"JanusVQVAEVectorQuantizer.__init__ø  st   ø€ Ý‰Œ×ÒÑÔÐØ$Ô3ˆÔØ#Ô-ˆÔÝ˜F F¨DÑ1Ô1ˆŒ	åœ dÔ&9¸4Ô;MÑNÔNˆŒØ!'Ô!3Ð 4°qÑ 8ˆÔÐÐrA   Úhidden_statec           
      óR  — |                      dddd¦  «                             ¦   «         }|                     d| j        ¦  «        }t	          j        |dz  dd¬¦  «        t	          j        | j        j        dz  d¬¦  «        z   dt	          j        d	|| j        j         	                    dd¦  «        ¦  «        z  z
  }t	          j
        |d¬¦  «        }|                      |¦  «                             |j        ¦  «        }t	          j        |                     ¦   «         |z
  dz  ¦  «        | j        t	          j        ||                     ¦   «         z
  dz  ¦  «        z  z   }|||z
                       ¦   «         z   }|                      dddd¦  «                             ¦   «         }|||fS )
Nr   r   r   r    r0   T)r�   Úkeepdim©r�   z	bd,dn->bn)rˆ   r¶   r‹   r+  r9   Úsumr-  rƒ   Úeinsumr—   Úargminr;   ÚmeanÚdetachr)  )r=   r/  Úhidden_state_flattenedÚ	distancesÚmin_encoding_indicesÚhidden_state_quantra   s          r@   rœ   z!JanusVQVAEVectorQuantizer.forward  s¨  € Ø#×+Ò+¨A¨q°!°QÑ7Ô7×BÒBÑDÔDˆØ!-×!2Ò!2°2°tÔ7IÑ!JÔ!JÐõ ŒIÐ,¨aÑ/°QÀÐEÑEÔEÝŒi˜œÔ-¨qÑ0°aÐ8Ñ8Ô8ñ9à•%”,˜{Ð,BÀDÄNÔDY×DcÒDcÐdeÐghÑDiÔDiÑjÔjÑjñkð 	õ  %œ|¨I¸1Ð=Ñ=Ô=ÐØ!Ÿ^š^Ð,@ÑAÔA×FÒFÀ|ÔGYÑZÔZÐõ ŒzÐ-×4Ò4Ñ6Ô6¸ÑEÈ!ÑKÑLÔLÈtÌyÕ[`Ô[eØ ,×"5Ò"5Ñ"7Ô"7Ñ7¸AÑ=ñ\
ô \
ñ P
ñ 
ˆð
 *Ð-?À,Ñ-N×,VÒ,VÑ,XÔ,XÑXÐð 0×7Ò7¸¸1¸aÀÑCÔC×NÒNÑPÔPÐà! 4Ð)=Ð=Ð=rA   Úimage_tokensr}   c                 ó:  — |j         d         }| j        j        j         d         }|                      |¦  «        }t          j        |dd¬¦  «        }|                     |g| j        ¢|‘R ¦  «        }|                     dddd¦  «                             ¦   «         }|S )Nr   r0   r   )r°   r�   r   r    )	r;   r-  rƒ   ÚFÚ	normalizer‹   r.  rˆ   r¶   )r=   r<  rÚ   Úemb_dimr;  s        r@   Úget_codebook_entryz,JanusVQVAEVectorQuantizer.get_codebook_entry  s£   € Ø!Ô'¨Ô*ˆ
Ø”~Ô,Ô2°2Ô6ˆð "Ÿ^š^¨LÑ9Ô9Ðåœ[Ð);¸qÀbÐIÑIÔIÐð 0×4Ò4°jÐ5bÀ4ÔCXÐ5bÐZaÐ5bÐ5bÑcÔcÐØ/×7Ò7¸¸1¸aÀÑCÔC×NÒNÑPÔPÐà!Ð!rA   )rB   rC   rD   rU   r#   rl   r9   r�   rœ   Ú
LongTensorrV   rA  rN   rO   s   @r@   r'  r'  í  s”   ø€ € € € € ðð ð9Ð/ð 9ð 9ð 9ð 9ð 9ð 9ð> E¤Lð >ð >ð >ð >ð6"¨uÔ/?ð "ÀEÔDUð "ð "ð "ð "ð "ð "ð "ð "rA   r'  c                   ó*   ‡ — e Zd Z	 	 dˆ fd„	Zd„ Zˆ xZS )ÚJanusVQVAEResnetBlockNFc                 óê  •— t          ¦   «                              ¦   «          || _        |€|n|| _        || _        t
          j                             d|dd¬¦  «        | _        t
          j         	                    ||ddd¬¦  «        | _
        t
          j                             d|dd¬¦  «        | _        t
          j                             |j        ¦  «        | _        t
          j         	                    ||ddd¬¦  «        | _        | j        | j        k    r]| j        r+t
          j         	                    ||ddd¬¦  «        | _        d S t
          j         	                    ||ddd¬¦  «        | _        d S d S )	Né    ç�íµ ÷Æ°>T©Ú
num_groupsrr   rñ   Úaffiner   r    ©rh   ri   rj   r   )r2   rl   rf   rg   Úuse_conv_shortcutr9   r   Ú	GroupNormÚnorm1rq   Úconv1Únorm2rÐ   r­   Úconv2Úconv_shortcutÚnin_shortcut)r=   r&   rf   rg   rR  r?   s        €r@   rl   zJanusVQVAEResnetBlock.__init__-  sR  ø€ õ 	‰Œ×ÒÑÔÐØ&ˆÔØ+7Ð+?˜K˜KÀ\ˆÔØ!.ˆÔå”X×'Ò'°2ÀKÐUYÐbfÐ'ÑgÔgˆŒ
Ý”X—_’_ [°,ÈAÐVWÐab�_ÑcÔcˆŒ
Ý”X×'Ò'°2ÀLÐVZÐcgÐ'ÑhÔhˆŒ
Ý”x×'Ò'¨¬Ñ7Ô7ˆŒÝ”X—_’_ \°<ÈQÐWXÐbc�_ÑdÔdˆŒ
ØÔ˜tÔ0Ò0Ð0ØÔ%ð sÝ%*¤X§_¢_°[À,Ð\]ÐfgÐqr _Ñ%sÔ%s�Ô"Ð"Ð"å$)¤H§O¢O°KÀÐ[\ÐefÐpq OÑ$rÔ$r�Ô!Ð!Ð!ð	 1Ð0rA   c                 óÂ  — |}|                       |¦  «        }|t          j        |¦  «        z  }|                      |¦  «        }|                      |¦  «        }|t          j        |¦  «        z  }|                      |¦  «        }|                      |¦  «        }| j        | j        k    r2| j	        r|  
                    |¦  «        }n|                      |¦  «        }||z   S rß   )rN  r9   ÚsigmoidrO  rP  r­   rQ  rf   rg   rL  rR  rS  )r=   r[   rø   s      r@   rœ   zJanusVQVAEResnetBlock.forwardD  sÓ   € Ø ˆØŸ
š
 =Ñ1Ô1ˆØ�œ }Ñ5Ô5Ñ5ˆØŸ
š
 =Ñ1Ô1ˆàŸ
š
 =Ñ1Ô1ˆØ�œ }Ñ5Ô5Ñ5ˆØŸš ]Ñ3Ô3ˆØŸ
š
 =Ñ1Ô1ˆàÔ˜tÔ0Ò0Ð0ØÔ%ð 7Ø×-Ò-¨hÑ7Ô7��à×,Ò,¨XÑ6Ô6�à˜-Ñ'Ð'rA   r  ©rB   rC   rD   rl   rœ   rN   rO   s   @r@   rD  rD  ,  sZ   ø€ € € € € ð
 Øðsð sð sð sð sð sð.(ð (ð (ð (ð (ð (ð (rA   rD  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚJanusVQVAEAttnBlockc                 óî  •— t          ¦   «                              ¦   «          || _        t          j                             d|dd¬¦  «        | _        t          j                             ||ddd¬¦  «        | _        t          j                             ||ddd¬¦  «        | _	        t          j                             ||ddd¬¦  «        | _
        t          j                             ||ddd¬¦  «        | _        d S )NrF  rG  TrH  r    r   rK  )r2   rl   rf   r9   r   rM  Únormrq   ÚqÚkÚvÚproj_out©r=   rf   r?   s     €r@   rl   zJanusVQVAEAttnBlock.__init__Y  sË   ø€ Ý‰Œ×ÒÑÔÐØ&ˆÔå”H×&Ò&°"À;ÐTXÐaeÐ&ÑfÔfˆŒ	Ý”—’ ¨kÀqÐQRÐ\]�Ñ^Ô^ˆŒÝ”—’ ¨kÀqÐQRÐ\]�Ñ^Ô^ˆŒÝ”—’ ¨kÀqÐQRÐ\]�Ñ^Ô^ˆŒÝœŸš¨°[ÈaÐXYÐcd˜ÑeÔeˆŒˆˆrA   c                 óÄ  — |}|                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|j        \  }}}}	|                     ||||	z  ¦  «                             ddd¦  «        }|                     ||||	z  ¦  «        }t          j        ||¦  «        }
|
t          |¦  «        dz  z  }
t          j        |
d¬¦  «        }
|                     ||||	z  ¦  «        }|
                     ddd¦  «        }
t          j        ||
¦  «                             ||||	¦  «        }|                      |¦  «        }||z   S )Nr   r   r    r¿   r2  )rZ  r[  r\  r]  r;   r‡   rˆ   r9   Úbmmrž   r>  r´   r^  )r=   r[   rø   rÜ   r·   r¸   rÚ   Úchannelsr{   r|   r¹   rº   s               r@   rœ   zJanusVQVAEAttnBlock.forwardc  s[  € Ø ˆØŸ	š	 -Ñ0Ô0ˆØ—v’v˜mÑ,Ô,ˆØ—V’V˜MÑ*Ô*ˆ
Ø—v’v˜mÑ,Ô,ˆð /;Ô.@Ñ+ˆ
�H˜f eØ#×+Ò+¨J¸À&È5Á.ÑQÔQ×YÒYÐZ[Ð]^Ð`aÑbÔbˆØ×'Ò'¨
°H¸fÀu¹nÑMÔMˆ
Ý”y ¨zÑ:Ô:ˆØ#¥s¨8¡}¤}¸Ñ'>Ñ?ˆÝ”y °1Ð5Ñ5Ô5ˆð $×+Ò+¨J¸À&È5Á.ÑQÔQˆØ#×+Ò+¨A¨q°!Ñ4Ô4ˆÝ”i ¨lÑ;Ô;×CÒCÀJÐPXÐZ`ÐbgÑhÔhˆà—m’m KÑ0Ô0ˆØ˜+Ñ%Ð%rA   rV  rO   s   @r@   rX  rX  X  sL   ø€ € € € € ðfð fð fð fð fð&ð &ð &ð &ð &ð &ð &rA   rX  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚJanusVQVAEConvDownsamplec                 ó„   •— t          ¦   «                              ¦   «          t          j        ||ddd¬¦  «        | _        d S )Nr   r   r   rK  )r2   rl   r   rq   Úconvr_  s     €r@   rl   z!JanusVQVAEConvDownsample.__init__|  s:   ø€ Ý‰Œ×ÒÑÔÐÝ”I˜k¨;ÀAÈaÐYZÐ[Ñ[Ô[ˆŒ	ˆ	ˆ	rA   c                 ó`   — t          j        |ddd¬¦  «        }|                      |¦  «        }|S )N)r   r    r   r    Úconstantr   )Úpadr�   rª   )r>  ri  rf  rí   s     r@   rœ   z JanusVQVAEConvDownsample.forward€  s2   € åœ˜m°ÀJÐVWÐXÑXÔXˆØŸ	š	 -Ñ0Ô0ˆØÐrA   rV  rO   s   @r@   rd  rd  {  sL   ø€ € € € € ð\ð \ð \ð \ð \ðð ð ð ð ð ð rA   rd  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚJanusVQVAEConvUpsamplec                 óš   •— t          ¦   «                              ¦   «          t          j                             ||ddd¬¦  «        | _        d S )Nr   r    rK  )r2   rl   r9   r   rq   rf  r_  s     €r@   rl   zJanusVQVAEConvUpsample.__init__ˆ  s>   ø€ Ý‰Œ×ÒÑÔÐÝ”H—O’O K°È!ÐTUÐ_`�OÑaÔaˆŒ	ˆ	ˆ	rA   c                 ó^   — t          j        |dd¬¦  «        }|                      |¦  «        }|S )Ng       @Únearest)Úscale_factorr�   )r>  rŠ   rf  rí   s     r@   rœ   zJanusVQVAEConvUpsample.forwardŒ  s/   € Ýœ mÀ#ÈIÐVÑVÔVˆØŸ	š	 -Ñ0Ô0ˆØÐrA   rV  rO   s   @r@   rk  rk  ‡  sL   ø€ € € € € ðbð bð bð bð bðð ð ð ð ð ð rA   rk  c                   óL   ‡ — e Zd Zdedefˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚJanusVQVAEMidBlockr&   rb  c                 óÌ   •— t          ¦   «                              ¦   «          t          |||¬¦  «        | _        t	          |¦  «        | _        t          |||¬¦  «        | _        d S )N©r&   rf   rg   )r2   rl   rD  Úblock_1rX  Úattn_1Úblock_2)r=   r&   rb  r?   s      €r@   rl   zJanusVQVAEMidBlock.__init__“  sl   ø€ Ý‰Œ×ÒÑÔÐÝ,ØØ Ø!ð
ñ 
ô 
ˆŒõ
 *¨(Ñ3Ô3ˆŒÝ,ØØ Ø!ð
ñ 
ô 
ˆŒˆˆrA   r[   r}   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S rß   )rt  ru  rv  rí   s     r@   rœ   zJanusVQVAEMidBlock.forward¡  s;   € ØŸš ]Ñ3Ô3ˆØŸš MÑ2Ô2ˆØŸš ]Ñ3Ô3ˆØÐrA   )
rB   rC   rD   r#   rž   rl   r9   r�   rœ   rN   rO   s   @r@   rq  rq  ’  sr   ø€ € € € € ð
Ð/ð 
¸3ð 
ð 
ð 
ð 
ð 
ð 
ð U¤\ð °e´lð ð ð ð ð ð ð ð rA   rq  c                   ó4   ‡ — e Zd Zˆ fd„Zdej        fd„Zˆ xZS )ÚJanusVQVAEEncoderc           	      ó¦  •— t          ¦   «                              ¦   «          t          |j        ¦  «        | _        |j        | _        |j        }|j        }|j        }|j	        }|j        }t          j                             ||ddd¬¦  «        | _        dt          |¦  «        z   }|| _        t          j        ¦   «         | _        t%          | j        ¦  «        D �]
}t          j        ¦   «         }	t          j        ¦   «         }
|||         z  }|||         z  }t%          | j        ¦  «        D ]Y}|	                     t)          |||¬¦  «        ¦  «         |}|| j        dz
  k    r"|
                     t+          |¦  «        ¦  «         ŒZt          j        ¦   «         }|	|_        |
|_        || j        dz
  k    rt3          |¦  «        |_        | j                             |¦  «         �Œt7          ||¦  «        | _        t          j                             d|dd¬	¦  «        | _        t          j                             ||rd
|z  n|ddd¬¦  «        | _        d S )Nr   r    rK  )r    rs  rF  rG  TrH  r   ) r2   rl   ÚlenÚchannel_multiplierÚnum_resolutionsÚnum_res_blocksÚbase_channelsrf   Údouble_latentÚlatent_channelsr9   r   rq   Úconv_inr^   Úin_channel_multiplierr   Údownr  ÚappendrD  rX  ÚModuleÚblockÚattnrd  Ú
downsamplerq  ÚmidrM  Únorm_outÚconv_out)r=   r&   r  rf   r€  r�  r|  rƒ  Úi_levelr‡  rˆ  Úblock_inÚ	block_outÚi_blockr„  r?   s                  €r@   rl   zJanusVQVAEEncoder.__init__©  s@  ø€ Ý‰Œ×ÒÑÔÐå" 6Ô#<Ñ=Ô=ˆÔØ$Ô3ˆÔØÔ,ˆØÔ(ˆØÔ,ˆØ Ô0ˆØ#Ô6Ðå”x—’ {°MÈqÐYZÐde�ÑfÔfˆŒà $¥uÐ-?Ñ'@Ô'@Ñ @ÐØ%:ˆÔ"Ý”M‘O”OˆŒ	Ý˜TÔ1Ñ2Ô2ð 	#ñ 	#ˆGÝ”M‘O”OˆEÝ”=‘?”?ˆDØ$Ð'<¸WÔ'EÑEˆHØ%Ð(:¸7Ô(CÑCˆIÝ  Ô!4Ñ5Ô5ð 
?ð 
?�Ø—’Ý)Ø%Ø$,Ø%.ðñ ô ñô ð ð %�Ø˜dÔ2°QÑ6Ò6Ð6Ø—K’KÕ 3°HÑ =Ô =Ñ>Ô>Ð>øå”9‘;”;ˆDØˆDŒJØˆDŒIØ˜$Ô.°Ñ2Ò2Ð2Ý":¸8Ñ"DÔ"D�”ØŒI×Ò˜TÑ"Ô"Ð"Ñ"å% f¨hÑ7Ô7ˆŒåœ×*Ò*°bÀxÐUYÐbfÐ*ÑgÔgˆŒÝœŸšØØ#0ÐEˆA�ÑÐ°oØØØð (ñ 
ô 
ˆŒˆˆrA   r’   c                 óØ  — |                       |¦  «        g}t          | j        ¦  «        D ]à}t          | j        ¦  «        D ]‚} | j        |         j        |         |d         ¦  «        }t          | j        |         j        ¦  «        dk    r! | j        |         j        |         |¦  «        }|                     |¦  «         Œƒ|| j        dz
  k    r9|                     | j        |          	                    |d         ¦  «        ¦  «         Œá|d         }|  
                    |¦  «        }|                      |¦  «        }|t          j        |¦  «        z  }|                      |¦  «        }|S )Nr0   r   r    )r‚  r  r}  r~  r„  r‡  r{  rˆ  r…  r‰  rŠ  r‹  r9   rU  rŒ  )r=   r’   r[   r�  r�  r/  rZ   s          r@   rœ   zJanusVQVAEEncoder.forwardÜ  so  € àŸš lÑ3Ô3Ð4ˆÝ˜TÔ1Ñ2Ô2ð 		Wð 		WˆGÝ  Ô!4Ñ5Ô5ð 3ð 3�Ø@˜tœy¨Ô1Ô7¸Ô@Ø! "Ô%ñ ô  �õ �t”y Ô)Ô.Ñ/Ô/°!Ò3Ð3Ø#C 4¤9¨WÔ#5Ô#:¸7Ô#CÀLÑ#QÔ#Q�LØ×$Ò$ \Ñ2Ô2Ð2Ð2Ø˜$Ô.°Ñ2Ò2Ð2Ø×$Ò$ T¤Y¨wÔ%7×%BÒ%BÀ=ÐQSÔCTÑ%UÔ%UÑVÔVÐVøð *¨"Ô-ÐØ ŸHšHÐ%6Ñ7Ô7Ðð !ŸMšMÐ*;Ñ<Ô<ÐØ�Uœ]Ð+<Ñ=Ô=Ñ=ÐØ ŸMšMÐ*;Ñ<Ô<ÐØ Ð rA   )rB   rC   rD   rl   r9   rB  rœ   rN   rO   s   @r@   ry  ry  ¨  sW   ø€ € € € € ð1
ð 1
ð 1
ð 1
ð 1
ðf! EÔ$4ð !ð !ð !ð !ð !ð !ð !ð !rA   ry  c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚJanusVQVAEDecoderc           	      óz  •— t          ¦   «                              ¦   «          t          |j        ¦  «        | _        |j        | _        |j        }|j        }|j        }||j        | j        dz
           z  }t          j
                             ||ddd¬¦  «        | _        t          ||¦  «        | _        t          j        ¦   «         | _        t#          t%          | j        ¦  «        ¦  «        D ]þ}t          j        ¦   «         }t          j        ¦   «         }||j        |         z  }	t%          | j        dz   ¦  «        D ]Y}
|                     t)          |||	¬¦  «        ¦  «         |	}|| j        dz
  k    r"|                     t+          |¦  «        ¦  «         ŒZt          j        ¦   «         }||_        ||_        |dk    rt3          |¦  «        |_        | j                             |¦  «         Œÿt          j
                             d|dd¬	¦  «        | _        t          j
                             ||ddd¬¦  «        | _        d S )
Nr    r   rK  rs  r   rF  rG  TrH  )r2   rl   r{  r|  r}  r~  r  r�  rg   r9   r   rq   r‚  rq  rŠ  r   ÚupÚreversedr  r…  rD  rX  r†  r‡  rˆ  rk  ÚupsamplerM  r‹  rŒ  )r=   r&   r  r�  rg   rŽ  r�  r‡  rˆ  r�  r�  r•  r?   s               €r@   rl   zJanusVQVAEDecoder.__init__ö  s  ø€ Ý‰Œ×ÒÑÔÐå" 6Ô#<Ñ=Ô=ˆÔØ$Ô3ˆÔØÔ,ˆØ Ô0ˆØÔ*ˆð ! 6Ô#<¸TÔ=QÐTUÑ=UÔ#VÑVˆõ ”x—’ ¸ÈaÐXYÐcd�ÑeÔeˆŒõ & f¨hÑ7Ô7ˆŒõ ”-‘/”/ˆŒÝ¥ dÔ&:Ñ ;Ô ;Ñ<Ô<ð 	ð 	ˆGÝ”M‘O”OˆEÝ”=‘?”?ˆDØ%¨Ô(AÀ'Ô(JÑJˆIÝ  Ô!4°qÑ!8Ñ9Ô9ð 
?ð 
?�Ø—’Ý)Ø%Ø$,Ø%.ðñ ô ñô ð ð %�Ø˜dÔ2°QÑ6Ò6Ð6Ø—K’KÕ 3°HÑ =Ô =Ñ>Ô>Ð>øÝ”‘”ˆBØˆBŒHØˆBŒGØ˜!Š|ˆ|Ý4°XÑ>Ô>�”ØŒG�NŠN˜2ÑÔÐÐõ œ×*Ò*°bÀxÐUYÐbfÐ*ÑgÔgˆŒÝœŸš¨°,ÈAÐVWÐab˜ÑcÔcˆŒˆˆrA   r/  r}   c                 ód  — |                       |¦  «        }|                      |¦  «        }t          | j        ¦  «        D ]¯}t          | j        dz   ¦  «        D ]g} | j        |         j        |         |¦  «        }t          | j        |         j        ¦  «        dk    r! | j        |         j        |         |¦  «        }Œh|| j        dz
  k    r | j        |          	                    |¦  «        }Œ°|  
                    |¦  «        }|t          j        |¦  «        z  }|                      |¦  «        }|S )Nr    r   )r‚  rŠ  r  r}  r~  r•  r‡  r{  rˆ  r—  r‹  r9   rU  rŒ  )r=   r/  r�  r�  s       r@   rœ   zJanusVQVAEDecoder.forward$  s)  € Ø—|’| LÑ1Ô1ˆð —x’x Ñ-Ô-ˆõ ˜TÔ1Ñ2Ô2ð 	Gð 	GˆGÝ  Ô!4°qÑ!8Ñ9Ô9ð Pð P�Ø>˜tœw wÔ/Ô5°gÔ>¸|ÑLÔL�Ý�t”w˜wÔ'Ô,Ñ-Ô-°Ò1Ð1Ø#A 4¤7¨7Ô#3Ô#8¸Ô#AÀ,Ñ#OÔ#O�LøØ˜$Ô.°Ñ2Ò2Ð2Ø#œw wÔ/×8Ò8¸ÑFÔF�øà—}’} \Ñ2Ô2ˆØ�œ lÑ3Ô3Ñ3ˆØ—}’} \Ñ2Ô2ˆØÐrA   )rB   rC   rD   rl   r9   rV   rœ   rN   rO   s   @r@   r“  r“  õ  sf   ø€ € € € € ð,dð ,dð ,dð ,dð ,dð\ EÔ$5ð ¸%Ô:Kð ð ð ð ð ð ð ð rA   r“  c                   ón   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	ej        dz  ed<   dS )ÚJanusVQVAEModelOutputa­  
    quantized_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
        Quantized last hidden state from the VQ-VAE model.
    image_tokens (`torch.FloatTensor` of shape `(batch_size, config.vocab_size`):
        Indices of the image tokens predicted by the VQ-VAE model.
    embedding_loss (`torch.FloatTensor`):
        The embedding loss computed during quantization.
    NÚquantized_last_hidden_stater<  rT   )
rB   rC   rD   rU   r›  r9   rV   rE   r<  rT   rW   rA   r@   rš  rš  9  sh   € € € € € € ðð ð =AÐ Ô!2°TÑ!9Ð@Ð@Ñ@Ø-1€L�%Ô# dÑ*Ð1Ð1Ñ1Ø/3€N�EÔ%¨Ñ,Ð3Ð3Ñ3Ð3Ð3rA   rš  aS  
    The VQ-VAE model used in Janus for encoding/decoding images into discrete tokens.
    This model follows the "Make-a-scene: Scene-based text-to-image generation with human priors" paper from
    [ Oran Gafni, Adam Polyak, Oron Ashual, Shelly Sheynin, Devi Parikh, and Yaniv
    Taigman](https://huggingface.co/papers/2203.13131).
    c                   ó  ‡ — e Zd ZU eed<   g d¢ZeedœZdZ	defˆ fd„Z
eedej        dee         defd„¦   «         ¦   «         Zd	ej        dej        fd
„Zeedej        deej        ej        f         fd„¦   «         ¦   «         Zˆ xZS )Ú
JanusVQVAEr&   )rX  rD  r'  r
  r’   c                 óà  •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          |¦  «        | _        t          j                             |j	        |j
        d¦  «        | _        t          j                             |j
        |j	        d¦  «        | _        |                      ¦   «          t          |¦  «        | _        d| _        |                      ¦   «          d S )Nr    F)r2   rl   ry  r  r'  Úquantizer9   r   rq   r�  rn   Ú
quant_convÚpost_quant_convÚevalr“  Údecoderr  r  ry   s     €r@   rl   zJanusVQVAE.__init___  s´   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å(¨Ñ0Ô0ˆŒÝ1°&Ñ9Ô9ˆŒÝœ(Ÿ/š/¨&Ô*@À&ÔBRÐTUÑVÔVˆŒÝ$œxŸš¨vÔ/?ÀÔAWÐYZÑ[Ô[ˆÔØ�	Š	‰ŒˆÝ(¨Ñ0Ô0ˆŒØ&+ˆÔ#Ø�ŠÑÔÐÐÐrA   r®   r}   c                 ó®   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        \  }}}t          ||||¬¦  «        S )N)rZ   r›  r<  rT   )r  r   rŸ  rš  )r=   r’   r®   r[   Úconv_hidden_statesr›  Úemb_lossÚindicess           r@   ÚencodezJanusVQVAE.encodek  sd   € ð Ÿš \Ñ2Ô2ˆØ!Ÿ_š_¨]Ñ;Ô;ÐØ9=¿ºÐGYÑ9ZÔ9ZÑ6Ð# X¨wÝ$Ø+Ø(CØ Ø#ð	
ñ 
ô 
ð 	
rA   r<  c                 ór  — |j         d         | j        j        d         | j        j        d         z  k    r>t          d| j        j        d         | j        j        d         z  › d|j         › d�¦  «        ‚| j                             |¦  «        }|                      |¦  «        }|                      |¦  «        }|S )aG  
        Decodes quantized token IDs into pixel values.
        Args:
            image_tokens (torch.LongTensor): Batch of token IDs.
        Returns:
            pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
                Pixel values decoded from the token IDs.
        r    r   z4Expected `image_tokens` to have shape `(batch_size, z)`, but got shape `z`.)r;   rŸ  r.  rÄ   rA  r¡  r£  )r=   r<  Úcodebook_entryr[   r’   s        r@   ÚdecodezJanusVQVAE.decodex  s×   € ð Ô˜aÔ  D¤MÔ$BÀ1Ô$EÈÌÔHfÐghÔHiÑ$iÒiÐiÝð9ÀtÄ}ÔGeÐfgÔGhÐkoÔkxô  lJð  KLô  lMñ  HMð 9ð 9Ø".Ô"4ð9ð 9ð 9ñô ð ð œ×9Ò9¸,ÑGÔGˆØ×,Ò,¨^Ñ<Ô<ˆØ—|’| MÑ2Ô2ˆØÐrA   c                 óÂ   — |j         d         } | j        |fddi|¤Ž}|                      |j                             |d¦  «        ¦  «        }t          ||j        ¦  «        S )Nr   Úreturn_dictTr0   )r;   r¨  r«  r<  r‹   rR   rT   )r=   r’   r®   rÚ   Úencode_outputsrS   s         r@   rœ   zJanusVQVAE.forward‹  si   € ð "Ô'¨Ô*ˆ
Ø$˜œ \ÐNÐN¸tÐNÀvÐNÐNˆØ#Ÿ{š{¨>Ô+F×+KÒ+KÈJÐXZÑ+[Ô+[Ñ\Ô\ÐåÐ 4°nÔ6SÑTÔTÐTrA   )rB   rC   rD   r#   rE   rI   rD  rX  r  r  rl   r   r   r9   rB  r   r   rš  r¨  rV   r«  r   r   r^   rœ   rN   rO   s   @r@   r�  r�  J  sO  ø€ € € € € € ð ÐÐÑðð ð Ðð /Ø)ðð Ðð %€Oð
Ð/ð 
ð 
ð 
ð 
ð 
ð 
ð  Øð	
 5Ô#3ð 	
¸vÐFXÔ?Yð 	
Ð^sð 	
ð 	
ð 	
ñ „_ñ  Ôð	
ð 5Ô#3ð ¸Ô8Ið ð ð ð ð& Øð	UàÔ'ð	Uð 
ˆuÔ  %Ô"3Ð3Ô	4ð		Uð 	Uð 	Uñ „^ñ Ôð	Uð 	Uð 	Uð 	Uð 	UrA   r�  c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚJanusVQVAEAlignerMLPr&   c                 ó0  •‡— t          ¦   «                              ¦   «          t          j        ‰j        ‰j        ¦  «        | _        t          j        ˆfd„t          d‰j	        ¦  «        D ¦   «         ¦  «        | _
        t          ‰j                 | _        d S )Nc                 óN   •— g | ]!}t          j        ‰j        ‰j        ¦  «        ‘Œ"S rW   r  rý   s     €r@   rÿ   z1JanusVQVAEAlignerMLP.__init__.<locals>.<listcomp>Ÿ  s+   ø€ ÐqÐqÐqÈ�RŒY�vÔ,¨fÔ.CÑDÔDÐqÐqÐqrA   r    )r2   rl   r   rÊ   rn   r  rç   r   r  r  r   r   rå   ræ   ry   s    `€r@   rl   zJanusVQVAEAlignerMLP.__init__š  s…   øø€ Ý‰Œ×ÒÑÔÐå”9˜VÔ-¨vÔ/DÑEÔEˆŒÝœ]ØqÐqÐqÐqÍeÐTUÐW]ÔWoÑNpÔNpÐqÑqÔqñ
ô 
ˆÔõ $ FÔ$5Ô6ˆÔÐÐrA   c                 ó„   — |                       |¦  «        }| j        D ]"}|                      |¦  «        } ||¦  «        }Œ#|S rß   r"  r#  s      r@   rœ   zJanusVQVAEAlignerMLP.forward£  r%  rA   )rB   rC   rD   r#   rl   rœ   rN   rO   s   @r@   r°  r°  ™  sT   ø€ € € € € ð7Ð/ð 7ð 7ð 7ð 7ð 7ð 7ðð ð ð ð ð ð rA   r°  c                   óL   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        fd„Z	ˆ xZ
S )ÚJanusVQVAEHeadzOHead used for sampling tokens in image generation, replacing the usual lm head.r&   c                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          |j                 | _	        t          j        |j        |j
        ¦  «        | _        d S rß   )r2   rl   r   rÊ   Úimage_token_embed_dimr  r^  r   rå   ræ   r*  Úvision_headry   s     €r@   rl   zJanusVQVAEHead.__init__®  sb   ø€ Ý‰Œ×ÒÑÔÐÝœ	 &Ô">ÀÔ@UÑVÔVˆŒÝ# FÔ$5Ô6ˆÔÝœ9 VÔ%:¸FÔ<QÑRÔRˆÔÐÐrA   r[   r}   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S rß   )r^  ræ   r¸  rí   s     r@   rœ   zJanusVQVAEHead.forward´  s?   € ØŸš mÑ4Ô4ˆØ×*Ò*¨=Ñ9Ô9ˆØ×(Ò(¨Ñ7Ô7ˆØÐrA   )rB   rC   rD   rU   r#   rl   r9   r�   Útensorrœ   rN   rO   s   @r@   rµ  rµ  «  sx   ø€ € € € € ØYÐYðSÐ/ð Sð Sð Sð Sð Sð Sð U¤\ð °e´lð ð ð ð ð ð ð ð rA   rµ  zl
    The Janus model which consists of a siglip vision backbone, a Llama language model and a VQ model.
    c                   óp  ‡ — e Zd Zdefˆ fd„Zeedej        de	e
         deez  fd„¦   «         ¦   «         Zdej        dej        d	ej        fd
„Zee	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dedz  dej        dz  dedz  deej        z  defd„¦   «         ¦   «         Zˆ xZS )Ú
JanusModelr&   c                 ó€  •— t          ¦   «                              |¦  «         || _        t                               |j        ¦  «        | _        t          | j        j        ¦  «        | _        t                               |j
        ¦  «        | _        t          j        | j        j        j        | j        j        j        ¦  «        | _        t#          | j        j        ¦  «        | _        t'          | j        j        ¦  «        | _        t+          j        |j        ¬¦  «        | _        d| _        |                      ¦   «          d S )N)r&   F)r2   rl   r&   r	  Ú_from_configÚvision_configÚvision_modelr  Úalignerr�  Ú	vq_configÚvqmodelr   rv   r*  rn   Úgeneration_embeddingsr°  Úgeneration_alignerrµ  Úgeneration_headr   Úfrom_configÚtext_configÚlanguage_modelr  r  ry   s     €r@   rl   zJanusModel.__init__Á  sñ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒå,×9Ò9¸&Ô:NÑOÔOˆÔÝ,¨TÔ->Ô-EÑFÔFˆŒå!×.Ò.¨vÔ/?Ñ@Ô@ˆŒõ &(¤\°$´,Ô2EÔ2TÐVZÔVbÔViÔVsÑ%tÔ%tˆÔ"Ý"6°t´|Ô7JÑ"KÔ"KˆÔÝ-¨d¬lÔ.AÑBÔBˆÔå'Ô3¸6Ô;MÐNÑNÔNˆÔà&+ˆÔ#à�ŠÑÔÐÐÐrA   r’   r®   r}   c                 ód   —  | j         |fddi|¤Ž}|                      |j        ¦  «        |_        |S )Nr­  T)rÀ  rÁ  rZ   r  )r=   r’   r®   Úvision_outputss       r@   Úget_image_featureszJanusModel.get_image_featuresÖ  s@   € ð
 +˜Ô*¨<ÐTÐTÀTÐTÈVÐTÐTˆØ'+§|¢|°NÔ4TÑ'UÔ'UˆÔ$àÐrA   Ú	input_idsr  Úimage_featuresc                 ó   — |€e| |                       ¦   «         t          j        | j        j        t          j        |j        ¬¦  «        ¦  «        k    }|                     d¦  «        }n|| j        j        k    }|                     ¦   «         }|j	        d         |j	        d         z  }| 
                    d¦  «                             |j        ¦  «        }t          ||j	        d         z  |                     ¦   «         k    d|› d|› �¦  «         |S )zï
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        N©r”   Údevicer0   r   r    z6Image features and image tokens do not match, tokens: z, features: )r  r9   rº  r&   Úimage_token_idÚlongrÑ  Úallr3  r;   r†   r•   r   Únumel)r=   rÍ  r  rÎ  Úspecial_image_maskÚn_image_tokensÚn_image_featuress          r@   Úget_placeholder_maskzJanusModel.get_placeholder_maskà  s  € ð ÐØ!.Ð2M°$×2KÒ2KÑ2MÔ2MÝ”˜Tœ[Ô7½u¼zÐR_ÔRfÐgÑgÔgñ3ô 3ò "Ðð "4×!7Ò!7¸Ñ!;Ô!;ÐÐà!*¨d¬kÔ.HÒ!HÐà+×/Ò/Ñ1Ô1ˆØ)Ô/°Ô2°^Ô5IÈ!Ô5LÑLÐØ/×9Ò9¸"Ñ=Ô=×@Ò@ÀÔAUÑVÔVÐÝØ˜]Ô0°Ô4Ñ4¸×8LÒ8LÑ8NÔ8NÒNØsÀ^ÐsÐsÐaqÐsÐsñ	
ô 	
ð 	
ð "Ð!rA   Nr   r«   r8   r-   Ú	use_cacheÚlogits_to_keepc	           
      ó
  — |d u |d uz  rt          d¦  «        ‚|€ |                      ¦   «         |¦  «        }|�‹|                      |d¬¦  «        j        }
|
                     d|j        d         ¦  «        }|                     |j        |j        ¦  «        }|  	                    |||¬¦  «        }| 
                    ||¦  «        } | j        d||||||dœ|	¤Ž}t          |j        |j        |j        |j        |�|
nd ¬¦  «        S )	NzaYou cannot specify both input_ids and inputs_embeds at the same time, and must specify either oneT)r­  r0   )r  rÎ  )r  r«   r8   r-   rÚ  rÛ  )rZ   r-   r[   r\   r]   rW   )rÄ   r  rÌ  r  r‡   r;   r•   rÑ  r”   rÙ  Úmasked_scatterrÉ  rY   rZ   r-   r[   r\   )r=   rÍ  r’   r«   r8   r-   r  rÚ  rÛ  r®   Úimage_embedsrÎ  Úimage_attention_maskÚ	lm_outputs                 r@   rœ   zJanusModel.forwardø  s]  € ð ˜Ð -°tÐ";Ñ<ð 	ÝØsñô ð ð Ð Ø7˜D×5Ò5Ñ7Ô7¸	ÑBÔBˆMàÐ#Ø×2Ò2°<ÈTÐ2ÑRÔRÔ`ˆLØ)×1Ò1°"°mÔ6IÈ"Ô6MÑNÔNˆNØ+×.Ò.¨}Ô/CÀ]ÔEXÑYÔYˆNØ#'×#<Ò#<Ø¨À~ð $=ñ $ô $Ð ð *×8Ò8Ð9MÈ~Ñ^Ô^ˆMà'�DÔ'ð 
Ø'Ø)Ø%Ø+ØØ)ð
ð 
ð ð
ð 
ˆ	õ ,Ø'Ô9Ø%Ô5Ø#Ô1Ø Ô+Ø0<Ð0H  Èdð
ñ 
ô 
ð 	
rA   )NNNNNNNr   )rB   rC   rD   r!   rl   r   r   r9   rV   r   r   r^   r   rÌ  rB  rÙ  r�   r	   rŸ   rž   rY   rœ   rN   rO   s   @r@   r¼  r¼  »  s£  ø€ € € € € ð˜{ð ð ð ð ð ð ð* ØðØ!Ô-ðØ9?Ð@RÔ9Sðà	Ð+Ñ	+ðð ð ñ „^ñ Ôðð"ØÔ)ð"Ø:?Ô:Kð"Ø]bÔ]nð"ð "ð "ð "ð0 Øð .2Ø15Ø.2Ø04Ø(,Ø26Ø!%Ø-.ð,
ð ,
àÔ# dÑ*ð,
ð Ô'¨$Ñ.ð,
ð œ tÑ+ð	,
ð
 Ô&¨Ñ-ð,
ð  ™ð,
ð Ô(¨4Ñ/ð,
ð ˜$‘;ð,
ð ˜eœlÑ*ð,
ð 
&ð,
ð ,
ð ,
ñ „^ñ Ôð,
ð ,
ð ,
ð ,
ð ,
rA   r¼  c                   óæ  ‡ — e Zd ZddiZdZdZdefˆ fd„Zdej	        dej	        fd	„Z
ee	 	 	 	 	 	 	 	 	 ddej        d
z  dej        d
z  dej	        d
z  dej        d
z  ded
z  dej        d
z  dej        d
z  ded
z  deej	        z  dee         defd„¦   «         ¦   «         Z	 	 	 	 	 	 dˆ fd„	Zdej	        fd„Z ej        ¦   «         	 	 	 ddej	        d
z  dej        d
z  ded
z  fˆ fd„¦   «         Zˆ xZS ) ÚJanusForConditionalGenerationzlm_head.weightz(model.language_model.embed_tokens.weightr(   Tr&   c                 ó  •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        t          j        |j        j        |j        j	        d¬¦  «        | _
        |                      ¦   «          d S )NFrÀ   )r2   rl   r&   r¼  r'   r   rÊ   rÈ  rm   Ú
vocab_sizeÚlm_headr  ry   s     €r@   rl   z&JanusForConditionalGeneration.__init__.  sn   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ Ñ'Ô'ˆŒ
Ý”y Ô!3Ô!?ÀÔASÔA^ÐejÐkÑkÔkˆŒð 	�ŠÑÔÐÐÐrA   Úinputsr}   c                 ón   — | j                              |¦  «        }| j                              |¦  «        }|S rß   )r'   rÄ  rÅ  )r=   ræ  r/  s      r@   Ú'prepare_embeddings_for_image_generationzEJanusForConditionalGeneration.prepare_embeddings_for_image_generation7  s2   € Ø”z×7Ò7¸Ñ?Ô?ˆØ”z×4Ò4°\ÑBÔBˆØÐrA   Nr   rÍ  r’   r«   r8   r-   r  ÚlabelsrÚ  rÛ  r®   c
                 óh  —  | j         d|||||||dœ|
¤Ž}|j        }t          |	t          ¦  «        rt	          |	 d¦  «        n|	}|                      |dd…|dd…f         ¦  «        }d}|�  | j        d||| j        j        j	        dœ|
¤Ž}t          |||j        |j        |j        |j        ¬¦  «        S )a­  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        )rÍ  r’   r«   r8   r-   r  rÚ  N)rb   ré  rä  )ra   rb   r-   r[   r\   r]   rW   )r'   rZ   r4   rž   Úslicerå  Úloss_functionr&   rÈ  rä  r`   r-   r[   r\   r]   )r=   rÍ  r’   r«   r8   r-   r  ré  rÚ  rÛ  r®   Úoutputsr[   Úslice_indicesrb   ra   s                   r@   rœ   z%JanusForConditionalGeneration.forward<  s  € ð* �$”*ð 	
ØØ%Ø)Ø%Ø+Ø'Øð	
ð 	
ð ð	
ð 	
ˆð  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%ð Ø f¸¼Ô9PÔ9[ðð Ø_eðð ˆDõ +ØØØ#Ô3Ø!Ô/ØÔ)Ø 'Ô ;ð
ñ 
ô 
ð 	
rA   Fc           	      ó‚   •—  t          ¦   «         j        |f|||||dœ|¤Ž}	|s|                     dd¦  «        s||	d<   |	S )N)r-   r  r«   rÛ  Úis_first_iterationrÚ  Tr’   )r2   Úprepare_inputs_for_generationÚget)r=   rÍ  r’   r-   r«   r  rÛ  rð  r®   Úmodel_inputsr?   s             €r@   rñ  z;JanusForConditionalGeneration.prepare_inputs_for_generationo  sr   ø€ ð =•u‘w”wÔ<Øð
à+Ø'Ø)Ø)Ø1ð
ð 
ð ð
ð 
ˆð ð 	8 V§Z¢Z°¸TÑ%BÔ%Bð 	8Ø+7ˆL˜Ñ(àÐrA   r<  c                 ót   — | j         j                             |¦  «        }|                     dddd¦  «        }|S )a,  
        Decodes generated image tokens from language model to continuous pixel values
        with VQGAN module via upsampling.
        Args:
            image_tokens (`torch.LongTensor` of shape `(batch_size, num_of_tokens)`):
                The tensors corresponding to the input images.
        r   r   r   r    )r'   rÃ  r«  rˆ   )r=   r<  Údecoded_images      r@   Údecode_image_tokensz1JanusForConditionalGeneration.decode_image_tokens�  s;   € ð œ
Ô*×1Ò1°,Ñ?Ô?ˆØ%×-Ò-¨a°°A°qÑ9Ô9ˆØÐrA   Úlogits_processorc           	      óâ
  •— |                      dd¦  «        } | j        |                      dd ¦  «        fi |¤Ž\  }}|dk    r t          ¦   «         j        d|||d dœ|¤ŽS |                     ¦   «         t
          j        t
          j        fvrt          d¦  «        ‚| 	                    ¦   «          |  
                    |                     ¦   «         ¦  «         |�|nt          ¦   «         }d|d<   |j        €!t                               d¦  «         d	|_        |j        |d
<   |                      ||j        |¦  «        \  }}	}|j        |j        }}
t)          |j        ¦  «        dk    rt          d|j        › d�¦  «        ‚|d u}|                      |||j        ¬¦  «         |j        r9|j        dk    r.|                     t1          |j        ¦  «        ¦  «         d |_        |                      ||j        d         |d ||¬¦  «        } | j        d|||j        dœ|¤Ž\  }}| j        j        j        j        }|j        \  }}|                      dd¦  «        }|                      dd ¦  «        }|                      dd¦  «        }||d<   ||d …d d …f         |j        k    ||d …d d …f         |j!        d         k    z  }||d …d d …f          "                    ||j#        ¦  «          |  $                    ¦   «         |¦  «        }| %                    dd ¦  «        €<|  &                    |j'        pd|dz  tQ          |j)        ||z   ¦  «        |¬¦  «        |d<   tU          j+        ||f|
|¬¦  «        }|j,        }|j-        }|j.        }|j/        }|j0        }|r|rdnd }|r|rdnd }|r|rdnd }|r|rdnd }tc          |¦  «        D �]Z} | j2        d||ddœ|¤Ž}d|v r#|d          3                    |j        ¦  «        |d<    | j        j4        di |¤||dœ¤Ž}|  5                    ||¦  «        }|j6        d d …dd d …f          7                    ¦   «         } | j         8                    | ¦  «        }! |||!¦  «        }"|j9        r@tU          j:        |"d¬¦  «        }#tU          j;        |#d¬¦  «         <                    d¦  «        }$ntU          j=        |"d¬¦  «        }$|$|d d …|f<   tU          j>        |$|$g¦  «        }$|$ ?                    d¦  «        }$|  @                    |$¦  «        }�Œ\|r:|r||!fz  }|r||  A                    ¦   «         fz  }|r
||jB        z  }|r
||jC        z  }|rt‰          ||!||||jE        ¬¦  «        S |S )NÚgeneration_moder*   Úgeneration_config)ræ  r«   rú  Úguidance_scalez’Got incompatible mode for Image Generation, should be one of greedy or sampling. Ensure that beam search is de-activated by setting `num_beams=1`.TrÚ  zU`guidance_scale` is required for CFG but not provided. Setting to default value of 5.é   rû  r   z;Expected input ids of shape (batch_size, seq_len), but got z3Passing `inputs embeds` is not supported currently.)rÑ  r    )rú  Úinput_ids_seq_lengthÚencoder_input_idsÚprefix_allowed_tokens_fnr÷  rÑ  )rÍ  r«   Úexpand_sizer«   Úboi_token_idr-   Ústatic)Úcache_implementationrÚ   Úmax_cache_lenÚmodel_kwargsrÐ  rW   )r  rÍ  rð  )Úoutput_attentionsÚoutput_hidden_statesr0   r2  )Únum_samples)Ú	sequencesÚscoresrb   r\   r[   r-   )FÚpopÚ_prepare_generation_configr2   ÚgenerateÚget_generation_moder   ÚSAMPLEÚGREEDY_SEARCHrÄ   ÚvalidateÚ_validate_model_kwargsÚcopyr   rû  ÚloggerÚwarningÚ_prepare_model_inputsÚbos_token_idr”   rÑ  r{  r;   Ú_prepare_special_tokensr…  r
   Ú_get_logits_processorÚ_expand_inputs_for_generationÚnum_return_sequencesr'   rÀ  r&   Únum_image_tokensÚrepeatÚgeneration_kwargsÚmasked_fill_Úpad_token_idr  rò  Ú_prepare_static_cacher  ÚmaxÚ
max_lengthr9   Úzerosr  r  Úoutput_scoresÚoutput_logitsÚreturn_dict_in_generater  rñ  r•   rÉ  Ú#_update_model_kwargs_for_generationrZ   ÚclonerÆ  Ú	do_sampler´   ÚmultinomialÚsqueezeÚargmaxÚcatr†   rè  Úfloatr\   r[   r   r-   )&r=   ræ  r«   r÷  r®   rù  rú  r  rÍ  Úmodel_input_namer”   rÑ  Úkwargs_has_attention_maskr  rÚ   rÛ   Úinput_tokensÚmaskr  Úgenerated_tokensr  r  r%  r&  r'  Ú
raw_scoresÚ
raw_logitsÚdecoder_hidden_statesÚdecoder_attentionsÚiró  rí  r/  r
  Únext_token_scoresÚprobsÚ
next_tokenr?   s&                                        €r@   r  z&JanusForConditionalGeneration.generate›  s4  ø€ ð !Ÿ*š*Ð%6¸Ñ?Ô?ˆØ*I¨$Ô*IØ�JŠJÐ*¨DÑ1Ô1ð+
ð +
Ø5;ð+
ð +
Ñ'Ð˜<ð
 ˜fÒ$Ð$à#•5‘7”7Ô#ð ØØ-Ø"3Ø#ð	ð ð
 ðð ð ð ×0Ò0Ñ2Ô2½>Ô;PÕR`ÔRnÐ:oÐoÐoÝðTñô ð ð 	×"Ò"Ñ$Ô$Ð$Ø×#Ò# L×$5Ò$5Ñ$7Ô$7Ñ8Ô8Ð8ð 0@Ð/KÐ+Ð+ÕQdÑQfÔQfÐð %)ˆ�[Ñ!àÔ+Ð3Ý�NŠNÐrÑsÔsÐsØ/0ÐÔ,Ø):Ô)IˆÐ%Ñ&ð 59×4NÒ4NØÐ%Ô2°Lñ5
ô 5
Ñ1ˆ	Ð# \ð "œ¨Ô)9ˆvˆåˆyŒÑÔ 1Ò$Ð$ÝðFÈiÌoð Fð Fð Fñô ð ð %3¸$Ð$>Ð!Ø×$Ò$Ð%6Ð8QÐZcÔZjÐ$ÑkÔkÐkð Ô+ð 	4Ð0AÔ0PÐSTÒ0TÐ0TØ×#Ò#Õ$IÐJ[ÔJjÑ$kÔ$kÑlÔlÐlØ/3ÐÔ,ð  ×5Ò5Ø/Ø!*¤°Ô!3Ø'Ø%)Ø-Øð 6ñ 
ô 
Ðð #E $Ô"Dð #
ØØ)Ø)Ô>ð#
ð #
ð ð	#
ð #
Ñˆ	�<ð  œ:Ô2Ô9ÔJÐØ'œoÑˆ
�Gà ×'Ò'¨¨1Ñ-Ô-ˆØ%×)Ò)Ð*:¸DÑAÔAˆØ'×.Ò.¨q°!Ñ4Ô4ˆØ)7ˆÐ%Ñ&ð ˜Z˜[˜[¨!¨!¨!˜^Ô,Ð0AÔ0NÒNØ˜˜˜ a a a˜Ô(Ð,=Ô,OÐP^Ô,_Ò_ñ
ˆð 	�Z�[�[ ! ! !�^Ô$×1Ò1°$Ð8IÔ8VÑWÔWÐWà3˜×1Ò1Ñ3Ô3°LÑAÔAˆà×ÒÐ-¨tÑ4Ô4Ð<à.2×.HÒ.HØ%6Ô%KÐ%WÈxà%¨™>å!Ð"3Ô">Ð@PÐSZÑ@ZÑ[Ô[Ø)ð /Iñ /ô /ˆLÐ*Ñ+õ !œ;¨
Ð4DÐ'EÈUÐ[aÐbÑbÔbÐð .Ô?ÐØ0ÔEÐØ)Ô7ˆØ)Ô7ˆØ"3Ô"KÐà3ÐP¸ÐP�R�RÈDˆ
Ø3ÐP¸ÐP�R�RÈDˆ
Ø'>Ð bÐCWÐ b  Ð^bÐØ$;Ð\Ð@QÐ\˜R˜RÐX\ÐåÐ'Ñ(Ô(ð (	Uñ (	UˆAð >˜4Ô=ð Ø+°|ÐX\ðð Ø`lðð ˆLð   <Ð/Ð/Ø1=Ð>NÔ1O×1RÒ1RÐS`ÔSgÑ1hÔ1h�Ð-Ñ.à/�d”jÔ/ð ð Øðà"3Ø%9ðð ð ð ˆGð  ×CÒCÀGÈ\ÑZÔZˆLØ"Ô4°Q°Q°Q¸¸A¸A¸A°XÔ>×DÒDÑFÔFˆLð ”Z×/Ò/°Ñ=Ô=ˆFØ 0Ð 0°¸FÑ CÔ CÐð !Ô*ð EÝœÐ&7¸RÐ@Ñ@Ô@�Ý"Ô.¨uÀ!ÐDÑDÔD×LÒLÈRÑPÔP�
�
å"œ\Ð*;ÀÐDÑDÔD�
à%/Ð˜Q˜Q˜Q ˜TÑ"õ œ J°
Ð#;Ñ<Ô<ˆJØ#×-Ò-¨bÑ1Ô1ˆJà ×HÒHÈÑTÔTˆM‰Mà"ð 	?Øð (Ø˜v˜iÑ'�
Øð 6Ø˜|×1Ò1Ñ3Ô3Ð5Ñ5�
Ø ð 9Ø" gÔ&8Ñ8Ð"Ø#ð ?Ø%¨Ô)>Ñ>Ð%à"ð 
	$Ý,Ø*ØØ!Ø-Ø3Ø 'Ô 7ðñ ô ð ð $Ð#rA   )	NNNNNNNNr   )NNNNNF)NNN)rB   rC   rD   Ú_tied_weights_keysÚoutput_modalitiesrM   r!   rl   r9   r�   rè  r   r   rB  rV   r	   rŸ   rž   r   r   r`   rœ   rñ  rö  Úno_gradr   r  rN   rO   s   @r@   râ  râ  )  sP  ø€ € € € € Ø*Ð,VÐWÐØ)ÐØ!Ðð˜{ð ð ð ð ð ð ð¸e¼lð ÈuÌ|ð ð ð ð ð
 Øð .2Ø15Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ð/
ð /
àÔ# dÑ*ð/
ð Ô'¨$Ñ.ð/
ð œ tÑ+ð	/
ð
 Ô&¨Ñ-ð/
ð  ™ð/
ð Ô(¨4Ñ/ð/
ð Ô  4Ñ'ð/
ð ˜$‘;ð/
ð ˜eœlÑ*ð/
ð Ð+Ô,ð/
ð 
%ð/
ð /
ð /
ñ „^ñ Ôð/
ðh ØØØØØ ðð ð ð ð ð ð@
°´ð 
ð 
ð 
ð 
ð €U„]�_„_ð '+Ø26Ø7;ð	$ð $à”˜tÑ#ð$ð Ô(¨4Ñ/ð$ð .°Ñ4ð	$ð $ð $ð $ð $ñ „_ð$ð $ð $ð $ð $rA   râ  )r%   râ  r¼  r�  r	  )r§   )VÚcollections.abcr   Údataclassesr   r9   Útorch.nn.functionalr   r‰   r>  Ú r   r6   Úactivationsr   Úcache_utilsr	   Ú
generationr
   r   r   r   Úgeneration.utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   r   Úutils.genericr   Úutils.output_capturingr   Úautor   Úconfiguration_janusr!   r"   r#   Ú
get_loggerrB   r  r%   rR   rY   r`   r†  r5   r�   rž   r¦   r/  r»   r½   rá   r,   rú   r	  r  r'  rD  rX  rd  rk  rq  ry  r“  rš  r�  r°  rµ  r¼  râ  Ú__all__rW   rA   r@   ú<module>rS     s<  ðð* %Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø  Ð  Ð  Ð  Ð  Ð  Ø uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø XÐ XÐ XÐ XÐ XÐ XÐ XÐ XÐ XÐ XØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uÐ uØ 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø Ð Ð Ð Ð Ð Ø QÐ QÐ QÐ QÐ QÐ QÐ QÐ QÐ QÐ Qð 
ˆÔ	˜HÑ	%Ô	%€ð ðið ið ið ið i˜?ñ iô iñ „ðið$ €ððñ ô ð
 ð	4ð 	4ð 	4ð 	4ð 	4�{ñ 	4ô 	4ñ „ñô ð	4ð €ððñ ô ð
 ð@ð @ð @ð @ð @ ;ñ @ô @ñ „ñô ð@ð* €ððñ ô ð
 ð@ð @ð @ð @ð @ +ñ @ô @ñ „ñô ð@ð4Hð Hð Hð Hð H˜BœIñ Hô Hð HðV	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð& ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð2I$ð I$ð I$ð I$ð I$˜2œ9ñ I$ô I$ð I$ðXð ð ð ð �R”Yñ ô ð ð( ð  ð  ð  ð  Ð8ñ  ô  ð  ðF@ð @ð @ð @ð @˜œñ @ô @ð @ðD ð3ð 3ð 3ð 3ð 3Ð+ñ 3ô 3ñ „ð3ðlð ð ð ð ˜BœIñ ô ð ð$<"ð <"ð <"ð <"ð <" ¤	ñ <"ô <"ð <"ð~)(ð )(ð )(ð )(ð )(˜BœIñ )(ô )(ð )(ðX &ð  &ð  &ð  &ð  &˜"œ)ñ  &ô  &ð  &ðF	ð 	ð 	ð 	ð 	˜rœyñ 	ô 	ð 	ðð ð ð ð ˜RœYñ ô ð ðð ð ð ð ˜œñ ô ð ð,J!ð J!ð J!ð J!ð J!˜œ	ñ J!ô J!ð J!ðZAð Að Að Að A˜œ	ñ Aô Að AðH Ø
ð4ð 4ð 4ð 4ð 4Ð6ñ 4ô 4ñ „ñ „ð4ð €ððñ ô ðDUð DUð DUð DUð DUÐ%ñ DUô DUñô ðDUðNð ð ð ð ˜2œ9ñ ô ð ð$ð ð ð ð �R”Yñ ô ð ð  €ððñ ô ð
f
ð f
ð f
ð f
ð f
Ð%ñ f
ô f
ñô ð
f
ðRr$ð r$ð r$ð r$ð r$Ð$8¸/ñ r$ô r$ð r$ðj	 tÐ
sÐ
s€€€rA   