§
    ‚Štj¶ ã                   óÊ  — d dl Z d dlmZ d dlmZ d dlmZ d dlmZ d dl	Z	d dl
mZ d dlmc mZ ddlmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZmZmZ ddlmZ ddlm Z  ddl!m"Z"m#Z#m$Z$ ddl%m&Z&m'Z' ddl(m)Z)m*Z* ddl+m,Z, ddl-m.Z.m/Z/m0Z0m1Z1 ddl2m3Z3m4Z4 ddl5m6Z6 ddl7m8Z8m9Z9m:Z: e/e G d„ de#¦  «        ¦   «         ¦   «         Z;d„ Z< ed¦  «        dfd„¦   «         Z=de	j>        de?de	j>        fd„Z@	 dgd!ejA        d"e	j>        d#e	j>        d$e	j>        d%e	j>        dz  d&eBd'eBd(e,e.         fd)„ZC ee=¦  «         G d*„ d+ejA        ¦  «        ¦   «         ZD ed,¦  «         G d-„ d.ejA        ¦  «        ¦   «         ZE G d/„ d0ejA        ¦  «        ZF G d1„ d2e ¦  «        ZG G d3„ d4ejA        ¦  «        ZH G d5„ d6ejA        ¦  «        ZI G d7„ d8ejA        ¦  «        ZJ G d9„ d:ejA        ¦  «        ZK G d;„ d<ejA        ¦  «        ZL G d=„ d>ejA        ¦  «        ZM G d?„ d@ejA        ¦  «        ZN G dA„ dBejA        ¦  «        ZO G dC„ dDejA        ¦  «        ZP G dE„ dFejA        ¦  «        ZQ G dG„ dHejR        ¦  «        ZS G dI„ dJejA        ¦  «        ZT G dK„ dLejA        ¦  «        ZU G dM„ dNejA        ¦  «        ZV G dO„ dPejA        ¦  «        ZW G dQ„ dRejA        ¦  «        ZX e/dS¬T¦  «         G dU„ dVe*¦  «        ¦   «         ZY G dW„ dX¦  «        ZZe/ G dY„ dZe*¦  «        ¦   «         Z[ G d[„ d\ejA        ¦  «        Z\e/ G d]„ d^e[¦  «        ¦   «         Z]e/ G d_„ d`e[e¦  «        ¦   «         Z^ G da„ dbe[¦  «        Z_ G dc„ dde[e¦  «        Z`g de¢ZadS )hé    N)ÚCallable)Ú	dataclass)Úcached_property)ÚOptionalé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hubÚuse_kernel_func_from_hubÚuse_kernelized_func)Úcreate_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚBaseModelOutputWithPoolingÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚtorch_compilable_check)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )Ú
Emu3ConfigÚEmu3TextConfigÚEmu3VQVAEConfigc                   ó2   — e Zd ZU dZdZej        dz  ed<   dS )ÚEmu3VQVAEModelOutputz™
    image_tokens (`torch.LongTensor` of shape `(batch_size, config.vocab_size`):
        Indices of the image tokens predicted by the VQ-VAE model.
    NÚimage_tokens)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r'   ÚtorchÚ
LongTensorÚ__annotations__© ó    úd/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/emu3/modeling_emu3.pyr&   r&   1   s6   € € € € € € ðð ð
 -1€L�%Ô" TÑ)Ð0Ð0Ñ0Ð0Ð0r0   r&   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..Néÿÿÿÿé   ©Údim)Úshaper,   Úcat)ÚxÚx1Úx2s      r1   Úrotate_halfr<   <   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'r0   Úrotary_pos_embc                 ó¾   — |                      |¦  «        }|                      |¦  «        }| |z  t          | ¦  «        |z  z   }||z  t          |¦  «        |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezer<   )ÚqÚkÚcosÚsinÚunsqueeze_dimÚq_embedÚk_embeds          r1   Úapply_rotary_pos_embrG   C   sc   € ð& �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�GÐÐr0   Úhidden_statesÚn_repÚreturnc                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r!   N)r7   ÚexpandÚreshape)rH   rI   ÚbatchÚnum_key_value_headsÚslenÚhead_dims         r1   Ú	repeat_kvrR   ]   s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr0   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 ó  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t
          j                             |
dt          j        ¬¦  «         	                    |j
        ¦  «        }
t
          j                             |
|| j        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )Nr4   r   r3   )r6   Údtype)ÚpÚtrainingr!   )rR   Únum_key_value_groupsr,   ÚmatmulÚ	transposeÚnnÚ
functionalÚsoftmaxÚfloat32Útor]   rZ   r_   Ú
contiguous)rT   rU   rV   rW   rX   rY   rZ   r[   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r1   Úeager_attention_forwardrm   i   sé   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r0   c                   óÎ   ‡ — e Zd ZdZdedefˆ fd„Z	 	 	 ddej        de	ej        ej        f         dz  dej        dz  d	e
dz  d
ee         de	ej        ej        f         fd„Zˆ xZS )ÚEmu3Attentionú=Multi-headed attention from 'Attention Is All You Need' paperÚconfigÚ	layer_idxc                 ó®  •— t          ¦   «                              ¦   «          || _        || _        t	          |d|j        |j        z  ¦  «        | _        |j        |j        z  | _	        | j        dz  | _
        |j        | _        d| _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        | j        z  |j        |j        ¬¦  «        | _        d S )NrQ   ç      à¿T©Úbias)ÚsuperÚ__init__rq   rr   ÚgetattrÚhidden_sizeÚnum_attention_headsrQ   rO   r`   rY   Úattention_dropoutÚ	is_causalrc   ÚLinearÚattention_biasÚq_projÚk_projÚv_projÚo_proj©Úselfrq   rr   Ú	__class__s      €r1   rx   zEmu3Attention.__init__†   sB  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒÝ ¨
°FÔ4FÈ&ÔJdÑ4dÑeÔeˆŒØ$*Ô$>À&ÔB\Ñ$\ˆÔ!Ø”} dÑ*ˆŒØ!'Ô!9ˆÔØˆŒå”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ&¨¬Ñ6¸Ô8JÐQWÔQfð
ñ 
ô 
ˆŒˆˆr0   NrH   Úposition_embeddingsrX   Úpast_key_valuesr[   rJ   c                 ó"  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }	|                      |¦  «                             |¦  «                             dd¦  «        }
|\  }}t          ||	||¦  «        \  }}	|�|                     |	|
| j	        ¦  «        \  }	}
t          j        | j        j        t          ¦  «        } || ||	|
|f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Nr3   r!   r4   rS   )rZ   rY   )r7   rQ   r€   Úviewrb   r�   r‚   rG   Úupdaterr   r   Úget_interfacerq   Ú_attn_implementationrm   r_   r|   rY   rM   rh   rƒ   )r…   rH   r‡   rX   rˆ   r[   Úinput_shapeÚhidden_shapeÚquery_statesri   rj   rB   rC   Úattention_interfacerl   rk   s                   r1   ÚforwardzEmu3Attention.forward�   sÀ  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r0   ©NNN)r(   r)   r*   r+   r"   Úintrx   r,   ÚTensorÚtupler
   r   r   r’   Ú__classcell__©r†   s   @r1   ro   ro   ‚   så   ø€ € € € € àGÐGð
˜zð 
°cð 
ð 
ð 
ð 
ð 
ð 
ð4 IMØ.2Ø(,ð&)ð &)à”|ð&)ð # 5¤<°´Ð#=Ô>ÀÑEð&)ð œ tÑ+ð	&)ð
  ™ð&)ð Ð+Ô,ð&)ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð&)ð &)ð &)ð &)ð &)ð &)ð &)ð &)r0   ro   ÚRMSNormc                   óT   ‡ — e Zd Zd	deddfˆ fd„Zdej        dej        fd„Zd„ Zˆ xZ	S )
ÚEmu3RMSNormç�íµ ÷Æ°>ÚepsrJ   Nc                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )z:
        Emu3RMSNorm is equivalent to T5LayerNorm
        N)rw   rx   rc   Ú	Parameterr,   ÚonesÚweightÚvariance_epsilon)r…   rz   r�   r†   s      €r1   rx   zEmu3RMSNorm.__init__È   sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐr0   rH   c                 ó  — |j         }|                     t          j        ¦  «        }|                     d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        |                     |¦  «        z  S )Nr4   r3   T)Úkeepdim)	r]   rg   r,   rf   ÚpowÚmeanÚrsqrtr¢   r¡   )r…   rH   Úinput_dtypeÚvariances       r1   r’   zEmu3RMSNorm.forwardÐ   s|   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆØŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:r0   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)r–   r¡   r7   r¢   ©r…   s    r1   Ú
extra_reprzEmu3RMSNorm.extra_repr×   s&   € Ý˜œÔ)Ñ*Ô*ÐIÐI°$Ô2GÐIÐIÐIr0   )rœ   )
r(   r)   r*   Úfloatrx   r,   r•   r’   r¬   r—   r˜   s   @r1   r›   r›   Æ   sŒ   ø€ € € € € ð$ð $¨ð $¸$ð $ð $ð $ð $ð $ð $ð; U¤\ð ;°e´lð ;ð ;ð ;ð ;ðJð Jð Jð Jð Jð Jð Jr0   r›   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚEmu3MLPc                 ó¶  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        t          j        | j        | j        |j        ¬¦  «        | _        t          j        | j        | j        |j        ¬¦  «        | _	        t          j        | j        | j        |j        ¬¦  «        | _
        t          |j                 | _        d S )Nru   )rw   rx   rq   rz   Úintermediate_sizerc   r~   Úmlp_biasÚ	gate_projÚup_projÚ	down_projr	   Ú
hidden_actÚact_fn©r…   rq   r†   s     €r1   rx   zEmu3MLP.__init__Ü   s¯   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ!'Ô!9ˆÔÝœ 4Ô#3°TÔ5KÐRXÔRaÐbÑbÔbˆŒÝ”y Ô!1°4Ô3IÐPVÔP_Ð`Ñ`Ô`ˆŒÝœ 4Ô#9¸4Ô;KÐRXÔRaÐbÑbÔbˆŒÝ˜VÔ.Ô/ˆŒˆˆr0   c                 ó¨   — |                       |                      |                      |¦  «        ¦  «        |                      |¦  «        z  ¦  «        }|S ©N)rµ   r·   r³   r´   )r…   r9   rµ   s      r1   r’   zEmu3MLP.forwardæ   sA   € Ø—N’N 4§;¢;¨t¯~ª~¸aÑ/@Ô/@Ñ#AÔ#AÀDÇLÂLÐQRÁOÄOÑ#SÑTÔTˆ	ØÐr0   ©r(   r)   r*   rx   r’   r—   r˜   s   @r1   r¯   r¯   Û   sG   ø€ € € € € ð0ð 0ð 0ð 0ð 0ðð ð ð ð ð ð r0   r¯   c                   óÒ   ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 ddej        dej        dz  dej        dz  d	e	dz  d
e
dz  deej        ej        f         dz  dee         dej        fd„Zˆ xZS )ÚEmu3DecoderLayerrq   rr   c                 óp  •— t          ¦   «                              ¦   «          |j        | _        t          ||¬¦  «        | _        t          |¦  «        | _        t          |j        |j        ¬¦  «        | _	        t          |j        |j        ¬¦  «        | _
        t          j        |j        ¦  «        | _        d S )N)rq   rr   ©r�   )rw   rx   rz   ro   Ú	self_attnr¯   Úmlpr›   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormrc   ÚDropoutr|   rZ   r„   s      €r1   rx   zEmu3DecoderLayer.__init__ì   s“   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔå&¨fÀ	ÐJÑJÔJˆŒå˜6‘?”?ˆŒÝ*¨6Ô+=À6ÔCVÐWÑWÔWˆÔÝ(3°FÔ4FÈFÔL_Ð(`Ñ(`Ô(`ˆÔ%Ý”z &Ô":Ñ;Ô;ˆŒˆˆr0   NFrH   rX   Úposition_idsrˆ   Ú	use_cacher‡   r[   rJ   c           
      ó  — |}|                       |¦  «        } | j        d||||||dœ|¤Ž\  }}	||                      |¦  «        z   }|}|                      |¦  «        }|                      |¦  «        }||                      |¦  «        z   }|S )N)rH   rX   rÆ   rˆ   rÇ   r‡   r/   )rÃ   rÀ   rZ   rÄ   rÁ   )
r…   rH   rX   rÆ   rˆ   rÇ   r‡   r[   ÚresidualÚ_s
             r1   r’   zEmu3DecoderLayer.forward÷   s·   € ð !ˆØ×,Ò,¨]Ñ;Ô;ˆà)˜4œ>ð 
Ø'Ø)Ø%Ø+ØØ 3ð
ð 
ð ð
ð 
Ñˆ�qð ! 4§<¢<°Ñ#>Ô#>Ñ>ˆà ˆØ×5Ò5°mÑDÔDˆØŸš Ñ/Ô/ˆØ  4§<¢<°Ñ#>Ô#>Ñ>ˆØÐr0   )NNNFN)r(   r)   r*   r"   r”   rx   r,   r•   r-   r
   Úboolr–   r   r   r’   r—   r˜   s   @r1   r½   r½   ë   s÷   ø€ € € € € ð	<˜zð 	<°cð 	<ð 	<ð 	<ð 	<ð 	<ð 	<ð /3Ø04Ø(,Ø!&ØHLðð à”|ðð œ tÑ+ðð Ô&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5¤<°´Ð#=Ô>ÀÑEðð Ð+Ô,ðð 
Œðð ð ð ð ð ð ð r0   r½   c                   ó>   ‡ — e Zd ZdZdefˆ fd„Zdej        fd„Zˆ xZ	S )ÚEmu3VQVAEVectorQuantizeraâ  
    A module for vector quantization using learned embedding vectors.

    This module implements the quantization process similar to te one described in
    the VQ-VAE (Vector Quantized Variational AutoEncoder) paper. It quantizes continuous
    input vectors into discrete codebook vectors, which are learned during training.
    Current implementation improves over previous ones by avoiding costly matrix multiplications
    and allowing for post-hoc remapping of indices.
    rq   c                 óú   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        | j        j        j         	                    d|j        z  d|j        z  ¦  «         d S )Ng      ð¿ç      ð?)
rw   rx   rc   Ú	EmbeddingÚcodebook_sizeÚ	embed_dimÚ	embeddingr¡   ÚdataÚuniform_r¸   s     €r1   rx   z!Emu3VQVAEVectorQuantizer.__init__!  sf   ø€ Ý‰Œ×ÒÑÔÐÝœ fÔ&:¸FÔ<LÑMÔMˆŒØŒÔÔ"×+Ò+¨D°6Ô3GÑ,GÈÈvÔOcÑIcÑdÔdÐdÐdÐdr0   Úhidden_statec                 óô  — |j         \  }}}}}|                     ddddd¦  «                             ¦   «         }|                     d|¦  «        }t	          j        |dz  dd¬¦  «        }t	          j        | j        j        dz  d¬	¦  «        }	dt	          j        || j        j         	                    dd¦  «        ¦  «        z  }
||	z   |
z
  }
t	          j
        |
d¬	¦  «        }|                     ||||¦  «        }|S )
Nr   r!   r   é   r4   r3   T)r6   r¤   r5   )r7   Úpermuterh   rŠ   r,   ÚsumrÓ   r¡   ra   rb   Úargmin)r…   rÖ   Ú
batch_sizeÚtemporalÚchannelsÚheightÚwidthÚhidden_state_flattenedÚhidden_state_sumÚembedding_sumÚ	distancesÚmin_encoding_indicess               r1   r’   z Emu3VQVAEVectorQuantizer.forward&  s  € Ø8DÔ8JÑ5ˆ
�H˜h¨°Ø#×+Ò+¨A¨q°!°Q¸Ñ:Ô:×EÒEÑGÔGˆØ!-×!2Ò!2°2°xÑ!@Ô!@Ðõ !œ9Ð%;¸QÑ%>ÀAÈtÐTÑTÔTÐÝœ	 $¤.Ô"7¸Ñ":ÀÐBÑBÔBˆð �œÐ%;¸T¼^Ô=R×=\Ò=\Ð]^Ð`aÑ=bÔ=bÑcÔcÑcˆ	Ø$ }Ñ4°yÑ@ˆ	å$œ|¨I¸1Ð=Ñ=Ô=ÐØ3×8Ò8¸ÀXÈvÐW\Ñ]Ô]ÐØ#Ð#r0   )
r(   r)   r*   r+   r$   rx   r,   r•   r’   r—   r˜   s   @r1   rÍ   rÍ     sr   ø€ € € € € ðð ðe˜ð eð eð eð eð eð eð
$ E¤Lð $ð $ð $ð $ð $ð $ð $ð $r0   rÍ   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚEmu3VQVAEEncoderConvDownsamplec                 ó„   •— t          ¦   «                              ¦   «          t          j        ||ddd¬¦  «        | _        d S )Nr   r4   r   ©Úkernel_sizeÚstrideÚpadding©rw   rx   rc   ÚConv2dÚconv©r…   Úin_channelsr†   s     €r1   rx   z'Emu3VQVAEEncoderConvDownsample.__init__9  ó:   ø€ Ý‰Œ×ÒÑÔÐÝ”I˜k¨;ÀAÈaÐYZÐ[Ñ[Ô[ˆŒ	ˆ	ˆ	r0   c                 ó`   — t          j        |ddd¬¦  «        }|                      |¦  «        }|S )N)r   r!   r   r!   Úconstantr   )ÚpadÚmoderW   )ÚFrõ   rï   ©r…   rH   s     r1   r’   z&Emu3VQVAEEncoderConvDownsample.forward=  s2   € åœ˜m°ÀJÐVWÐXÑXÔXˆØŸ	š	 -Ñ0Ô0ˆØÐr0   r»   r˜   s   @r1   rç   rç   8  sL   ø€ € € € € ð\ð \ð \ð \ð \ðð ð ð ð ð ð r0   rç   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚEmu3VQVAEEncoderConvUpsamplec                 ó„   •— t          ¦   «                              ¦   «          t          j        ||ddd¬¦  «        | _        d S )Nr   r!   ré   rí   rð   s     €r1   rx   z%Emu3VQVAEEncoderConvUpsample.__init__E  rò   r0   c                 ó^   — t          j        |dd¬¦  «        }|                      |¦  «        }|S )Nç       @Únearest©Úscale_factorrö   )r÷   Úinterpolaterï   rø   s     r1   r’   z$Emu3VQVAEEncoderConvUpsample.forwardI  s/   € Ýœ mÀ#ÈIÐVÑVÔVˆØŸ	š	 -Ñ0Ô0ˆØÐr0   r»   r˜   s   @r1   rú   rú   D  sL   ø€ € € € € ð\ð \ð \ð \ð \ðð ð ð ð ð ð r0   rú   c            	       ó^   ‡ — e Zd Zdededee         dee         fˆ fd„Zdej        fd„Zˆ xZ	S )ÚEmu3VQVAEConv3dÚ
in_channelÚout_channelrê   rë   c                 ó\  •— t          ¦   «                              ¦   «          d„ t          |dd …         |dd …         ¦  «        D ¦   «         }d| _        |d d d…         D ] }| xj        |dz  |dz  z   |dz  fz  c_        Œ!| xj        dz  c_        t	          j        ||||¬¦  «        | _        d S )Nc                 ó   — g | ]
\  }}||z
  ‘ŒS r/   r/   )Ú.0Ú
one_kernelÚ
one_strides      r1   ú
<listcomp>z,Emu3VQVAEConv3d.__init__.<locals>.<listcomp>Y  s"   € ÐpÐpÐpÑ5K°ZÀ˜ jÑ0ÐpÐpÐpr0   r!   r/   r3   r4   )r4   r   )rë   )rw   rx   Úziprì   rc   ÚConv3drï   )r…   r  r  rê   rë   Úpadding_sizesÚpad_sizer†   s          €r1   rx   zEmu3VQVAEConv3d.__init__P  sÕ   ø€ õ 	‰Œ×ÒÑÔÐàpÐpÍsÐS^Ð_`Ð_aÐ_aÔSbÐdjÐklÐkmÐkmÔdnÑOoÔOoÐpÑpÔpˆØˆŒØ% d d¨ dÔ+ð 	Jð 	JˆHØˆLŒL˜X¨™]¨X¸©\Ñ9¸8Àq¹=ÐIÑIˆLŒLˆLØˆŒ˜ÑˆŒå”IØØØØð	
ñ 
ô 
ˆŒ	ˆ	ˆ	r0   rH   c                 ód   — t          j        || j        ¦  «        }|                      |¦  «        }|S rº   )r÷   rõ   rì   rï   rø   s     r1   r’   zEmu3VQVAEConv3d.forwardf  s,   € Ýœ˜m¨T¬\Ñ:Ô:ˆØŸ	š	 -Ñ0Ô0ˆØÐr0   )
r(   r)   r*   r”   r–   rx   r,   r•   r’   r—   r˜   s   @r1   r  r  O  sˆ   ø€ € € € € ð
àð
ð ð
ð ˜3”Zð	
ð
 �c”
ð
ð 
ð 
ð 
ð 
ð 
ð, U¤\ð ð ð ð ð ð ð ð r0   r  c                   óL   ‡ — e Zd Zdedefˆ fd„Zdej        dej        fd„Zˆ xZS )ÚEmu3VQVAESpatialNormrñ   Úout_channelsc                 óú   •— t          ¦   «                              ¦   «          t          j        |ddd¬¦  «        | _        t          j        ||ddd¬¦  «        | _        t          j        ||ddd¬¦  «        | _        d S )Né    rœ   T©Únum_channelsÚ
num_groupsr�   Úaffiner!   r   ré   )rw   rx   rc   Ú	GroupNormÚ
norm_layerrî   Úconv_yÚconv_b©r…   rñ   r  r†   s      €r1   rx   zEmu3VQVAESpatialNorm.__init__m  s•   ø€ õ
 	‰Œ×ÒÑÔÐÝœ,Ø%ØØØð	
ñ 
ô 
ˆŒõ ”iØØØØØð
ñ 
ô 
ˆŒõ ”iØØØØØð
ñ 
ô 
ˆŒˆˆr0   rH   Úquant_statesc                 óÔ   — t          j        ||j        dd …         d¬¦  «        }|                      |¦  «        }||                      |¦  «        z  |                      |¦  «        z   }|S )Néþÿÿÿrþ   )Úsizerö   )r÷   r  r7   r  r  r  )r…   rH   r  s      r1   r’   zEmu3VQVAESpatialNorm.forward‰  sd   € Ý”} \¸Ô8KÈBÈCÈCÔ8PÐW`ÐaÑaÔaˆØŸš¨Ñ6Ô6ˆØ%¨¯ª°LÑ(AÔ(AÑAÀDÇKÂKÐP\ÑD]ÔD]Ñ]ˆØÐr0   ©	r(   r)   r*   r”   rx   r,   r•   r’   r—   r˜   s   @r1   r  r  l  su   ø€ € € € € ð
àð
ð ð
ð 
ð 
ð 
ð 
ð 
ð8 U¤\ð ÀÄð ð ð ð ð ð ð ð r0   r  c                   ó>   ‡ — e Zd Zdedefˆ fd„Zdej        fd„Zˆ xZS )ÚEmu3VQVAETemporalUpsampler  r  c                 óx   •— t          ¦   «                              ¦   «          t          ||dd¬¦  «        | _        d S )N©r   r   r   ©r!   r!   r!   ©rê   rë   ©rw   rx   r  rï   ©r…   r  r  r†   s      €r1   rx   z"Emu3VQVAETemporalUpsample.__init__‘  óA   ø€ õ
 	‰Œ×ÒÑÔÐÝ#ØØØ!Øð	
ñ 
ô 
ˆŒ	ˆ	ˆ	r0   rH   c                 ó|  — |j         \  }}}}}|                     ddddd¦  «                             ¦   «                              |d|¦  «        }t	          j        |dd¬	¦  «        }|                     ||||d¦  «                             ddddd¦  «                             ¦   «         }|                      |¦  «        }|S )
Nr   r!   r   rØ   r4   r3   rý   rþ   rÿ   )r7   rÙ   rh   rŠ   r÷   r  rï   )r…   rH   rÜ   rÞ   rÝ   rß   rà   s          r1   r’   z!Emu3VQVAETemporalUpsample.forwardž  sÀ   € Ø8EÔ8KÑ5ˆ
�H˜h¨°Ø%×-Ò-¨a°°A°q¸!Ñ<Ô<×GÒGÑIÔI×NÒNÈzÐ[]Ð_gÑhÔhˆÝœ mÀ#ÈIÐVÑVÔVˆØ%×*Ò*¨:°xÀÈÐPRÑSÔS×[Ò[Ð\]Ð_`ÐbcÐefÐhiÑjÔj×uÒuÑwÔwˆØŸ	š	 -Ñ0Ô0ˆØÐr0   r#  r˜   s   @r1   r%  r%  �  sl   ø€ € € € € ð
àð
ð ð
ð 
ð 
ð 
ð 
ð 
ð U¤\ð ð ð ð ð ð ð ð r0   r%  c                   ó>   ‡ — e Zd Zdedefˆ fd„Zdej        fd„Zˆ xZS )ÚEmu3VQVAETemporalDownsampler  r  c                 óx   •— t          ¦   «                              ¦   «          t          ||dd¬¦  «        | _        d S )N)rØ   r   r   )r4   r!   r!   r)  r*  r+  s      €r1   rx   z$Emu3VQVAETemporalDownsample.__init__¨  r,  r0   rH   c                 ó0   — |                       |¦  «        }|S rº   )rï   rø   s     r1   r’   z#Emu3VQVAETemporalDownsample.forwardµ  s   € ØŸ	š	 -Ñ0Ô0ˆØÐr0   r#  r˜   s   @r1   r/  r/  §  sl   ø€ € € € € ð
àð
ð ð
ð 
ð 
ð 
ð 
ð 
ð U¤\ð ð ð ð ð ð ð ð r0   r/  c                   ó(   ‡ — e Zd Z	 dˆ fd„	Zd„ Zˆ xZS )ÚEmu3VQVAETemporalResnetBlockNc                 ó�  •— t          ¦   «                              ¦   «          || _        |€|n|| _        t	          j        |¦  «        | _        t          ||dd¬¦  «        | _        t	          j        |¦  «        | _	        t          ||dd¬¦  «        | _
        | j        | j        k    r t	          j        ||ddd¬¦  «        | _        d S d S )Nr'  r(  r)  r!   r   ré   )rw   rx   rñ   r  rc   ÚBatchNorm3dÚnorm1r  Úconv1Únorm2Úconv2r  Únin_shortcutr  s      €r1   rx   z%Emu3VQVAETemporalResnetBlock.__init__»  så   ø€ õ
 	‰Œ×ÒÑÔÐØ&ˆÔØ+7Ð+?˜K˜KÀ\ˆÔå”^ KÑ0Ô0ˆŒ
Ý$ØØØ!Øð	
ñ 
ô 
ˆŒ
õ ”^ LÑ1Ô1ˆŒ
Ý$ØØØ!Øð	
ñ 
ô 
ˆŒ
ð Ô˜tÔ0Ò0Ð0Ý "¤	ØØØØØð!ñ !ô !ˆDÔÐÐð 1Ð0r0   c                 ó^  — |}|                       |¦  «        }|t          j        |¦  «        z  }|                      |¦  «        }|                      |¦  «        }|t          j        |¦  «        z  }|                      |¦  «        }| j        | j        k    r|                      |¦  «        }||z   S rº   )	r6  r,   Úsigmoidr7  r8  r9  rñ   r  r:  )r…   rH   rÉ   s      r1   r’   z$Emu3VQVAETemporalResnetBlock.forwardÛ  s¢   € Ø ˆØŸ
š
 =Ñ1Ô1ˆØ�œ }Ñ5Ô5Ñ5ˆØŸ
š
 =Ñ1Ô1ˆàŸ
š
 =Ñ1Ô1ˆØ�œ }Ñ5Ô5Ñ5ˆØŸ
š
 =Ñ1Ô1ˆàÔ˜tÔ0Ò0Ð0Ø×(Ò(¨Ñ2Ô2ˆHà˜-Ñ'Ð'r0   rº   r»   r˜   s   @r1   r3  r3  º  sR   ø€ € € € € ð ðð ð ð ð ð ð@(ð (ð (ð (ð (ð (ð (r0   r3  c                   ój   ‡ — e Zd Z	 	 ddededz  dedz  fˆ fd„Zd	dej        dej        dz  fd„Zˆ xZS )
ÚEmu3VQVAEResnetBlockNrñ   r  Úquant_channelsc                 ó$  •— t          ¦   «                              ¦   «          || _        |€|n|}|| _        || _        |€;t          j        |ddd¬¦  «        | _        t          j        |ddd¬¦  «        | _        n*t          ||¦  «        | _        t          ||¦  «        | _        t          j
        ||ddd¬¦  «        | _        t          j
        ||ddd¬¦  «        | _        | j        | j        k    r t          j
        ||ddd¬¦  «        | _        d S d S )	Nr  rœ   Tr  r   r!   ré   r   )rw   rx   rñ   r  r?  rc   r  r6  r8  r  rî   r7  r9  r:  )r…   rñ   r  r?  r†   s       €r1   rx   zEmu3VQVAEResnetBlock.__init__ì  s<  ø€ õ 	‰Œ×ÒÑÔÐØ&ˆÔØ&2Ð&:�{�{ÀˆØ(ˆÔØ,ˆÔàÐ!Ýœ°;È2ÐSWÐ`dÐeÑeÔeˆDŒJÝœ°<ÈBÐTXÐaeÐfÑfÔfˆDŒJˆJå-¨n¸kÑJÔJˆDŒJÝ-¨n¸lÑKÔKˆDŒJå”YØØØØØð
ñ 
ô 
ˆŒ
õ ”YØØØØØð
ñ 
ô 
ˆŒ
ð Ô˜tÔ0Ò0Ð0Ý "¤	ØØØØØð!ñ !ô !ˆDÔÐÐð 1Ð0r0   rH   c                 óZ  — | j         €dn|f}|} | j        |g|¢R Ž }|t          j        |¦  «        z  }|                      |¦  «        } | j        |g|¢R Ž }|t          j        |¦  «        z  }|                      |¦  «        }| j        | j        k    r|  	                    |¦  «        }||z   S ©Nr/   )
r?  r6  r,   r<  r7  r8  r9  rñ   r  r:  )r…   rH   r?  Ú	norm_argsrÉ   s        r1   r’   zEmu3VQVAEResnetBlock.forward  sÆ   € ØÔ-Ð5�B�B¸NÐ;Lˆ	à ˆØ"˜œ
 =Ð=°9Ð=Ð=Ð=ˆØ�œ }Ñ5Ô5Ñ5ˆØŸ
š
 =Ñ1Ô1ˆà"˜œ
 =Ð=°9Ð=Ð=Ð=ˆØ�œ }Ñ5Ô5Ñ5ˆØŸ
š
 =Ñ1Ô1ˆàÔ˜tÔ0Ò0Ð0Ø×(Ò(¨Ñ2Ô2ˆHà˜-Ñ'Ð'r0   )NNrº   r#  r˜   s   @r1   r>  r>  ë  s    ø€ € € € € ð $(Ø%)ð	*ð *àð*ð ˜D‘jð*ð ˜d™
ð	*ð *ð *ð *ð *ð *ðX(ð ( U¤\ð (À5Ä<ÐRVÑCVð (ð (ð (ð (ð (ð (ð (ð (r0   r>  c            
       ó„   ‡ — e Zd ZdZdefˆ fd„Z	 d	dej        dej        dz  deej        ej        dz  f         fd„Z	ˆ xZ
S )
ÚEmu3VQVAEAttentionBlockrp   rq   c                 ó�  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        | j        | j        z  | _        | j        | j        z  | j        k    r t          d| j        › d| j        › d�¦  «        ‚| j        dz  | _	        |j
        | _        d| _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        d| _        d S )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).rt   Fr!   )rw   rx   rq   rz   rÒ   r{   Ú	num_headsrQ   Ú
ValueErrorÚscaler|   rZ   r}   rc   r~   r�   r‚   r€   Úout_projr`   r¸   s     €r1   rx   z Emu3VQVAEAttentionBlock.__init__-  s   ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ+ˆŒØÔ3ˆŒØœ¨$¬.Ñ8ˆŒØŒ=˜4œ>Ñ)¨T¬^Ò;Ð;Ýð'ÈdÌnð 'ð 'Ø”Nð'ð 'ð 'ñô ð ð ”] DÑ(ˆŒ
ØÔ/ˆŒØˆŒå”i ¤°´Ñ?Ô?ˆŒÝ”i ¤°´Ñ?Ô?ˆŒÝ”i ¤°´Ñ?Ô?ˆŒÝœ	 $¤.°$´.ÑAÔAˆŒð %&ˆÔ!Ð!Ð!r0   NrH   rX   rJ   c           
      ó¼  — |j         dd…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }t          j        | j	        j
        t          ¦  «        }	 |	| ||||| j        | j        | j        sdn| j        ¬¦  «        \  }
} |
j        g |¢d‘R Ž                      ¦   «         }
|                      |
¦  «        }
|
|fS )z#Input shape: Batch x Time x ChannelNr3   r!   r4   rS   )r}   rY   rZ   )r7   rQ   r€   rŠ   rb   r�   r‚   r   rŒ   rq   r�   rm   r}   rI  r_   rZ   rM   rh   rJ  )r…   rH   rX   r[   rŽ   r�   ÚqueriesÚkeysÚvaluesr‘   rl   rk   s               r1   r’   zEmu3VQVAEAttentionBlock.forwardD  sg  € ð $Ô)¨#¨2¨#Ô.ˆà8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆØ—+’+˜mÑ,Ô,×1Ò1°,Ñ?Ô?×IÒIÈ!ÈQÑOÔOˆØ�{Š{˜=Ñ)Ô)×.Ò.¨|Ñ<Ô<×FÒFÀqÈ!ÑLÔLˆØ—’˜]Ñ+Ô+×0Ò0°Ñ>Ô>×HÒHÈÈAÑNÔNˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØØ”nØ”JØ#œ}Ð>�C�C°$´,ð	%
ñ 	%
ô 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—m’m KÑ0Ô0ˆà˜LÐ(Ð(r0   rº   )r(   r)   r*   r+   r$   rx   r,   r•   r–   r’   r—   r˜   s   @r1   rE  rE  *  s    ø€ € € € € ØGÐGð&˜ð &ð &ð &ð &ð &ð &ð4 /3ð!)ð !)à”|ð!)ð œ tÑ+ð!)ð
 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð!)ð !)ð !)ð !)ð !)ð !)ð !)ð !)r0   rE  c                   ó*   ‡ — e Zd ZdZˆ fd„Zdd„Zˆ xZS )ÚEmu3VQVAEGroupNormzá
    Same as the torch GroupNorm with the only difference that this ones accepts
    an optional kwarg `quant_states` which is not used. This class makes it easier to
    use SpatialNorm or GroupNorm without conditionals
    c                 ó:   •—  t          ¦   «         j        di |¤Ž d S rB  )rw   rx   )r…   r[   r†   s     €r1   rx   zEmu3VQVAEGroupNorm.__init__o  s&   ø€ Ø�‰ŒÔÐ"Ð"˜6Ð"Ð"Ð"Ð"Ð"r0   Nc                 óZ   — t          j        || j        | j        | j        | j        ¦  «        S rº   )r÷   Ú
group_normr  r¡   rv   r�   )r…   Úinputr  s      r1   r’   zEmu3VQVAEGroupNorm.forwardr  s"   € ÝŒ|˜E 4¤?°D´KÀÄÈDÌHÑUÔUÐUr0   rº   )r(   r)   r*   r+   rx   r’   r—   r˜   s   @r1   rP  rP  h  s^   ø€ € € € € ðð ð#ð #ð #ð #ð #ðVð Vð Vð Vð Vð Vð Vð Vr0   rP  c                   óL   ‡ — e Zd Zdˆ fd„	Zddej        dej        dz  fd„Zˆ xZS )ÚEmu3VQVAEMiddleBlockNc                 ó,  •— t          ¦   «                              ¦   «          t          |||¬¦  «        | _        t	          |¦  «        | _        |€t          |ddd¬¦  «        | _        nt          ||¦  «        | _        t          |||¬¦  «        | _	        d S )N©rñ   r  r?  r  rœ   Tr  )
rw   rx   r>  Úblock_1rE  Úattn_1rP  Ú	attn_normr  Úblock_2)r…   rq   rñ   r?  r†   s       €r1   rx   zEmu3VQVAEMiddleBlock.__init__w  s    ø€ Ý‰Œ×ÒÑÔÐå+Ø#Ø$Ø)ð
ñ 
ô 
ˆŒõ
 .¨fÑ5Ô5ˆŒØÐ!Ý/¸[ÐUWÐ]aÐjnÐoÑoÔoˆDŒNˆNå1°.À+ÑNÔNˆDŒNå+Ø#Ø$Ø)ð
ñ 
ô 
ˆŒˆˆr0   rH   r  c                 óž  — |                       ||¦  «        }|}|                      ||¦  «        }|j        \  }}}}|                     ||||z  ¦  «                             dd¦  «        }|                      |¦  «        d         }|                     ||||¦  «                             dddd¦  «        }||z   }|                      ||¦  «        }|S )Nr!   r4   r   r   )	rY  r[  r7   rŠ   rb   rZ  rM   rÙ   r\  )r…   rH   r  rÉ   rÜ   rÞ   rß   rà   s           r1   r’   zEmu3VQVAEMiddleBlock.forward‹  sÕ   € ØŸš ]°LÑAÔAˆØ ˆØŸš }°lÑCÔCˆØ.;Ô.AÑ+ˆ
�H˜f eØ%×*Ò*¨:°xÀÈ%ÁÑPÔP×ZÒZÐ[\Ð^_Ñ`Ô`ˆØŸš MÑ2Ô2°1Ô5ˆØ%×-Ò-¨j¸&À%ÈÑRÔR×ZÒZÐ[\Ð^_ÐabÐdeÑfÔfˆØ  =Ñ0ˆØŸš ]°LÑAÔAˆØÐr0   rº   ©r(   r)   r*   rx   r,   ÚFloatTensorr’   r—   r˜   s   @r1   rV  rV  v  sp   ø€ € € € € ð
ð 
ð 
ð 
ð 
ð 
ð(
ð 
 UÔ%6ð 
ÀeÔFWÐZ^ÑF^ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r0   rV  c                   ó4   ‡ — e Zd Zˆ fd„Zdej        fd„Zˆ xZS )ÚEmu3VQVAEDownBlockc           
      óÜ  •— t          ¦   «                              ¦   «          t          |j        ¦  «        | _        |j        | _        |j        }|j        }dt          |¦  «        z   }|| _        t          j
        ¦   «         | _        t          | j        ¦  «        D �]P}t          j
        ¦   «         }t          j
        ¦   «         }t          j
        ¦   «         }|||         z  }	|||         z  }
t          | j        ¦  «        D ]…}|                     t          |	|
¬¦  «        ¦  «         |
}	|j        �V||j        v rM|                     t!          |¦  «        ¦  «         |                     t          j        |	ddd¬¦  «        ¦  «         Œ†t          j        ¦   «         }||_        ||_        ||_        || j        dz
  k    rt-          |	¦  «        |_        | j                             |¦  «         �ŒRd S )N©r!   ©rñ   r  r  rœ   Tr  r!   )rw   rx   ÚlenÚchannel_multiplierÚnum_resolutionsÚnum_res_blocksÚbase_channelsr–   Úin_channel_multiplierrc   Ú
ModuleListÚdownÚrangeÚappendr>  Úattn_resolutionsrE  r  ÚModuleÚblockÚattnÚ
attn_normsrç   Ú
downsample)r…   rq   ri  rf  rj  Úi_levelrq  rr  rs  Úblock_inÚ	block_outÚi_blockrl  r†   s                €r1   rx   zEmu3VQVAEDownBlock.__init__™  sÞ  ø€ Ý‰Œ×ÒÑÔÐå" 6Ô#<Ñ=Ô=ˆÔØ$Ô3ˆÔØÔ,ˆØ#Ô6Ðà $¥uÐ-?Ñ'@Ô'@Ñ @ÐØ%:ˆÔ"Ý”M‘O”OˆŒ	Ý˜TÔ1Ñ2Ô2ð 	#ñ 	#ˆGÝ”M‘O”OˆEÝ”=‘?”?ˆDÝœ™œˆJØ$Ð'<¸WÔ'EÑEˆHØ%Ð(:¸7Ô(CÑCˆIÝ  Ô!4Ñ5Ô5ð 
qð 
q�Ø—’Ý(Ø$,Ø%.ðñ ô ñô ð ð %�ØÔ*Ð6¸7ÀfÔF]Ð;]Ð;]Ø—K’KÕ 7¸Ñ ?Ô ?Ñ@Ô@Ð@Ø×%Ò%¥b¤lÀÐUWÐ]aÐjnÐ&oÑ&oÔ&oÑpÔpÐpøå”9‘;”;ˆDØˆDŒJØˆDŒIØ(ˆDŒOØ˜$Ô.°Ñ2Ò2Ð2Ý"@ÀÑ"JÔ"J�”ØŒI×Ò˜TÑ"Ô"Ð"Ñ"ð1	#ð 	#r0   rH   c                 óP  — t          | j        ¦  «        D �]\  }}t          | j        ¦  «        D ]Ñ} |j        |         |¦  «        }t          |j        ¦  «        dk    r¡|} |j        |         |¦  «        }|j        \  }}}}	| 	                    ||||	z  ¦  «         
                    dd¦  «        } |j        |         |¦  «        d         }|                     |||	|¦  «                             dddd¦  «        }||z   }ŒÒ|| j        dz
  k    r|                     |¦  «        }�Œ|S )Nr   r!   r4   r   )Ú	enumeraterl  rm  rh  rq  re  rr  rs  r7   rŠ   rb   rM   rÙ   rg  rt  )
r…   rH   ru  Úblocksrx  rÉ   rÜ   rÞ   rß   rà   s
             r1   r’   zEmu3VQVAEDownBlock.forward¾  sF  € Ý(¨¬Ñ3Ô3ð 	Añ 	A‰OˆG�VÝ  Ô!4Ñ5Ô5ð =ð =�Ø 5 ¤¨WÔ 5°mÑ DÔ D�Ý�v”{Ñ#Ô# aÒ'Ð'Ø,�HØ$> FÔ$5°gÔ$>¸}Ñ$MÔ$M�Mà:GÔ:MÑ7�J ¨&°%Ø$1×$6Ò$6°zÀ8ÈVÐV[É^Ñ$\Ô$\×$fÒ$fÐghÐjkÑ$lÔ$l�MØ$8 F¤K°Ô$8¸Ñ$GÔ$GÈÔ$J�Mà$1×$9Ò$9¸*ÀfÈeÐU]Ñ$^Ô$^×$fÒ$fÐghÐjkÐmnÐpqÑ$rÔ$r�MØ$,¨}Ñ$<�Møà˜$Ô.°Ñ2Ò2Ð2Ø &× 1Ò 1°-Ñ @Ô @�ùàÐr0   r^  r˜   s   @r1   ra  ra  ˜  sW   ø€ € € € € ð##ð ##ð ##ð ##ð ##ðJ UÔ%6ð ð ð ð ð ð ð ð r0   ra  c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚEmu3VQVAEUpBlockc           	      óº  •— t          ¦   «                              ¦   «          t          |j        ¦  «        | _        |j        | _        |j        }|j        |j        d         z  }t          j	        ¦   «         | _
        t          t          | j        ¦  «        ¦  «        D �]=}t          j	        ¦   «         }t          j	        ¦   «         }t          j	        ¦   «         }|j        |j        |         z  }t          | j        dz   ¦  «        D ]w}	|                     t          |||¬¦  «        ¦  «         |}||j        v rE|                     t!          |¦  «        ¦  «         |                     t#          ||¦  «        ¦  «         Œxt          j        ¦   «         }
||
_        ||
_        ||
_        |dk    rt-          |¦  «        |
_        | j
                             d|
¦  «         �Œ?d S )Nr3   r!   rX  r   )rw   rx   re  rf  rg  rh  rÒ   ri  rc   rk  ÚupÚreversedrm  rn  r>  ro  rE  r  rp  rq  rr  rs  rú   ÚupsampleÚinsert)r…   rq   r?  rv  ru  rq  rr  rs  rw  rx  r  r†   s              €r1   rx   zEmu3VQVAEUpBlock.__init__Ô  s¶  ø€ Ý‰Œ×ÒÑÔÐå" 6Ô#<Ñ=Ô=ˆÔØ$Ô3ˆÔàÔ)ˆØÔ'¨&Ô*CÀBÔ*GÑGˆå”-‘/”/ˆŒÝ¥ dÔ&:Ñ ;Ô ;Ñ<Ô<ð 	"ñ 	"ˆGÝ”M‘O”OˆEÝ”=‘?”?ˆDÝœ™œˆJØÔ,¨vÔ/HÈÔ/QÑQˆIÝ  Ô!4°qÑ!8Ñ9Ô9ð Vð V�Ø—’Ý(Ø$,Ø%.Ø'5ðñ ô ñô ð ð %�Ø˜fÔ5Ð5Ð5Ø—K’KÕ 7¸Ñ ?Ô ?Ñ@Ô@Ð@Ø×%Ò%Õ&:¸>È8Ñ&TÔ&TÑUÔUÐUøå”‘”ˆBØˆBŒHØˆBŒGØ&ˆBŒMØ˜!Š|ˆ|Ý:¸8ÑDÔD�”àŒG�NŠN˜1˜bÑ!Ô!Ð!Ñ!ð3	"ð 	"r0   rH   r  c                 ó†  — t          | j        d d d…         ¦  «        D �]!\  }}t          | j        dz   ¦  «        D ]Ó} |j        |         ||¦  «        }t          |j        ¦  «        dk    r¢|} |j        |         ||¦  «        }|j        \  }}}	}
| 	                    |||	|
z  ¦  «         
                    dd¦  «        } |j        |         |¦  «        d         }|                     ||	|
|¦  «                             dddd¦  «        }||z   }ŒÔ|t          | j        ¦  «        dz
  k    r|                     |¦  «        }�Œ#|S )Nr3   r!   r   r4   r   )rz  r  rm  rh  rq  re  rr  rs  r7   rŠ   rb   rM   rÙ   r�  )r…   rH   r  ru  r{  rx  rÉ   rÜ   rÞ   rß   rà   s              r1   r’   zEmu3VQVAEUpBlock.forwardù  sZ  € Ý(¨¬°°°2°¬Ñ7Ô7ð 	?ñ 	?‰OˆG�VÝ  Ô!4°qÑ!8Ñ9Ô9ð =ð =�Ø 5 ¤¨WÔ 5°mÀ\Ñ RÔ R�Ý�v”{Ñ#Ô# aÒ'Ð'Ø,�HØ$> FÔ$5°gÔ$>¸}ÈlÑ$[Ô$[�Mà:GÔ:MÑ7�J ¨&°%Ø$1×$6Ò$6°zÀ8ÈVÐV[É^Ñ$\Ô$\×$fÒ$fÐghÐjkÑ$lÔ$l�MØ$8 F¤K°Ô$8¸Ñ$GÔ$GÈÔ$J�Mà$1×$9Ò$9¸*ÀfÈeÐU]Ñ$^Ô$^×$fÒ$fÐghÐjkÐmnÐpqÑ$rÔ$r�MØ$,¨}Ñ$<�MøØ�#˜dœg™,œ,¨Ñ*Ò*Ð*Ø &§¢°Ñ >Ô >�ùàÐr0   r^  r˜   s   @r1   r}  r}  Ó  sa   ø€ € € € € ð#"ð #"ð #"ð #"ð #"ðJ UÔ%6ð ÀeÔFWð ð ð ð ð ð ð ð r0   r}  c                   ó4   ‡ — e Zd Zˆ fd„Zdej        fd„Zˆ xZS )ÚEmu3VQVAEEncoderc                 ó¤  •— t          ¦   «                              ¦   «          |j        }|j        }|j        }|j        }|j        }|rd|z  n|}||d         z  }t          j         	                    ||ddd¬¦  «        | _
        t          |¦  «        | _        t          ||¦  «        | _        t          j                             d|dd¬	¦  «        | _        t          j         	                    ||ddd¬¦  «        | _        t%          t'          j        |j        ¦  «        ¦  «        }	t          j        ¦   «         | _        t          j        ¦   «         | _        t3          |	¦  «        D ],}
t5          ||¦  «        }| j                             |¦  «         Œ-t3          |j        ¦  «        D ]-}t;          ||¬
¦  «        }| j                             |¦  «         Œ.d S )Nr4   r3   r   r!   ré   r  rœ   T)r  r  r�   r  rd  )rw   rx   ri  rñ   Údouble_latentÚlatent_channelsrf  r,   rc   rî   Úconv_inra  Ú
down_blockrV  Úmiddle_blockr  Únorm_outÚconv_outr”   ÚmathÚlog2Útemporal_downsample_factorrk  Ú	time_convÚtime_res_stackrm  r/  rn  rh  r3  )r…   rq   ri  rñ   r‡  rˆ  rf  r  rv  Útemporal_down_blocksÚirï   rÊ   Útime_res_convr†   s                 €r1   rx   zEmu3VQVAEEncoder.__init__  sÇ  ø€ Ý‰Œ×ÒÑÔÐàÔ,ˆØÔ(ˆØÔ,ˆØ Ô0ˆØ#Ô6ÐØ.;ÐP�q˜?Ñ*Ð*ÀˆØ Ð#5°bÔ#9Ñ9ˆå”x—’ {°MÈqÐYZÐde�ÑfÔfˆŒÝ,¨VÑ4Ô4ˆŒÝ0°¸ÑBÔBˆÔåœ×*Ò*°bÀxÐUYÐbfÐ*ÑgÔgˆŒÝœŸšØØØØØð (ñ 
ô 
ˆŒõ  #¥4¤9¨VÔ-NÑ#OÔ#OÑPÔPÐÝœ™œˆŒÝ œm™oœoˆÔåÐ+Ñ,Ô,ð 	(ð 	(ˆAÝ.¨|¸\ÑJÔJˆDØŒN×!Ò! $Ñ'Ô'Ð'Ð'å�vÔ,Ñ-Ô-ð 	6ð 	6ˆAÝ8Ø(Ø)ðñ ô ˆMð Ô×&Ò& }Ñ5Ô5Ð5Ð5ð	6ð 	6r0   Úpixel_valuesc                 ót  — |j         d         } |j        dg|j         dd …         ¢R Ž }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|t          j        |¦  «        z  }|                      |¦  «        } |j        d|g|j         dd …         ¢R Ž }| 	                    ddddd¦  «        }| j
        D ]$} ||¦  «        }|t          j        |¦  «        z  }Œ%| j        D ]} ||¦  «        }Œ| 	                    ddddd¦  «        }|S )Nr!   r3   r4   r   r   rØ   )r7   rM   r‰  rŠ  r‹  rŒ  r,   r<  r�  rÙ   r‘  r’  )r…   r–  Útemporal_dimrH   rï   Úlayers         r1   r’   zEmu3VQVAEEncoder.forward5  sf  € Ø#Ô)¨!Ô,ˆØ+�|Ô+¨BÐH°Ô1CÀAÀBÀBÔ1GÐHÐHÐHˆð Ÿš \Ñ2Ô2ˆØŸš¨Ñ6Ô6ˆØ×)Ò)¨-Ñ8Ô8ˆð Ÿš mÑ4Ô4ˆØ�œ }Ñ5Ô5Ñ5ˆØŸš mÑ4Ô4ˆà-˜Ô-¨b°,ÐYÀÔATÐUVÐUWÐUWÔAXÐYÐYÐYˆØ%×-Ò-¨a°°A°q¸!Ñ<Ô<ˆð ”Nð 	:ð 	:ˆDØ ˜D Ñ/Ô/ˆMØ�Uœ]¨=Ñ9Ô9Ñ9ˆMˆMàÔ(ð 	1ð 	1ˆEØ!˜E -Ñ0Ô0ˆMˆMà%×-Ò-¨a°°A°q¸!Ñ<Ô<ˆàÐr0   )r(   r)   r*   rx   r,   r-   r’   r—   r˜   s   @r1   r…  r…    sW   ø€ € € € € ð%6ð %6ð %6ð %6ð %6ðN EÔ$4ð ð ð ð ð ð ð ð r0   r…  c                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )ÚEmu3VQVAEDecoderrq   c                 ó†  •— t          ¦   «                              ¦   «          |j        }|j        |j        d         z  }t          j        ¦   «         | _        t          |j	        ¦  «        D ]7}t          |j        |j        ¬¦  «        }| j                             |¦  «         Œ8t          t          j        |j        ¦  «        ¦  «        }t          j        ¦   «         | _        t          |¦  «        D ]6}t%          |j        |j        ¦  «        }| j                             |¦  «         Œ7t          j        |j        |ddd¬¦  «        | _        t+          |||¬¦  «        | _        t/          |¦  «        | _        |j        |j        d         z  }t3          ||¦  «        | _        t          j        ||j        ddd¬¦  «        | _        d S )Nr3   rd  r   r!   ré   )r?  r   )rw   rx   rÒ   ri  rf  rc   rk  r’  rm  rh  r3  rˆ  rn  r”   rŽ  r�  r�  r‘  r%  rî   r‰  rV  r‹  r}  Úup_blockr  rŒ  r  r�  )
r…   rq   r?  rv  rÊ   r•  Útemp_upsample_block_numr”  rï   r†   s
            €r1   rx   zEmu3VQVAEDecoder.__init__T  s®  ø€ Ý‰Œ×ÒÑÔÐàÔ)ˆØÔ'¨&Ô*CÀBÔ*GÑGˆÝ œm™oœoˆÔÝ�vÔ,Ñ-Ô-ð 	6ð 	6ˆAÝ8Ø"Ô2ÀÔAWðñ ô ˆMð Ô×&Ò& }Ñ5Ô5Ð5Ð5å"%¥d¤i°Ô0QÑ&RÔ&RÑ"SÔ"SÐÝœ™œˆŒÝÐ.Ñ/Ô/ð 	(ð 	(ˆAÝ,¨VÔ-CÀVÔE[Ñ\Ô\ˆDØŒN×!Ò! $Ñ'Ô'Ð'Ð'å”yØÔ"ØØØØð
ñ 
ô 
ˆŒõ 1°¸ÐR`ÐaÑaÔaˆÔÝ(¨Ñ0Ô0ˆŒàÔ'¨&Ô*CÀAÔ*FÑFˆÝ,¨^¸XÑFÔFˆŒÝœ	ØØÔØØØð
ñ 
ô 
ˆŒˆˆr0   rH   r  c                 óÂ  — t          j        ||fd¬¦  «        }|                     ddddd¦  «        }| j        D ]} ||¦  «        }Œ| j        D ]$} ||¦  «        }|t          j        |¦  «        z  }Œ%|                     ddddd¦  «        }t          j        |dd¬¦  «        \  }} |j        dg|j        dd …         ¢R Ž } |j        dg|j        dd …         ¢R Ž }|  	                    |¦  «        }|  
                    ||¦  «        }|                      ||¦  «        }|                      ||¦  «        }|t          j        |¦  «        z  }|                      |¦  «        }|S )Nr   r5   r4   r!   r   rØ   r3   )r,   r8   rÙ   r’  r‘  r<  ÚchunkrM   r7   r‰  r‹  r�  rŒ  r�  )r…   rH   r  Úhidden_quant_statesr™  s        r1   r’   zEmu3VQVAEDecoder.forward{  sš  € Ý#œi¨¸Ð(EÈ1ÐMÑMÔMÐØ1×9Ò9¸!¸QÀÀ1ÀaÑHÔHÐð Ô(ð 	=ð 	=ˆEØ"' %Ð(;Ñ"<Ô"<ÐÐà”^ð 	Fð 	FˆEØ"' %Ð(;Ñ"<Ô"<ÐØ¥5¤=Ð1DÑ#EÔ#EÑEÐÐà1×9Ò9¸!¸QÀÀ1ÀaÑHÔHÐÝ&+¤kÐ2EÀqÈaÐ&PÑ&PÔ&PÑ#ˆ�|Ø-˜Ô-¨bÐK°=Ô3FÀqÀrÀrÔ3JÐKÐKÐKˆØ+�|Ô+¨BÐH°Ô1CÀAÀBÀBÔ1GÐHÐHÐHˆàŸš ]Ñ3Ô3ˆð ×)Ò)¨-¸ÑFÔFˆØŸš m°\ÑBÔBˆàŸš m°\ÑBÔBˆØ�œ }Ñ5Ô5Ñ5ˆØŸš mÑ4Ô4ˆàÐr0   )	r(   r)   r*   r$   rx   r,   r•   r’   r—   r˜   s   @r1   r›  r›  S  sk   ø€ € € € € ð%
˜ð %
ð %
ð %
ð %
ð %
ð %
ðN U¤\ð ÀÄð ð ð ð ð ð ð ð r0   r›  aR  
    The VQ-VAE model used in Emu3 for encoding/decoding images into discrete tokens.
    This model follows the "Make-a-scene: Scene-based text-to-image generation with human priors" paper from
    [ Oran Gafni, Adam Polyak, Oron Ashual, Shelly Sheynin, Devi Parikh, and Yaniv
    Taigman](https://huggingface.co/papers/2203.13131).
    ©Úcustom_introc            
       ó  ‡ — e Zd ZU eed<   dZdZdZdZdZ	dZ
dZg d¢ZeegedœZ ej        ¦   «         ˆ fd„¦   «         Zdefˆ fd	„Zeedej        d
ej        dee         defd„¦   «         ¦   «         Zdej        fd„Zˆ xZS )Ú	Emu3VQVAErq   Ú
emuvideovqr–  )ÚimageT)r3  rE  r>  rÍ   ©rH   Ú
attentionsc                 ó  •— t          ¦   «                              |¦  «         t          |t          j        t          j        f¦  «        r†t          j        |j        dd¬¦  «         |j	        �at          j        j                             |j        ¦  «        \  }}dt          j        |¦  «        z  }t          j        |j	        | |¦  «         d S d S t          |t          j        ¦  «        rŸt          j        |j        t          j        d¦  «        ¬¦  «         |j	        �it          j        j                             |j        ¦  «        \  }}|dk    rdt          j        |¦  «        z  nd}t          j        |j	        | |¦  «         d S d S t          |t          j        ¦  «        r\t          j        |j        ¦  «         |j        �>t)          |j        dd	¦  «        s*t          j        |j        |j                 ¦  «         d S d S d S d S )
NÚfan_outÚrelu)rö   Únonlinearityr!   é   )Úar   Ú_is_hf_initializedF)rw   Ú_init_weightsÚ
isinstancerc   rî   r  ÚinitÚkaiming_normal_r¡   rv   r,   Ú_calculate_fan_in_and_fan_outrŽ  ÚsqrtrÕ   r~   Úkaiming_uniform_rÐ   Únormal_Úpadding_idxry   Úzeros_)r…   rT   Úfan_inrÊ   Úboundr†   s        €r1   r±  zEmu3VQVAE._init_weightsµ  sÏ  ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�f�rœy­"¬)Ð4Ñ5Ô5ð 	?ÝÔ  ¤°YÈVÐTÑTÔTÐTØŒ{Ð&Ý!œHœM×GÒGÈÌÑVÔV‘	�˜Ø�DœI fÑ-Ô-Ñ-�Ý”˜fœk¨E¨6°5Ñ9Ô9Ð9Ð9Ð9ð 'Ð&õ ˜¥¤	Ñ*Ô*ð 
	?ÝÔ! &¤-µ4´9¸Q±<´<Ð@Ñ@Ô@Ð@ØŒ{Ð&Ý!œHœM×GÒGÈÌÑVÔV‘	�˜Ø17¸!²°˜�DœI fÑ-Ô-Ñ-Ð-À�Ý”˜fœk¨E¨6°5Ñ9Ô9Ð9Ð9Ð9ð 'Ð&õ ˜¥¤Ñ-Ô-ð 	?ÝŒL˜œÑ'Ô'Ð'àÔ!Ð-µg¸f¼mÐMaÐchÑ6iÔ6iÐ-Ý”˜FœM¨&Ô*<Ô=Ñ>Ô>Ð>Ð>Ð>ð		?ð 	?ð .Ð-Ð-Ð-r0   c                 ó$  •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        t          |¦  «        | _        t          |¦  «        | _        dt          |j
        ¦  «        dz
  z  | _        t          |j        |j        dd¬¦  «        | _        t          |j        |j        dd¬¦  «        | _        dt          |j
        ¦  «        dz
  z  | _        |                      ¦   «          |                      ¦   «          d S )Nr4   r!   )r   r!   r!   r(  r)  )rw   rx   rq   r…  Úencoderr›  ÚdecoderrÍ   Úquantizere  rf  Úvision_spatial_factorr  rˆ  rÒ   Ú
quant_convÚpost_quant_convÚspatial_scale_factorÚevalÚ	post_initr¸   s     €r1   rx   zEmu3VQVAE.__init__Ê  sý   ø€ Ý‰Œ×Ò˜Ñ Ô Ð àˆŒå'¨Ñ/Ô/ˆŒÝ'¨Ñ/Ô/ˆŒÝ0°Ñ8Ô8ˆŒØ%&­3¨vÔ/HÑ+IÔ+IÈAÑ+MÑ%NˆÔ"å)ØÔ" FÔ$4À)ÐT]ð
ñ 
ô 
ˆŒõ  /ØÔ˜fÔ4À)ÐT]ð 
ñ  
ô  
ˆÔð %&­#¨fÔ.GÑ*HÔ*HÈ1Ñ*LÑ$MˆÔ!Ø�	Š	‰Œˆà�ŠÑÔÐÐÐr0   Úimage_sizesr[   rJ   c                 ó0  ‡ — |j         dk    }|rE‰ j        j        }|j        \  }}}}	|                     d¦  «                             d|ddd¦  «        }n|j        \  }}}}}	‰                      |¦  «        }
|
                     ddddd¦  «        }‰                      |¦  «        }|                     ddddd¦  «        }‰  	                    |¦  «        }|r| 
                    d¦  «        n|}ˆ fd„t          ||¦  «        D ¦   «         }t          |
|¬¦  «        S )NrØ   r!   r   r4   r   c           	      óž   •— g | ]I\  }}|d t          |d         ‰j        z  ¦  «        …d t          |d         ‰j        z  ¦  «        …f         ‘ŒJS )Nr   r!   )r”   rÁ  )r  Úsingle_imager"  r…   s      €r1   r  z$Emu3VQVAE.encode.<locals>.<listcomp>ø  sm   ø€ ð 
ð 
ð 
á"�˜dð ÐD�3˜t Aœw¨Ô)CÑCÑDÔDÐDÐFqÍÈDÐQRÌGÐVZÔVpÑLpÑHqÔHqÐFqÐqÔrð
ð 
ð 
r0   )Úlast_hidden_stater'   )Úndimrq   r�  r7   r?   Úrepeatr¾  rÙ   rÂ  rÀ  Úsqueezer  r&   )r…   r–  rÇ  r[   Úis_imagerÝ   rÜ   rÞ   rß   rà   rH   Úconv_hidden_statesÚcodesr'   s   `             r1   ÚencodezEmu3VQVAE.encodeß  sR  ø€ ð
  Ô$¨Ò)ˆØð 	OØ”{Ô=ˆHØ2>Ô2DÑ/ˆJ˜ &¨%Ø'×1Ò1°!Ñ4Ô4×;Ò;¸A¸xÈÈAÈqÑQÔQˆLˆLà<HÔ<NÑ9ˆJ˜ (¨F°EàŸš \Ñ2Ô2ˆð +×2Ò2°1°a¸¸A¸qÑAÔAÐØ!Ÿ_š_Ð-?Ñ@Ô@Ðð 0×7Ò7¸¸1¸aÀÀAÑFÔFÐØ—’Ð0Ñ1Ô1ˆà+3Ð>�u—}’} QÑ'Ô'Ð'¸ˆð
ð 
ð 
ð 
å&)¨,¸Ñ&DÔ&Dð
ñ 
ô 
ˆõ
 $Ø+Ø%ð
ñ 
ô 
ð 	
r0   rH   c                 ó®  — |j         dk    }|r|                     d¦  «        }|j        \  }}}}| j                             |                     ¦   «         ¦  «        }|j        d         }|                     |||||¦  «                             ddddd¦  «                             ¦   «         }|  	                    |¦  «        }	|                     ddddd¦  «        }|	                     ddddd¦  «        }	|  
                    |	|¦  «        }
|
                     ||| j        j        z  | j        j        || j        z  || j        z  ¦  «        }
|r|
d d …df         n|
S )Nr   r!   r3   r   rØ   r4   )rÌ  r?   r7   rÀ  rÓ   ÚflattenrŠ   rÙ   rh   rÃ  r¿  rM   rq   r�  r  rÄ  )r…   rH   rÏ  rÜ   rÝ   rß   rà   ÚquantrÞ   Ú
post_quantÚvideos              r1   ÚdecodezEmu3VQVAE.decode  s_  € Ø Ô%¨Ò*ˆØð 	7Ø)×3Ò3°AÑ6Ô6ˆMà.;Ô.AÑ+ˆ
�H˜f eØ”×'Ò'¨×(=Ò(=Ñ(?Ô(?Ñ@Ô@ˆà”;˜r”?ˆØ—
’
˜: x°¸ÀÑIÔI×QÒQÐRSÐUVÐXYÐ[\Ð^_Ñ`Ô`×kÒkÑmÔmˆØ×)Ò)¨%Ñ0Ô0ˆ
à—’˜a  A q¨!Ñ,Ô,ˆØ×'Ò'¨¨1¨a°°AÑ6Ô6ˆ
à—’˜Z¨Ñ/Ô/ˆØ—’ØØ�t”{Ô=Ñ=ØŒKÔ$Ø�TÔ.Ñ.Ø�DÔ-Ñ-ñ
ô 
ˆð 'Ð1ˆu�Q�Q�Q˜�TŒ{ˆ{¨EÐ1r0   )r(   r)   r*   r$   r.   Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚ_supports_sdpaÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_attention_backendÚ_no_split_modulesr>  r3  rE  Ú_can_record_outputsr,   Úno_gradr±  rx   r   r    r•   r   r   r&   rÒ  rØ  r—   r˜   s   @r1   r¥  r¥  ™  sI  ø€ € € € € € ð ÐÐÑØ$ÐØ$€OØ!ÐØ€NØÐØÐØ"&Ððð ð Ðð /Ð0LÐMØ-ðð Ðð
 €U„]�_„_ð?ð ?ð ?ð ?ñ „_ð?ð(˜ð ð ð ð ð ð ð*  Øð
Ø!œLð
Ø7<´|ð
ØOUÐVhÔOið
à	ð
ð 
ð 
ñ „_ñ  Ôð
ðB2 E¤Lð 2ð 2ð 2ð 2ð 2ð 2ð 2ð 2r0   r¥  c                   óð   — e Zd ZdZd„ Zed„ ¦   «         Zed„ ¦   «         Zed„ ¦   «         Zed„ ¦   «         Z	ed„ ¦   «         Z
ed„ ¦   «         Zd	eej                 d
ej        fd„Zd	ej        d
ej        fd„ZdS )ÚEmu3ImageVocabularyMappingzM
    A class for mapping discrete image tokens from VQGAN to BPE tokens.
    c                 ó|   — || _         |                     d¦  «        | _        |                     d¦  «        | _        d S )Nz<|extra_200|>z<image>)Ú	vocab_mapÚgetÚeol_token_idÚimage_token_id)r…   ræ  s     r1   rx   z#Emu3ImageVocabularyMapping.__init__!  s7   € Ø"ˆŒØ%ŸMšM¨/Ñ:Ô:ˆÔØ'Ÿmšm¨IÑ6Ô6ˆÔÐÐr0   c                 ób   — t          d„ | j                             ¦   «         D ¦   «         ¦  «        S )Nc                 óB   — g | ]\  }}|                      d ¦  «        ¯|‘ŒS ©z<|visual token©Ú
startswith©r  ÚnameÚvals      r1   r  z;Emu3ImageVocabularyMapping.image_tokens.<locals>.<listcomp>(  s.   € ÐhÐhÐh™y˜t SÀdÇoÂoÐVfÑFgÔFgÐh�sÐhÐhÐhr0   ©Úsortedræ  Úitemsr«   s    r1   r'   z'Emu3ImageVocabularyMapping.image_tokens&  s-   € åÐhÐh¨D¬N×,@Ò,@Ñ,BÔ,BÐhÑhÔhÑiÔiÐir0   c                 ób   — t          d„ | j                             ¦   «         D ¦   «         ¦  «        S )Nc                 óB   — g | ]\  }}|                      d ¦  «        ¯|‘ŒS rì  rí  rï  s      r1   r  z?Emu3ImageVocabularyMapping.image_tokens_str.<locals>.<listcomp>,  s.   € ÐiÐiÐi¡	  cÀtÇÂÐWgÑGhÔGhÐi�tÐiÐiÐir0   rò  r«   s    r1   Úimage_tokens_strz+Emu3ImageVocabularyMapping.image_tokens_str*  s-   € åÐiÐi¨T¬^×-AÒ-AÑ-CÔ-CÐiÑiÔiÑjÔjÐjr0   c                 ó*   ‡ — ˆ fd„‰ j         D ¦   «         S )Nc                 óV   •— i | ]%}t          |d d…         ¦  «        ‰j        |         “Œ&S )iøÿÿÿr!  )r”   ræ  )r  Útokenr…   s     €r1   ú
<dictcomp>z6Emu3ImageVocabularyMapping.img2bpe.<locals>.<dictcomp>0  s2   ø€ Ð\Ð\Ð\¸U•�E˜"˜R˜%”LÑ!Ô! 4¤>°%Ô#8Ð\Ð\Ð\r0   )r÷  r«   s   `r1   Úimg2bpez"Emu3ImageVocabularyMapping.img2bpe.  s    ø€ à\Ð\Ð\Ð\ÀdÔF[Ð\Ñ\Ô\Ð\r0   c                 óH   — d„ | j                              ¦   «         D ¦   «         S )Nc                 ó   — i | ]\  }}||“Œ	S r/   r/   )r  rA   Úvs      r1   rû  z6Emu3ImageVocabularyMapping.bpe2img.<locals>.<dictcomp>4  s   € Ð6Ð6Ð6™˜˜A��1Ð6Ð6Ð6r0   )rü  rô  r«   s    r1   Úbpe2imgz"Emu3ImageVocabularyMapping.bpe2img2  s$   € à6Ð6 ¤×!3Ò!3Ñ!5Ô!5Ð6Ñ6Ô6Ð6r0   c                 óÜ   — t          j        t          | j                             ¦   «         ¦  «        dz   t           j        ¬¦  «        }| j                             ¦   «         D ]
\  }}|||<   Œ|S ©Nr!   ©r]   )r,   ÚzerosÚmaxr   rM  r”   rô  ©r…   ÚmappingrA   rÿ  s       r1   Úbpe2img_mapping_tensorz1Emu3ImageVocabularyMapping.bpe2img_mapping_tensor6  ód   € å”+�c $¤,×"3Ò"3Ñ"5Ô"5Ñ6Ô6¸Ñ:Å%Ä)ÐLÑLÔLˆØ”L×&Ò&Ñ(Ô(ð 	ð 	‰DˆAˆqØˆG�A‰JˆJØˆr0   c                 óÜ   — t          j        t          | j                             ¦   «         ¦  «        dz   t           j        ¬¦  «        }| j                             ¦   «         D ]
\  }}|||<   Œ|S r  )r,   r  r  rü  rM  r”   rô  r  s       r1   Úimg2bpe_mapping_tensorz1Emu3ImageVocabularyMapping.img2bpe_mapping_tensor=  r	  r0   Ú	img_batchrJ   c                 ó  — |j         }t          j        |j        d         dft          j        ¬¦  «        | j        z  }| j        |                     d¦  «                 }t          j        ||gd¬¦  «        }|                     |¦  «        S )Nr   r!   r  Úcpur3   r5   )	Údevicer,   r    r7   r”   rè  r  rg   r8   )r…   r  r  Úeol_rowÚ
img_tokenss        r1   Úconvert_img2bpez*Emu3ImageVocabularyMapping.convert_img2bpeD  sx   € ØÔ!ˆÝ”*˜iœo¨aÔ0°!Ð4½E¼IÐFÑFÔFÈÔIZÑZˆØÔ0°·²¸eÑ1DÔ1DÔEˆ
Ý”Y 
¨GÐ4¸"Ð=Ñ=Ô=ˆ
Ø�}Š}˜VÑ$Ô$Ð$r0   c                 ó’   — |j         }|dd d…f         }| j        |                     d¦  «                 }|                     |¦  «        S )N.r3   r  )r  r  rg   )r…   r  r  r  s       r1   Úconvert_bpe2imgz*Emu3ImageVocabularyMapping.convert_bpe2imgK  sG   € ØÔ!ˆØ˜c 3 B 3˜hÔ'ˆ	ØÔ0°·²¸eÑ1DÔ1DÔEˆ
Ø�}Š}˜VÑ$Ô$Ð$r0   N)r(   r)   r*   r+   rx   r   r'   r÷  rü  r   r  r  Úlistr,   r•   r  r  r/   r0   r1   rä  rä    s)  € € € € € ðð ð7ð 7ð 7ð
 ðjð jñ „_ðjð ðkð kñ „_ðkð ð]ð ]ñ „_ð]ð ð7ð 7ñ „_ð7ð ðð ñ „_ðð ðð ñ „_ðð%¨¨e¬lÔ);ð %ÀÄð %ð %ð %ð %ð%¨¬ð %¸%¼,ð %ð %ð %ð %ð %ð %r0   rä  c                   óR   — e Zd ZU eed<   dZdZdZdgZddgZ	dZ
dZdZdZdZeedœZd	S )
ÚEmu3PreTrainedModelrq   Úmodel©r§  ÚtextTr½   rˆ   Úcausal_maskr¨  N)r(   r)   r*   r"   r.   rÙ  rÛ  Úsupports_gradient_checkpointingrà  Ú_skip_keys_device_placementrÝ  rÜ  Ú_can_compile_fullgraphrÞ  rß  r½   ro   rá  r/   r0   r1   r  r  R  sx   € € € € € € àÐÐÑØÐØ(ÐØ&*Ð#àðÐð $5°mÐ"DÐØÐØ€Nà!ÐØÐØ"&Ðà)Ø#ðð ÐÐÐr0   r  c                   óÔ   ‡ — e Zd ZU ej        ed<   ddefˆ fd„Ze	 	 	 ddedz  de	d         de
dz  ded	ef         fd
„¦   «         Z ej        ¦   «         ed„ ¦   «         ¦   «         Zˆ xZS )ÚEmu3RotaryEmbeddingÚinv_freqNrq   c                 ó²  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        | j        j        d         | _        | j        }| j        dk    rt          | j                 } || j        |¦  «        \  }| _
        |                      d|d¬¦  «         |                      d|                     ¦   «         d¬¦  «         d S )NÚ	rope_typeÚdefaultr!  F)Ú
persistentÚoriginal_inv_freq)rw   rx   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrq   Úrope_parametersr#  Úcompute_default_rope_parametersr   Úattention_scalingÚregister_bufferÚclone)r…   rq   r  Úrope_init_fnr!  r†   s        €r1   rx   zEmu3RotaryEmbedding.__init__k  sÊ   ø€ Ý‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!àˆŒàœÔ4°[ÔAˆŒØ!%Ô!EˆØŒ>˜YÒ&Ð&Ý.¨t¬~Ô>ˆLØ+7¨<¸¼ÀVÑ+LÔ+LÑ(ˆ�$Ô(à×Ò˜Z¨¸eÐÑDÔDÐDØ×ÒÐ0°(·.².Ñ2BÔ2BÈuÐÑUÔUÐUÐUÐUr0   r  ztorch.deviceÚseq_lenrJ   ztorch.Tensorc                 óü   — | j         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «                             |t          j        ¬¦  «        |z  z  z  }||fS )	a¨  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetarQ   NrÏ   r   r4   r  )r  r]   )	r*  ry   rz   r{   r,   ÚarangeÚint64rg   r­   )rq   r  r0  Úbaser6   Úattention_factorr!  s          r1   r+  z3Emu3RotaryEmbedding.compute_default_rope_parameters{  sŒ   € ð& Ô% lÔ3ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐð Ø•U”\ ! S¨!µ5´;Ð?Ñ?Ô?×BÒBÈ&ÕX]ÔXcÐBÑdÔdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r0   c                 óN  — | j         d d d …d f                              ¦   «                              |j        d         dd¦  «                             |j        ¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  |                     ¦   «         |                     ¦   «         z   
                    dd¦  «        }t          j        ||fd¬	¦  «        }|                     ¦   «         | j        z  }|                     ¦   «         | j        z  }	d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬
¦  «        |	                     |j        ¬
¦  «        fS )Nr   r3   r!   Úmpsr  F)Údevice_typeÚenabledr4   r5   r  )r!  r­   rL   r7   rg   r  r²  ÚtypeÚstrr   rb   r,   r8   rB   r,  rC   r]   )
r…   r9   rÆ   Úinv_freq_expandedÚposition_ids_expandedr9  ÚfreqsÚembrB   rC   s
             r1   r’   zEmu3RotaryEmbedding.forward™  s·  € ð !œM¨$°°°°4¨-Ô8×>Ò>Ñ@Ô@×GÒGÈÔHZÐ[\ÔH]Ð_aÐcdÑeÔe×hÒhÐijÔiqÑrÔrÐØ ,¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	5ð 	5Ø&×,Ò,Ñ.Ô.Ð1F×1LÒ1LÑ1NÔ1NÑN×YÒYÐZ[Ð]^Ñ_Ô_ˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCð		5ð 	5ð 	5ñ 	5ô 	5ð 	5ð 	5ð 	5ð 	5ð 	5ð 	5øøøð 	5ð 	5ð 	5ð 	5ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   ÃBE&Å&E*Å-E*rº   r“   )r(   r)   r*   r,   r•   r.   r"   rx   Ústaticmethodr   r”   r–   r­   r+  râ  r   r’   r—   r˜   s   @r1   r   r   h  sù   ø€ € € € € € ØŒlÐÐÑðVð V˜zð Vð Vð Vð Vð Vð Vð  à$(Ø+/Ø"ð*ð *Ø˜TÑ!ð*à˜Ô(ð*ð �t‘ð*ð 
ˆ~˜uÐ$Ô	%ð	*ð *ð *ñ „\ð*ð: €U„]�_„_Øð<ð <ñ Ôñ „_ð<ð <ð <ð <ð <r0   r   c                   óî   ‡ — e Zd ZU eed<   defˆ fd„Zeee	 	 	 	 	 	 dde	j
        dz  de	j        dz  de	j
        dz  dedz  de	j        dz  d	edz  d
ee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚEmu3TextModelrq   c                 óÞ  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          j        ‰j        ‰j        | j        ¦  «        | _        t          j	        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        t!          ‰¬¦  «        | _        d| _        |                      ¦   «          d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r/   )r½   )r  rr   rq   s     €r1   r  z*Emu3TextModel.__init__.<locals>.<listcomp>´  s$   ø€ ÐbÐbÐb°YÕ˜f iÑ0Ô0ÐbÐbÐbr0   r¿   ©rq   F)rw   rx   Úpad_token_idr¹  Ú
vocab_sizerc   rÐ   rz   Úembed_tokensrk  rm  Únum_hidden_layersÚlayersr›   rÂ   Únormr   Ú
rotary_embÚgradient_checkpointingrÆ  r¸   s    `€r1   rx   zEmu3TextModel.__init__­  sÔ   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒåœL¨Ô):¸FÔ<NÐPTÔP`ÑaÔaˆÔÝ”mØbÐbÐbÐbÅ%ÈÔH`ÑBaÔBaÐbÑbÔbñ
ô 
ˆŒõ   Ô 2¸Ô8KÐLÑLÔLˆŒ	Ý-°VÐ<Ñ<Ô<ˆŒØ&+ˆÔ#ð 	�ŠÑÔÐÐÐr0   NÚ	input_idsrX   rÆ   rˆ   Úinputs_embedsrÇ   r[   rJ   c           
      óH  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }| 	                    d¦  «        }t          | j        ||||¬¦  «        }	|}
|                      |
|¬¦  «        }| j        d | j        j        …         D ]} ||
f|	||||dœ|¤Ž}
Œ|                      |
¦  «        }
t          |
|¬	¦  «        S )
Nz:You must specify exactly one of input_ids or inputs_embedsrF  r   r!   )r  )rq   rP  rX   rˆ   rÆ   )rÆ   )rX   r‡   rÆ   rˆ   rÇ   )rË  rˆ   )rH  rI  r   rq   Úget_seq_lengthr,   r3  r7   r  r?   r   rM  rK  rJ  rL  r   )r…   rO  rX   rÆ   rˆ   rP  rÇ   r[   Úpast_seen_tokensr  rH   r‡   Údecoder_layers                r1   r’   zEmu3TextModel.forward½  sŠ  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø*.×*;Ò*;¸IÑ*FÔ*FˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå(Ø”;Ø'Ø)Ø+Ø%ð
ñ 
ô 
ˆð &ˆØ"Ÿošo¨mÈ,˜oÑWÔWÐà!œ[Ð)H¨4¬;Ô+HÐ)HÔIð 		ð 		ˆMØ)˜MØðà*Ø$7Ø)Ø /Ø#ðð ð ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆÝ&Ø+Ø+ð
ñ 
ô 
ð 	
r0   )NNNNNN)r(   r)   r*   r#   r.   rx   r   r    r   r,   r-   r•   r
   r_  rË   r   r   r   r’   r—   r˜   s   @r1   rC  rC  ©  s  ø€ € € € € € àÐÐÑð˜~ð ð ð ð ð ð ð   ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ð2
ð 2
àÔ# dÑ*ð2
ð œ tÑ+ð2
ð Ô&¨Ñ-ð	2
ð
  ™ð2
ð Ô(¨4Ñ/ð2
ð ˜$‘;ð2
ð Ð+Ô,ð2
ð 
!ð2
ð 2
ð 2
ñ „^ñ „_ñ  Ôð2
ð 2
ð 2
ð 2
ð 2
r0   rC  c                   ó$  ‡ — e Zd ZU ddiZddiZddgdgfiZeed<   ˆ fd„Ze	e
	 	 	 	 	 	 	 	 ddej        d	z  dej        d	z  dej        d	z  ded	z  dej        d	z  dej        d	z  ded	z  deej        z  dee         defd„¦   «         ¦   «         Zˆ xZS )ÚEmu3ForCausalLMúlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputrH   Úlogitsrq   c                 óþ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        | _        t          j        |j        |j        d¬¦  «        | _        |  	                    ¦   «          d S ©NFru   )
rw   rx   rC  r  rH  rc   r~   rz   rX  rÆ  r¸   s     €r1   rx   zEmu3ForCausalLM.__init__ü  sj   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý" 6Ñ*Ô*ˆŒ
Ø Ô+ˆŒÝ”y Ô!3°VÔ5FÈUÐSÑSÔSˆŒð 	�ŠÑÔÐÐÐr0   Nr   rO  rX   rÆ   rˆ   rP  ÚlabelsrÇ   Úlogits_to_keepr[   rJ   c	           
      óP  —  | j         d||||||dœ|	¤Ž}
|
j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }d}|� | j        d||| j        j        dœ|	¤Ž}t          |||
j
        |
j        |
j        ¬¦  «        S )aÃ  
        Example:

        ```python
        >>> from transformers import Emu3Processor, Emu3ForConditionalGeneration
        >>> import torch
        >>> import httpx
        >>> from io import BytesIO
        >>> from PIL import Image

        >>> model = Emu3ForCausalLM.from_pretrained("BAAI/Emu3-Chat-hf", dtype=torch.bfloat16)
        >>> processor = Emu3Processor.from_pretrained("BAAI/Emu3-Chat-hf")

        >>> inputs = processor(text=["Can you write me a poem about winter."], return_tensors="pt").to(model.device)

        >>> generated_ids = model.generate(**inputs, max_new_tokens=100, do_sample=False)
        >>> processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        ```©rO  rX   rÆ   rˆ   rP  rÇ   N©rZ  r]  rH  ©ÚlossrZ  rˆ   rH   r©  r/   )r  rË  r²  r”   ÚslicerX  Úloss_functionrq   rH  r   rˆ   rH   r©  )r…   rO  rX   rÆ   rˆ   rP  r]  rÇ   r^  r[   ÚoutputsrH   Úslice_indicesrZ  rc  s                  r1   r’   zEmu3ForCausalLM.forward  sõ   € ð@ ,6¨4¬:ð ,
ØØ)Ø%Ø+Ø'Øð,
ð ,
ð ð,
ð ,
ˆð  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%Ðp¨V¸FÈtÌ{ÔOeÐpÐpÐioÐpÐpˆDå%ØØØ#Ô3Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r0   )NNNNNNNr   )r(   r)   r*   Ú_tied_weights_keysÚ_tp_planÚ_pp_planr#   r.   rx   r   r   r,   r-   r•   r
   r_  rË   r”   r   r   r   r’   r—   r˜   s   @r1   rV  rV  õ  sY  ø€ € € € € € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐÐÑðð ð ð ð ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ð7
ð 7
àÔ# dÑ*ð7
ð œ tÑ+ð7
ð Ô&¨Ñ-ð	7
ð
  ™ð7
ð Ô(¨4Ñ/ð7
ð Ô  4Ñ'ð7
ð ˜$‘;ð7
ð ˜eœlÑ*ð7
ð Ð+Ô,ð7
ð 
 ð7
ð 7
ð 7
ñ „^ñ Ôð7
ð 7
ð 7
ð 7
ð 7
r0   rV  c                   ó(  ‡ — e Zd Zˆ fd„Zd„ Zd„ Zdej        dej        dej        fd„Z	e
 ed¬	¦  «        dej        dej        d
ee         deez  fd„¦   «         ¦   «         Z ej        ¦   «         dej        dedefd„¦   «         Zdej        dej        dej        fd„Ze
e	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dedz  dej        dz  dedz  d
ee         deez  fd„¦   «         ¦   «         Zˆ xZS )Ú	Emu3Modelc                 ó  •— t          ¦   «                              |¦  «         t                               |j        ¦  «        | _        t          |j        ¦  «        | _        t          |j
        ¦  «        | _        |                      ¦   «          d S rº   )rw   rx   rC  Ú_from_configÚtext_configÚ
text_modelr¥  Ú	vq_configÚvqmodelrä  Úvocabulary_mapÚvocabulary_mappingrÆ  r¸   s     €r1   rx   zEmu3Model.__init__B  sp   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý'×4Ò4°VÔ5GÑHÔHˆŒÝ  Ô!1Ñ2Ô2ˆŒÝ"<¸VÔ=RÑ"SÔ"SˆÔð 	�ŠÑÔÐÐÐr0   c                 ó4   — | j                              ¦   «         S rº   )rp  Úget_input_embeddingsr«   s    r1   rv  zEmu3Model.get_input_embeddingsK  s   € ØŒ×3Ò3Ñ5Ô5Ð5r0   c                 ó:   — | j                              |¦  «         d S rº   )rp  Úset_input_embeddings©r…   rW   s     r1   rx  zEmu3Model.set_input_embeddingsN  s   € ØŒ×,Ò,¨UÑ3Ô3Ð3Ð3Ð3r0   r–  rÇ  rJ   c                 ó�   ‡ — ‰ j                              ||d¬¦  «        }ˆ fd„|j        D ¦   «         }t          j        |¦  «        }|S )a  
        Tokenizes images into discrete tokens with VQGAN module. Converts
        obtained image tokens into BPE tokens and wraps with "boi" and "eoi"
        special tokens.

        Args:
            pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
                The tensors corresponding to the input images.
            image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`):
                The sizes of the images in the batch, being (height, width) for each image.
        T)Úreturn_dictc                 óh   •— g | ].}‰j                              |¦  «                             ¦   «         ‘Œ/S r/   ©rt  r  rÔ  ©r  Útokensr…   s     €r1   r  z.Emu3Model.get_image_tokens.<locals>.<listcomp>^  óC   ø€ ð 
ð 
ð 
ØJPˆDÔ#×3Ò3°FÑ;Ô;×CÒCÑEÔEð
ð 
ð 
r0   )rr  rÒ  r'   r,   r8   )r…   r–  rÇ  Úvqmodel_outputsÚbpe_tokens_listÚ
bpe_tokenss   `     r1   Úget_image_tokenszEmu3Model.get_image_tokensQ  se   ø€ ð 15´×0CÒ0CÀLÐR]ÐkoÐ0CÑ0pÔ0pˆð
ð 
ð 
ð 
ØTcÔTpð
ñ 
ô 
ˆõ ”Y˜Ñ/Ô/ˆ
ØÐr0   zbTokenizes images into discrete tokens with VQGAN module and embeds them with text embeddings layerr¢  r[   c                 ó  ‡ —  ‰ j         j        ||fddi|¤Ž}ˆ fd„|D ¦   «         }ˆ fd„|j        D ¦   «         }t          j        |¦  «        } ‰                      ¦   «         |¦  «        }t          j        ||¦  «        }	|	|_        |S )z­
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)):
            The tensors corresponding to the input images.
        r{  Tc                 óZ   •— g | ]'\  }}|‰j         j        z  |‰j         j        z  d z   z  ‘Œ(S rc  )rr  rÁ  )r  rß   rà   r…   s      €r1   r  z0Emu3Model.get_image_features.<locals>.<listcomp>r  sL   ø€ ð 
ð 
ð 
á�˜ð �t”|Ô9Ñ9¸eÀtÄ|ÔGiÑ>iÐlmÑ>mÑnð
ð 
ð 
r0   c                 óh   •— g | ].}‰j                              |¦  «                             ¦   «         ‘Œ/S r/   r}  r~  s     €r1   r  z0Emu3Model.get_image_features.<locals>.<listcomp>v  r€  r0   )rr  rÒ  r'   r,   r8   rv  ÚsplitÚpooler_output)
r…   r–  rÇ  r[   r�  Úsplit_sizesr‚  rƒ  Úimage_embeddingsÚimage_featuress
   `         r1   Úget_image_featureszEmu3Model.get_image_featuresd  sÖ   ø€ ð 1D°´Ô0CØ˜+ð1
ð 1
Ø37ð1
Ø;Að1
ð 1
ˆð
ð 
ð 
ð 
à!,ð
ñ 
ô 
ˆð
ð 
ð 
ð 
ØTcÔTpð
ñ 
ô 
ˆõ ”Y˜Ñ/Ô/ˆ
Ø6˜4×4Ò4Ñ6Ô6°zÑBÔBÐÝœÐ%5°{ÑCÔCˆØ(6ˆÔ%àÐr0   r'   rß   rà   c                 óº   — |dd…dd…f                               d||dz   ¦  «        }| j                             |¦  «        }| j                             |¦  «        }|S )aå  
        Decodes generated image tokens from language model to continuous pixel values
        with VQGAN module via upsampling.

        Args:
            image_tokens (`torch.LongTensor` of shape `(batch_size, num_of_tokens)`):
                The tensors corresponding to the input images.
            height (`int`):
                Height of the generated image before upsampling.
            width (`int`):
                Width of the generated image before upsampling.
        Néýÿÿÿr3   r!   )rŠ   rt  r  rr  rØ  )r…   r'   rß   rà   Ú	sequencesr§  s         r1   Údecode_image_tokenszEmu3Model.decode_image_tokens€  s`   € ð !    C R C Ô(×-Ò-¨b°&¸%À!¹)ÑDÔDˆ	ØÔ.×>Ò>¸yÑIÔIˆØ”×#Ò# LÑ1Ô1ˆØˆr0   rO  rP  rŒ  c                 ó   — |€e| |                       ¦   «         t          j        | j        j        t          j        |j        ¬¦  «        ¦  «        k    }|                     d¦  «        }n|| j        j        k    }|                     ¦   «         }|j	        d         |j	        d         z  }| 
                    d¦  «                             |j        ¦  «        }t          ||j	        d         z  |                     ¦   «         k    d|› d|› �¦  «         |S )zï
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        N)r]   r  r3   r   r!   z6Image features and image tokens do not match, tokens: z, features: )rv  r,   Útensorrt  ré  Úlongr  ÚallrÚ   r7   r?   rg   r   Únumel)r…   rO  rP  rŒ  Úspecial_image_maskÚn_image_tokensÚn_image_featuress          r1   Úget_placeholder_maskzEmu3Model.get_placeholder_mask“  s  € ð ÐØ!.Ð2M°$×2KÒ2KÑ2MÔ2MÝ”˜TÔ4ÔCÍ5Ì:Ð^kÔ^rÐsÑsÔsñ3ô 3ò "Ðð "4×!7Ò!7¸Ñ!;Ô!;ÐÐà!*¨dÔ.EÔ.TÒ!TÐà+×/Ò/Ñ1Ô1ˆØ)Ô/°Ô2°^Ô5IÈ!Ô5LÑLÐØ/×9Ò9¸"Ñ=Ô=×@Ò@ÀÔAUÑVÔVÐÝØ˜]Ô0°Ô4Ñ4¸×8LÒ8LÑ8NÔ8NÒNØsÀ^ÐsÐsÐaqÐsÐsñ	
ô 	
ð 	
ð "Ð!r0   NrX   rÆ   rˆ   rÇ   c	           	      ó\  — |du |duz  rt          d¦  «        ‚|€ |                      ¦   «         |¦  «        }|�_|                      ||¦  «        j        }
t	          j        |
d¬¦  «        }
|                      |||
¬¦  «        }|                     ||
¦  «        } | j        d|||||dœ|	¤Ž}|S )ap  
        image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`):
            The sizes of the images in the batch, being (height, width) for each image. Image sizes can be obtained using
            [`AutoImageProcessor`]. See [`Emu3ImageProcessor.__call__`] for details ([]`Emu3Processor`] uses
            [`Emu3ImageProcessor`] for processing images).
        NzaYou cannot specify both input_ids and inputs_embeds at the same time, and must specify either oner   r5   )rP  rŒ  )rX   rÆ   rˆ   rP  rÇ   r/   )	rH  rv  r�  r‰  r,   r8   rš  Úmasked_scatterrp  )r…   rO  r–  rÇ  rX   rÆ   rˆ   rP  rÇ   r[   rŒ  r—  rf  s                r1   r’   zEmu3Model.forward«  sþ   € ð( ˜Ð -°tÐ";Ñ<ð 	ÝØsñô ð ð Ð Ø7˜D×5Ò5Ñ7Ô7¸	ÑBÔBˆMàÐ#Ø!×4Ò4°\À;ÑOÔOÔ]ˆNÝ"œY ~¸1Ð=Ñ=Ô=ˆNØ!%×!:Ò!:Ø¨À~ð ";ñ "ô "Ðð *×8Ò8Ð9KÈ^Ñ\Ô\ˆMð "�$”/ð 
Ø)Ø%Ø+Ø'Øð
ð 
ð ð
ð 
ˆð ˆr0   )NNNNNNNN)r(   r)   r*   rx   rv  rx  r,   r_  r-   r„  r   r   r   r   r–   r&   r�  râ  r”   r‘  rš  r•   r
   rË   r   r’   r—   r˜   s   @r1   rl  rl  A  sa  ø€ € € € € ðð ð ð ð ð6ð 6ð 6ð4ð 4ð 4ð¨UÔ->ð ÈUÔM]ð ÐbgÔbrð ð ð ð ð& Ø€^Øyðñ ô ðØ!Ô-ðØ<AÔ<LðØX^Ð_qÔXrðà	Ð%Ñ	%ðð ð ñô ñ Ôðð0 €U„]�_„_ð°Ô0@ð È#ð ÐVYð ð ð ñ „_ðð$"ØÔ)ð"Ø:?Ô:Kð"Ø]bÔ]nð"ð "ð "ð "ð0 Øð .2Ø15Ø+/Ø.2Ø04Ø(,Ø26Ø!%ð,ð ,àÔ# dÑ*ð,ð Ô'¨$Ñ.ð,ð ”\ DÑ(ð	,ð
 œ tÑ+ð,ð Ô&¨Ñ-ð,ð  ™ð,ð Ô(¨4Ñ/ð,ð ˜$‘;ð,ð Ð+Ô,ð,ð 
Ð'Ñ	'ð,ð ,ð ,ñ „^ñ Ôð,ð ,ð ,ð ,ð ,r0   rl  c                   óx  ‡ — e Zd ZdZddiZˆ fd„Zd„ Zd„ Zdej	        fd„Z
d	„ Zee	 	 	 	 	 	 	 	 	 	 ddej        d
z  dej        d
z  dej        d
z  dej        d
z  dej        d
z  ded
z  dej        d
z  ded
z  dej        d
z  deej        z  dee         deez  fd„¦   «         ¦   «         Z	 	 	 	 	 	 	 dˆ fd„	Zˆ xZS )ÚEmu3ForConditionalGenerationr  rW  z$model.text_model.embed_tokens.weightc                 óú   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        j        |j        j        d¬¦  «        | _	        |  
                    ¦   «          d S r\  )rw   rx   rl  r  rc   r~   ro  rz   rH  rX  rÆ  r¸   s     €r1   rx   z%Emu3ForConditionalGeneration.__init__à  se   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý˜vÑ&Ô&ˆŒ
Ý”y Ô!3Ô!?ÀÔASÔA^ÐejÐkÑkÔkˆŒà�ŠÑÔÐÐÐr0   c                 ó4   — | j                              ¦   «         S rº   )r  rv  r«   s    r1   rv  z1Emu3ForConditionalGeneration.get_input_embeddingsç  s   € ØŒz×.Ò.Ñ0Ô0Ð0r0   c                 ó:   — | j                              |¦  «         d S rº   )r  rx  ry  s     r1   rx  z1Emu3ForConditionalGeneration.set_input_embeddingsê  s   € ØŒ
×'Ò'¨Ñ.Ô.Ð.Ð.Ð.r0   rJ   c                 ó   — | j         S rº   )rX  r«   s    r1   Úget_output_embeddingsz2Emu3ForConditionalGeneration.get_output_embeddingsí  s
   € ØŒ|Ðr0   c                 ó&   —  | j         j        di |¤ŽS rB  )r  r‘  )r…   r[   s     r1   r‘  z0Emu3ForConditionalGeneration.decode_image_tokensð  s   € Ø-ˆtŒzÔ-Ð7Ð7°Ð7Ð7Ð7r0   Nr   rO  r–  rÇ  rX   rÆ   rˆ   rP  rÇ   r]  r^  r[   c           
      ó\  —  | j         d||||||dœ|¤Ž}|d         }t          |
t          ¦  «        rt          |
 d¦  «        n|
}|                      |dd…|dd…f         ¦  «        }d}|	�  | j        d||	| j        j        j        dœ|¤Ž}t          |||j
        |j        |j        ¬¦  «        S )aÕ  
        image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`):
            The sizes of the images in the batch, being (height, width) for each image. Image sizes can be obtained using
            [`AutoImageProcessor`]. See [`Emu3ImageProcessor.__call__`] for details ([]`Emu3Processor`] uses
            [`Emu3ImageProcessor`] for processing images).
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import Emu3Processor, Emu3ForConditionalGeneration
        >>> import torch
        >>> import httpx
        >>> from io import BytesIO
        >>> from PIL import Image

        >>> model = Emu3ForConditionalGeneration.from_pretrained("BAAI/Emu3-Chat-hf", dtype=torch.bfloat16)
        >>> processor = Emu3Processor.from_pretrained("BAAI/Emu3-Chat-hf")

        >>> conversation = [
        ...     {
        ...     "role": "system",
        ...     "content": [
        ...         {"type": "text", "text": "You are a helpful assistant."},
        ...         ],
        ...     },
        ...     {
        ...     "role": "user",
        ...     "content": [
        ...         {"type": "image"},
        ...         {"type": "text", "text": "Please describe the image."},
        ...         ],
        ...     },
        ... ]

        >>> prompt = processor.apply_chat_template(conversation, add_generation_prompt=True)
        >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> inputs = processor(images=[image], text=[prompt], return_tensors="pt").to(model.device, torch.bfloat16)

        >>> generated_ids = model.generate(**inputs, max_new_tokens=100, do_sample=False)
        >>> processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        ```r`  r   Nra  rb  r/   )r  r²  r”   rd  rX  re  rq   ro  rH  r   rˆ   rH   r©  )r…   rO  r–  rÇ  rX   rÆ   rˆ   rP  rÇ   r]  r^  r[   rf  rH   rg  rZ  rc  s                    r1   r’   z$Emu3ForConditionalGeneration.forwardó  s  € ð@ �$”*ð 
ØØ)Ø%Ø+Ø'Øð
ð 
ð ð
ð 
ˆð   œ
ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%ð Ø f¸¼Ô9PÔ9[ðð Ø_eðð ˆDõ &ØØØ#Ô3Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r0   TFc	                 ó^   •—  t          ¦   «         j        |f|||||||dœ|	¤Ž}
|s|rd |
d<   |
S )N)rˆ   rX   rP  rÆ   r–  rÇ   Úis_first_iterationr–  )rw   Úprepare_inputs_for_generation)r…   rO  rˆ   rX   rP  rÆ   rÇ   r–  r§  r[   Úmodel_inputsr†   s              €r1   r¨  z:Emu3ForConditionalGeneration.prepare_inputs_for_generationP  sj   ø€ ð =•u‘w”wÔ<Øð

à+Ø)Ø'Ø%Ø%ØØ1ð

ð 

ð ð

ð 

ˆð "ð 	0 ið 	0Ø+/ˆL˜Ñ(àÐr0   )
NNNNNNNNNr   )NNNNTNF)r(   r)   r*   Úoutput_modalitiesrh  rx   rv  rx  rc   rp  r£  r‘  r   r   r,   r-   r_  r•   r
   rË   r”   r   r   r–   r   r’   r¨  r—   r˜   s   @r1   rž  rž  Ü  sò  ø€ € € € € Ø)ÐØ*Ð,RÐSÐðð ð ð ð ð1ð 1ð 1ð/ð /ð /ð r¤yð ð ð ð ð8ð 8ð 8ð Øð .2Ø15Ø+/Ø.2Ø04Ø(,Ø26Ø!%Ø*.Ø-.ðY
ð Y
àÔ# dÑ*ðY
ð Ô'¨$Ñ.ðY
ð ”\ DÑ(ð	Y
ð
 œ tÑ+ðY
ð Ô&¨Ñ-ðY
ð  ™ðY
ð Ô(¨4Ñ/ðY
ð ˜$‘;ðY
ð Ô  4Ñ'ðY
ð ˜eœlÑ*ðY
ð Ð+Ô,ðY
ð 
Ð'Ñ	'ðY
ð Y
ð Y
ñ „^ñ ÔðY
ð| ØØØØØØ ðð ð ð ð ð ð ð ð ð r0   rž  )rž  rV  rC  r  r¥  rl  rc  )rS   )brŽ  Úcollections.abcr   Údataclassesr   Ú	functoolsr   Útypingr   r,   Útorch.nnrc   Útorch.nn.functionalrd   r÷   Ú r   r³  Úactivationsr	   Úcache_utilsr
   r   Ú
generationr   Úintegrationsr   r   r   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   r   Úutils.output_capturingr    Úconfiguration_emu3r"   r#   r$   r&   r<   rG   r•   r”   rR   rp  r­   rm   ro   r›   r¯   r½   rÍ   rç   rú   r  r  r%  r/  r3  r>  rE  r  rP  rV  ra  r}  r…  r›  r¥  rä  r  r   rC  rV  rl  rž  Ú__all__r/   r0   r1   ú<module>rÁ     sÓ  ðð, €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !Ø %Ð %Ð %Ð %Ð %Ð %Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )Ø fÐ fÐ fÐ fÐ fÐ fÐ fÐ fÐ fÐ fØ /Ð /Ð /Ð /Ð /Ð /Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø kÐ kÐ kÐ kÐ kÐ kÐ kÐ kÐ kÐ kØ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aØ GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø KÐ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KÐ Kð Ø
ð1ð 1ð 1ð 1ð 1Ð5ñ 1ô 1ñ „ñ „ð1ð(ð (ð (ð ÐÐ*Ñ+Ô+ðð ð ñ ,Ô+ðð2	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð& ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð2 ÐÐ)Ñ*Ô*ð@)ð @)ð @)ð @)ð @)�B”Iñ @)ô @)ñ +Ô*ð@)ðF Ð˜YÑ'Ô'ðJð Jð Jð Jð J�"”)ñ Jô Jñ (Ô'ðJð(ð ð ð ð ˆbŒiñ ô ð ð (ð (ð (ð (ð (Ð1ñ (ô (ð (ðV$ð $ð $ð $ð $˜rœyñ $ô $ð $ðD	ð 	ð 	ð 	ð 	 R¤Yñ 	ô 	ð 	ðð ð ð ð  2¤9ñ ô ð ðð ð ð ð �b”iñ ô ð ð:!ð !ð !ð !ð !˜2œ9ñ !ô !ð !ðHð ð ð ð  ¤	ñ ô ð ð.ð ð ð ð  "¤)ñ ô ð ð&.(ð .(ð .(ð .(ð .( 2¤9ñ .(ô .(ð .(ðb<(ð <(ð <(ð <(ð <(˜2œ9ñ <(ô <(ð <(ð~;)ð ;)ð ;)ð ;)ð ;)˜bœiñ ;)ô ;)ð ;)ð|Vð Vð Vð Vð V˜œñ Vô Vð Vðð ð ð ð ˜2œ9ñ ô ð ðD8ð 8ð 8ð 8ð 8˜œñ 8ô 8ð 8ðv7ð 7ð 7ð 7ð 7�r”yñ 7ô 7ð 7ðtCð Cð Cð Cð C�r”yñ Cô Cð CðLCð Cð Cð Cð C�r”yñ Cô Cð CðL €ððñ ô ðx2ð x2ð x2ð x2ð x2�ñ x2ô x2ñô ðx2ðv3%ð 3%ð 3%ð 3%ð 3%ñ 3%ô 3%ð 3%ðl ðð ð ð ð ˜/ñ ô ñ „ðð*><ð ><ð ><ð ><ð ><˜"œ)ñ ><ô ><ð ><ðB ðH
ð H
ð H
ð H
ð H
Ð'ñ H
ô H
ñ „ðH
ðV ðH
ð H
ð H
ð H
ð H
Ð)¨?ñ H
ô H
ñ „ðH
ðVXð Xð Xð Xð XÐ#ñ Xô Xð XðvQð Qð Qð Qð QÐ#6¸ñ Qô Qð Qðhð ð €€€r0   