§
    ‚ŠtjÒn  ã                   ó&  — d Z ddlZddlmZ ddlZddlZddlmc m	Z
 ddlmZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZ ddlmZmZ ddlmZ ddl m!Z!m"Z" ddl#m$Z$m%Z% ddl&m'Z'm(Z(m)Z) ddl*m+Z+ ddl,m-Z- ddl.m/Z/m0Z0m1Z1 ddl2m3Z3 ddl4m5Z5  e)j6        e7¦  «        Z8e' G d„ de5¦  «        ¦   «         Z9 G d„ de!d¬¦  «        Z:e' e-d¬¦  «         G d „ d!e3¦  «        ¦   «         ¦   «         Z; e'd"¬#¦  «        e G d$„ d%e¦  «        ¦   «         ¦   «         Z<d&ej=        d'efd(„Z> G d)„ d*ej?        ¦  «        Z@ G d+„ d,ej?        ¦  «        ZAe' G d-„ d.e¦  «        ¦   «         ZBe' G d/„ d0eB¦  «        ¦   «         ZC G d1„ d2eB¦  «        ZDg d3¢ZEdS )4zTPI0 model: PaliGemma + Action Expert with flow matching for robot action prediction.é    N)ÚCallable)Ústrict)Únné   )Úinitialization)ÚCache)ÚPreTrainedConfig)ÚBatchFeature)Ú
ImageInputÚmake_nested_list_of_images)Úcreate_causal_mask)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚPreTrainedModel)ÚProcessingKwargsÚUnpack)ÚPreTokenizedInputÚ	TextInput)Úauto_docstringÚcan_return_tupleÚlogging)Úmaybe_autocast)Úrequiresé   )ÚCONFIG_MAPPINGÚ
AutoConfigÚ	AutoModel)ÚPaligemmaProcessor)ÚSiglipImageProcessorc                   ó&   — e Zd ZdddœZdddœZdZdS )ÚPI0ImageProcessoréà   )Ú
max_heightÚ	max_width)ÚheightÚwidthTN)Ú__name__Ú
__module__Ú__qualname__ÚsizeÚpad_sizeÚdo_pad© ó    úa/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/pi0/modular_pi0.pyr!   r!   .   s-   € € € € € à¨CÐ0Ð0€DØ¨Ð,Ð,€HØ€F€F€Fr.   r!   c                   ó$   — e Zd ZddddœddidœZdS )	ÚPI0ProcessorKwargsÚ
max_lengthé0   Úright)Úpaddingr2   Úpadding_sideÚreturn_tensorsÚpt)Útext_kwargsÚcommon_kwargsN)r'   r(   r)   Ú	_defaultsr-   r.   r/   r1   r1   5   s9   € € € € € ð $ØØ#ð
ð 
ð
 +¨DÐ1ðð €I€I€Ir.   r1   F)Útotal)ÚvisionÚtorch)Úbackendsc                   ó   ‡ — e Zd Zdˆ fd„	Z	 	 	 ddeee         z  eee                  z  dz  deez  ee         z  ee         z  dz  deej	        z  e
j        z  dz  deej	        z  e
j        z  dz  dee         defd	„Zeˆ fd
„¦   «         Zˆ xZS )ÚPI0ProcessorNc                 óŠ  •— |j         d         |j         d         c| _        | _        |                     dg d¢¦  «        }|                     dg d¢¦  «        }|                     dg d¢¦  «        }|                     d	g d
¢¦  «        }t	          j        |¦  «        | _        t	          j        |¦  «        | _        t	          j        |¦  «        | _        t	          j        |¦  «        | _	        |                     dd¦  «        | _
        |                     dd¦  «        | _        t          ¦   «                              ||¦  «         d S )Nr%   r&   Ú
state_mean)gù g³ês¥¿grŠŽäò¢?g•Ô	h"lê?g‡§WÊ2D@g\ AñcÌá¿gZd;ßOÅ¿gB>èÙ¬úœ?g“©‚QI�¿Ú	state_std)gt$—ÿ�~»?gL¦
F%uÂ?g.ÿ!ýöuÐ?g/n£¼Ö?gëâ6Àó?g¶óýÔxéÖ?gÌ]KÈ=‹?gF%uš‹?Úactions_mean)gÂ&S£’?gâX·Ñ ®?gW[±¿ìž¬¿gHPüs×r?gàœ¥½Ág?gÒ Þ	Š¿gHPüs·¿Úactions_std)g®GázÒ?g`åÐ"ÛùÖ?gÁ9#J{ƒ×?gvOjM£?g>yX¨5Í«?g46<½R¶?gèj+ö—Ýï?Úmax_state_dimé    Ú
chunk_sizeé2   )r*   r%   r&   Úgetr>   ÚtensorrC   rD   rE   rF   rG   rI   ÚsuperÚ__init__)
ÚselfÚimage_processorÚ	tokenizerÚchat_templateÚkwargsrC   rD   rE   rF   Ú	__class__s
            €r/   rN   zPI0Processor.__init__C   s  ø€ Ø"1Ô"6°xÔ"@À/ÔBVÐW^ÔB_ÐˆŒ�T”ZØ—Z’Z Ð.rÐ.rÐ.rÑsÔsˆ
Ø—J’J˜{Ð,lÐ,lÐ,lÑmÔmˆ	Ø—z’z .Ð2mÐ2mÐ2mÑnÔnˆØ—j’j Ð0hÐ0hÐ0hÑiÔiˆåœ, zÑ2Ô2ˆŒÝœ iÑ0Ô0ˆŒÝ!œL¨Ñ6Ô6ˆÔÝ œ<¨Ñ4Ô4ˆÔØ#ŸZšZ¨¸Ñ<Ô<ˆÔØ Ÿ*š* \°2Ñ6Ô6ˆŒÝ‰Œ×Ò˜¨)Ñ4Ô4Ð4Ð4Ð4r.   ÚimagesÚtextÚactionsÚstaterS   Úreturnc                 óª  —  | j         t          fd| j        j        i|¤Ž}|€t                               d¦  «         d}t          |t          ¦  «        r|g}t          |¦  «        }t          |¦  «        t          |¦  «        k    r0t          dt          |¦  «        › dt          |¦  «        › d�¦  «        ‚|d                              d	d¦  «        }|d
                              d	d¦  «         g }	t          ||¦  «        D ]J\  }
}| j        | j        z  t          |¦  «        z  › | j        j        › |
› d�}
|	                     |
¦  «         ŒK | j        |	fi |d         ¤Ž}t#          d„ |D ¦   «         ¦  «        }t%          j        t          |¦  «        |ft$          j        ¬¦  «        }t%          j        t          |¦  «        |d| j        | j        ¦  «        }t/          |¦  «        D ]B\  }} | j        |fd	di|d
         ¤Ž}t          |¦  «        }d||d|…f<   |d         ||d|…f<   ŒCi |¥||dœ¥}|�‹t%          j        |¦  «        | j        z
  | j        dz   z  }|j        d         | j        k     r*t=          j        |d| j        |j        d         z
  f¦  «        }|                      d| j!        | j        ¦  «        |d<   |�…t%          j        |¦  «        | j"        z
  | j#        dz   z  }|j        d         | j        k     r*t=          j        |d| j        |j        d         z
  f¦  «        }|                      d| j        ¦  «        |d<   tI          ||¬¦  «        S )aC  
        actions (`list | np.ndarray | torch.Tensor`, *optional*):
            Actions to be predicted by the model. If provided, padding, mean and std normalization will be applied.
        state (`list | np.ndarray | torch.Tensor`, *optional*):
            Robotic states to be predicted by the model. If provided, padding, mean and std normalization will be applied.

        Returns:
            [`BatchFeature`]: A [`BatchFeature`] with the following fields:

            - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`. If `suffix`
              is provided, the `input_ids` will also contain the suffix input ids.
            - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when
              `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not
              `None`).
            - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`.
            - **pixel_attention_mask** -- Pixel values padding mask to be fed to a model. Returned when `images` is not `None`.
            - **state** -- Robot state compatible with model if `state` is not None
            - **actions** -- Label-actions compatible with training if `actions` is not None
        Útokenizer_init_kwargsNzPYou are using PI0 without a text prefix. The processor will use an empty prompt.Ú z	Received z image samples for z\ prompts. Each prompt should be associated with one sample (with one or more camera images).r9   r7   Úimages_kwargsú
c              3   ó4   K  — | ]}t          |¦  «        V — Œd S ©N)Úlen)Ú.0Úsample_imagess     r/   ú	<genexpr>z(PI0Processor.__call__.<locals>.<genexpr>Œ   s*   è è € ÐUÐU°]�c -Ñ0Ô0ÐUÐUÐUÐUÐUÐUr.   ©Údtyper   r8   TÚpixel_values)rg   Úpixel_attention_maskg:Œ0âŽyE>éÿÿÿÿr   rW   rX   )ÚdataÚtensor_type)%Ú_merge_kwargsr1   rQ   Úinit_kwargsÚloggerÚwarning_onceÚ
isinstanceÚstrr   ra   Ú
ValueErrorÚpopÚzipÚimage_tokenÚimage_seq_lengthÚ	bos_tokenÚappendÚmaxr>   ÚzerosÚboolr%   r&   Ú	enumeraterP   rL   rE   rF   ÚshaperG   ÚFÚpadÚviewrI   rC   rD   r
   )rO   rU   rV   rW   rX   rS   Úoutput_kwargsÚbatched_imagesr7   Úprompt_stringsÚsampleÚ
image_listÚtext_inputsÚmax_num_camerasrh   Úpadded_pixel_valuesÚbatchrc   Ú	processedÚnum_camerasÚreturn_datas                        r/   Ú__call__zPI0Processor.__call__R   sÏ  € ð6 +˜Ô*Ýð
ð 
Ø6:´nÔ6Pð
ØTZð
ð 
ˆð ˆ<Ý×ÒÐ rÑsÔsÐsØˆDå�d�CÑ Ô ð 	Ø�6ˆDå3°FÑ;Ô;ˆÝˆ~ÑÔ¥# d¡)¤)Ò+Ð+Ýðe�C Ñ/Ô/ð eð eÅCÈÁIÄIð eð eð eñô ð ð
 ' }Ô5×9Ò9Ð:JÈDÑQÔQˆØ�oÔ&×*Ò*Ð+;¸TÑBÔBÐBàˆÝ"% d¨NÑ";Ô";ð 	*ð 	*ÑˆF�JàÔ# dÔ&;Ñ;½cÀ*¹o¼oÑMÐsÈtÌ~ÔOgÐsÐioÐsÐsÐsð ð ×!Ò! &Ñ)Ô)Ð)Ð)à$�d”n ^ÐTÐT°}À]Ô7SÐTÐTˆõ ÐUÐUÀnÐUÑUÔUÑUÔUˆÝ$œ{­C°Ñ,?Ô,?ÀÐ+QÕY^ÔYcÐdÑdÔdÐÝ#œk­#¨nÑ*=Ô*=¸ÐPQÐSWÔS^Ð`dÔ`jÑkÔkÐå$-¨nÑ$=Ô$=ð 	Qð 	QÑ ˆE�=Ø,˜Ô,¨]ÐrÐrÈ4ÐrÐS`ÐapÔSqÐrÐrˆIå˜mÑ,Ô,ˆKØ8<Ð  ¨¨¨Ð!4Ñ5Ø7@ÀÔ7PÐ  |¨ |Ð 3Ñ4Ð4ð
Øð
à/Ø$8ð
ð 
ð 
ˆð ÐÝ”| GÑ,Ô,¨tÔ/@Ñ@ÀTÔEUÐX]ÑE]Ñ^ˆGØŒ}˜RÔ  4Ô#5Ò5Ð5Ýœ% ¨!¨TÔ-?À'Ä-ÐPRÔBSÑ-SÐ)TÑUÔU�Ø%,§\¢\°"°d´oÀtÔGYÑ%ZÔ%ZˆK˜	Ñ"àÐÝ”\ %Ñ(Ô(¨4¬?Ñ:¸t¼~ÐPUÑ?UÑVˆEØŒ{˜2Œ Ô!3Ò3Ð3Ýœ˜e a¨Ô);¸e¼kÈ"¼oÑ)MÐ%NÑOÔO�Ø#(§:¢:¨b°$Ô2DÑ#EÔ#EˆK˜Ñ å ¸.ÐIÑIÔIÐIr.   c                 ó2   •— t          ¦   «         j        dgz   S )Nrh   )rM   Úmodel_input_names)rO   rT   s    €r/   r�   zPI0Processor.model_input_names«   s   ø€ å‰wŒwÔ(Ð,BÐ+CÑCÐCr.   )NNN)r'   r(   r)   rN   r   Úlistr   r   ÚnpÚndarrayr>   ÚTensorr   r1   r
   r�   Úpropertyr�   Ú__classcell__©rT   s   @r/   rA   rA   @   sL  ø€ € € € € ð5ð 5ð 5ð 5ð 5ð 5ð$ bfØ;?Ø9=ðWJð WJà˜T *Ô-Ñ-°°T¸*Ô5EÔ0FÑFÈÑMðWJð Ð+Ñ+¨d°9¬oÑ=ÀÐEVÔ@WÑWÐZ^Ñ^ðWJð ˜œ
Ñ" U¤\Ñ1°DÑ8ð	WJð
 �b”jÑ  5¤<Ñ/°$Ñ6ðWJð Ð+Ô,ðWJð 
ðWJð WJð WJð WJðr ðDð Dð Dð Dñ „XðDð Dð Dð Dð Dr.   rA   zlerobot/pi0_base)Ú
checkpointc                   ó  ‡ — e Zd ZU dZdZeedœZdZee	z  dz  e
d<   dZee	z  dz  e
d<   dZee
d<   d	Zee
d
<   d	Zee
d<   dZee
d<   dZee
d<   dZee
d<   dZee
d<   dZee
d<   dZee
d<   dZee
d<   dZee
d<   ˆ fd„Zd„ Zˆ xZS )Ú	PI0ConfigaÜ  
    vlm_config (`dict`, *optional*):
        Configuration for the vlm backbone (PaliGemmaModel).
    dit_config (`dict`, *optional*):
        Configuration for the DiT backbone. Defaults to a Gemma 300M variant.
    chunk_size (`int`, *optional*, defaults to 50):
        Number of action steps to predict per chunk.
    max_state_dim (`int`, *optional*, defaults to 32):
        Maximum state vector dimension (shorter vectors are zero-padded).
    max_action_dim (`int`, *optional*, defaults to 32):
        Maximum action vector dimension (shorter vectors are zero-padded).
    num_inference_steps (`int`, *optional*, defaults to 10):
        Number of denoising steps during inference.
    time_sampling_beta_alpha (`float`, *optional*, defaults to 1.5):
        Alpha parameter for Beta distribution used to sample diffusion time during training.
    time_sampling_beta_beta (`float`, *optional*, defaults to 1.0):
        Beta parameter for Beta distribution used to sample diffusion time during training.
    time_sampling_scale (`float`, *optional*, defaults to 0.999):
        Scale factor for sampled time values.
    time_sampling_offset (`float`, *optional*, defaults to 0.001):
        Offset added to sampled time values.
    min_period (`float`, *optional*, defaults to 0.004):
        Minimum period for sinusoidal time embedding.
    max_period (`float`, *optional*, defaults to 4.0):
        Maximum period for sinusoidal time embedding.
    loss_reduction (`str`, *optional*, defaults to `"mean"`):
        The reduction to use on MSE loss.

    Example:
    ```python
    >>> from transformers import PI0ForConditionalGeneration, PI0Config

    >>> config = PI0Config()
    >>> model = PI0ForConditionalGeneration(config)
    ```
    Úpi0)Ú
vlm_configÚ
dit_configNr›   rœ   rJ   rI   rH   rG   Úmax_action_dimé
   Únum_inference_stepsg      ø?Útime_sampling_beta_alphaç      ð?Útime_sampling_beta_betag+‡ÙÎ÷ï?Útime_sampling_scalegü©ñÒMbP?Útime_sampling_offsetgü©ñÒMbp?Ú
min_periodg      @Ú
max_periodÚmeanÚloss_reductionc                 ó°  •— t          | j        t          ¦  «        r8| j                             dd¦  «        }t	          |         di | j        ¤Ž| _        n7| j        €0t	          d         ddddddd	d
œdddddddd	ddœ	dd	¬¦  «        | _        t          | j        t          ¦  «        r8| j                             dd¦  «        }t	          |         di | j        ¤Ž| _        n7| j        €0t	          d         dddddd| j        j        j        ¬¦  «        | _        d| j        _        d| j        _	        d| j        j        _	         t          ¦   «         j        di |¤Ž d S )NÚ
model_typeÚ	paligemmaÚgemmai   é   i @  é   é   i€ì )rª   Úhidden_sizeÚnum_hidden_layersÚintermediate_sizeÚnum_attention_headsÚnum_key_value_headsÚ
vocab_sizeÚsiglip_vision_modeliÐ  i€  é   r"   é   é   F)	rª   r²   r°   Ú
patch_sizeÚ
image_sizer±   r³   rµ   Úvision_use_head)Útext_configÚvision_configÚprojection_dimÚimage_token_idi   i   é   )r°   r±   r²   r³   r´   Úhead_dimrµ   Tr-   )rp   r›   ÚdictrK   r   rœ   r½   rµ   Ú	is_causalÚuse_bidirectional_attentionrM   Ú__post_init__)rO   rS   Úvlm_model_typeÚdit_model_typerT   s       €r/   rÆ   zPI0Config.__post_init__é   sŽ  ø€ Ý�d”o¥tÑ,Ô,ð 	Ø!œ_×0Ò0°¸{ÑKÔKˆNÝ,¨^Ô<ÐOÐO¸t¼ÐOÐOˆDŒOˆOØŒ_Ð$Ý,¨[Ô9à")Ø#'Ø)+Ø).Ø+,Ø+,Ø"(ðð ð #8Ø)-Ø#'Ø"$Ø"%Ø)+Ø+-Ø"(Ø',ð
ð 
ð  $Ø%ð-ñ ô ˆDŒOõ2 �d”o¥tÑ,Ô,ð 	Ø!œ_×0Ò0°¸wÑGÔGˆNÝ,¨^Ô<ÐOÐO¸t¼ÐOÐOˆDŒOˆOØŒ_Ð$Ý,¨WÔ5Ø Ø"$Ø"&Ø$%Ø$%ØØœ?Ô6ÔAðñ ô ˆDŒOð %)ˆŒÔ!Ø6:ˆŒÔ3ØBFˆŒÔ#Ô?Ø�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'r.   c                 óp   — | j         j        dz  dk    r"t          d| j        j         j        › d�¦  «        ‚dS )zOPart of `@strict`-powered validation. Validates the architecture of the config.r   r   zDiT hidden dim=(z) must be divisible by 2N)rœ   r°   rr   Úconfig©rO   s    r/   Úvalidate_architecturezPI0Config.validate_architecture  sA   € àŒ?Ô&¨Ñ*¨aÒ/Ð/ÝÐl°´Ô0FÔ0RÐlÐlÐlÑmÔmÐmð 0Ð/r.   )r'   r(   r)   Ú__doc__rª   r   Úsub_configsr›   rÃ   r	   Ú__annotations__rœ   rI   ÚintrG   r�   rŸ   r    Úfloatr¢   r£   r¤   r¥   r¦   r¨   rq   rÆ   rÌ   r•   r–   s   @r/   r™   r™   °   s[  ø€ € € € € € ð#ð #ðJ €JØ!+¸:ÐFÐF€Kà15€J�Ð'Ñ'¨$Ñ.Ð5Ð5Ñ5Ø15€J�Ð'Ñ'¨$Ñ.Ð5Ð5Ñ5Ø€J�ÐÐÑØ€M�3ÐÐÑØ€N�CÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø&)Ð˜eÐ)Ð)Ñ)Ø%(Ð˜UÐ(Ð(Ñ(Ø!&Ð˜Ð&Ð&Ñ&Ø"'Ð˜%Ð'Ð'Ñ'Ø€J�ÐÐÑØ€J�ÐÐÑØ €N�CÐ Ð Ñ ð0(ð 0(ð 0(ð 0(ð 0(ðdnð nð nð nð nð nð nr.   r™   Úblock_boundariesrY   c           
      óZ   ‡ — dt           dt           dt           dt           dt          f
ˆ fd„}|S )NÚ	batch_idxÚhead_idxÚq_idxÚkv_idxrY   c                 ód   •— t          j        |‰¦  «        }t          j        |‰¦  «        }||k    S r`   )r>   Ú	bucketize)rÔ   rÕ   rÖ   r×   Úq_blockÚkv_blockrÒ   s         €r/   Ú
inner_maskz0blockwise_bidirectional_mask.<locals>.inner_mask"  s2   ø€ Ý”/ %Ð)9Ñ:Ô:ˆÝ”? 6Ð+;Ñ<Ô<ˆØ˜7Ò"Ð"r.   )rÐ   r{   )rÒ   rÜ   s   ` r/   Úblockwise_bidirectional_maskrÝ   !  sL   ø€ ð#�cð #­Sð #½ð #Åcð #Ídð #ð #ð #ð #ð #ð #ð
 Ðr.   c                   ó:   ‡ — e Zd Zˆ fd„Zed„ ¦   «         Zd„ Zˆ xZS )ÚPI0TimestepEmbeddingsc                 ó°   •— t          ¦   «                              ¦   «          || _        |                      |¦  «        }|                      d|d¬¦  «         d S )NÚsinusoid_freqF)Ú
persistent)rM   rN   rÊ   Úcompute_freqsÚregister_buffer)rO   rÊ   rá   rT   s      €r/   rN   zPI0TimestepEmbeddings.__init__+  sT   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ×*Ò*¨6Ñ2Ô2ˆØ×Ò˜_¨mÈÐÑNÔNÐNÐNÐNr.   c                 óÂ   — t          j        dd| j        j        dz  t           j        ¬¦  «        }| j        | j        | j        z  |z  z  }d|z  dz  t          j        z  }|S )Nç        r¡   r   re   )	r>   Úlinspacerœ   r°   Úfloat32r¥   r¦   ÚmathÚpi)rÊ   ÚfractionÚperiodrá   s       r/   rã   z#PI0TimestepEmbeddings.compute_freqs1  sb   € å”> # s¨FÔ,=Ô,IÈQÑ,NÕV[ÔVcÐdÑdÔdˆØÔ" fÔ&7¸&Ô:KÑ&KÐPXÑ%XÑXˆØ˜f™ qÑ(­4¬7Ñ2ˆØÐr.   c                 óŠ  — t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t	          |d¬¦  «        5  | j        d d d …f         }||d d …d f         z  }t          j        |                     ¦   «         | 	                    ¦   «         gd¬¦  «        }d d d ¦  «         n# 1 swxY w Y   |S )NÚmpsÚcpuF)Údevice_typeÚenabledr¯   ©Údim)
rp   ÚdeviceÚtyperq   r   rá   r>   ÚcatÚsinÚcos)rO   Útimerð   rá   ÚembÚtime_embedss         r/   ÚforwardzPI0TimestepEmbeddings.forward8  s  € Ý*4°T´[Ô5EÅsÑ*KÔ*KÐtÐPTÔP[ÔP`ÐdiÒPiÐPi�d”kÔ&Ð&ÐotˆÝ¨¸UÐCÑCÔCð 	Cð 	CØ Ô.¨t°Q°Q°Q¨wÔ7ˆMØ $ q q q¨$ w¤-Ñ/ˆCÝœ) S§W¢W¡Y¤Y°·²±	´	Ð$:ÀÐBÑBÔBˆKð	Cð 	Cð 	Cñ 	Cô 	Cð 	Cð 	Cð 	Cð 	Cð 	Cð 	Cøøøð 	Cð 	Cð 	Cð 	Cð Ðs   ÁAB8Â8B<Â?B<)r'   r(   r)   rN   Ústaticmethodrã   rü   r•   r–   s   @r/   rß   rß   *  sg   ø€ € € € € ðOð Oð Oð Oð Oð ðð ñ „\ððð ð ð ð ð ð r.   rß   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚPI0ActionTimeEmbeddingc                 óÒ  •— t          ¦   «                              ¦   «          t          |¦  «        | _        t	          j        |j        |j        j        ¦  «        | _	        t	          j        |j
        |j        j        ¦  «        | _        t	          j        d|j        j        z  |j        j        ¦  «        | _        t	          j        |j        j        |j        j        ¦  «        | _        d S )Nr   )rM   rN   rß   Úsinusoid_embedsr   ÚLinearr�   rœ   r°   Úaction_in_projrG   Ú
state_projÚaction_time_mlp_inÚaction_time_mlp_out©rO   rÊ   rT   s     €r/   rN   zPI0ActionTimeEmbedding.__init__B  s­   ø€ Ý‰Œ×ÒÑÔÐÝ4°VÑ<Ô<ˆÔÝ œi¨Ô(=¸vÔ?PÔ?\Ñ]Ô]ˆÔÝœ) FÔ$8¸&Ô:KÔ:WÑXÔXˆŒÝ"$¤)¨A°Ô0AÔ0MÑ,MÈvÔO`ÔOlÑ"mÔ"mˆÔÝ#%¤9¨VÔ->Ô-JÈFÔL]ÔLiÑ#jÔ#jˆÔ Ð Ð r.   c                 óè  — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|d d …d d d …f                              |¦  «                             |j        ¬¦  «        }t          j        ||gd¬¦  «        }|                      t          j
        |                      |¦  «        ¦  «        ¦  «        }t          j        |d d …d d d …f         |gd¬¦  «        }|S )Nre   r   rò   r¯   )r  r  r  Ú	expand_asÚtorf   r>   rö   r  r~   Úsilur  )	rO   rX   ÚnoiseÚtimestepÚstate_embedsÚaction_embedsrû   Úaction_time_embedsÚaction_embeds_mergeds	            r/   rü   zPI0ActionTimeEmbedding.forwardJ  sð   € Ø—’ uÑ-Ô-ˆØ×+Ò+¨EÑ2Ô2ˆà×*Ò*¨8Ñ4Ô4ˆØ! ! ! ! T¨1¨1¨1 *Ô-×7Ò7¸ÑFÔF×IÒIÐP]ÔPcÐIÑdÔdˆå"œY¨°{Ð'CÈÐKÑKÔKÐØ!×5Ò5µa´f¸T×=TÒ=TÐUgÑ=hÔ=hÑ6iÔ6iÑjÔjÐÝ$œy¨,°q°q°q¸$ÀÀÀ°zÔ*BÐDVÐ)WÐ]^Ð_Ñ_Ô_ÐØ#Ð#r.   )r'   r(   r)   rN   rü   r•   r–   s   @r/   rÿ   rÿ   A  sL   ø€ € € € € ðkð kð kð kð kð
$ð 
$ð 
$ð 
$ð 
$ð 
$ð 
$r.   rÿ   c                   óT   ‡ — e Zd ZU eed<   dZdZdZdgZdZ	dZ
dZdZdZdZˆ fd„Zˆ xZS )ÚPI0PreTrainedModelrÊ   ÚmodelrX   TÚpast_key_values)ÚimagerV   c                 óÜ   •— t          ¦   «                              |¦  «         t          |t          ¦  «        r4t	          j        |j        |                     |j        ¦  «        ¦  «         d S d S r`   )	rM   Ú_init_weightsrp   rß   ÚinitÚcopy_rá   rã   rÊ   )rO   ÚmodulerT   s     €r/   r  z PI0PreTrainedModel._init_weightse  sf   ø€ Ý‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ3Ñ4Ô4ð 	RÝŒJ�vÔ+¨V×-AÒ-AÀ&Ä-Ñ-PÔ-PÑQÔQÐQÐQÐQð	Rð 	Rr.   )r'   r(   r)   r™   rÏ   Úbase_model_prefixÚmain_input_nameÚsupports_gradient_checkpointingÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendÚinput_modalitiesr  r•   r–   s   @r/   r  r  W  sŒ   ø€ € € € € € àÐÐÑØÐØ€OØ&*Ð#Ø#4Ð"5ÐØÐØ€NØÐØ!ÐØ"&ÐØ(ÐðRð Rð Rð Rð Rð Rð Rð Rð Rr.   r  c                   ó  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zdd„Zee		 	 	 	 	 	 	 dde
j        de
j        dz  d	e
j        dz  d
e
j        dz  de
j        dz  de
j        dz  de
j        dz  dedz  defd„¦   «         ¦   «         Zˆ xZS )ÚPI0ModelrÊ   c                 óê   •— t          ¦   «                              |¦  «         t          j        |j        ¦  «        | _        t          j        |j        ¦  «        | _        |                      ¦   «          d S r`   )	rM   rN   r   Úfrom_configrœ   Úditr›   ÚvlmÚ	post_initr  s     €r/   rN   zPI0Model.__init__m  s[   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ÝÔ(¨Ô):Ñ;Ô;ˆŒÝÔ(¨Ô):Ñ;Ô;ˆŒØ�ŠÑÔÐÐÐr.   c                 ó4   — | j                              ¦   «         S r`   )r+  Úget_input_embeddingsrË   s    r/   r.  zPI0Model.get_input_embeddingss  s   € ØŒx×,Ò,Ñ.Ô.Ð.r.   c                 ó:   — | j                              |¦  «         d S r`   )r+  Úset_input_embeddings)rO   Úvalues     r/   r0  zPI0Model.set_input_embeddingsv  s   € ØŒ×%Ò% eÑ,Ô,Ð,Ð,Ð,r.   Nc                 óÎ  — |j         d         }|                     dd¦  «        }| j                             |¦  «        j        }|                     d||j         d         |j         d         ¦  «        }g }t          |¦  «        D ](\  }}	||         |	         }
|                     |
¦  «         Œ)t          j	        |d¬¦  «        }| 
                    ¦   «         }d||| j        j        j        k    <    | j                             ¦   «         |¦  «        }|| j        j        j        k                         d¦  «                             |j        ¦  «        }|                     ||¦  «        }|S )Nr¯   r   ri   r   rò   )r}   Úflattenr+  Úget_image_featuresÚpooler_outputÚreshaper|   rx   r>   rö   ÚclonerÊ   r›   rÀ   r.  Ú	unsqueezer
  rô   Úmasked_scatter)rO   Ú	input_idsrg   rh   Úattention_maskr‡   Úimage_featuresÚtotal_image_featuresrÔ   ÚmaskÚunpadded_image_featuresÚllm_input_idsÚinputs_embedsÚspecial_image_masks                 r/   Úembed_prefixzPI0Model.embed_prefixy  sa  € Ø.Ô4°QÔ7ˆØ#×+Ò+¨A¨qÑ1Ô1ˆØœ×4Ò4°\ÑBÔBÔPˆØ'×/Ò/°°OÀ^ÔEYÐZ[ÔE\Ð^lÔ^rÐstÔ^uÑvÔvˆà!ÐÝ(Ð)=Ñ>Ô>ð 	Að 	A‰OˆI�tØ&4°YÔ&?ÀÔ&EÐ#Ø ×'Ò'Ð(?Ñ@Ô@Ð@Ð@Ý$œyÐ)=À1ÐEÑEÔEÐà!ŸšÑ)Ô)ˆØLMˆ�i 4¤;Ô#9Ô#HÒHÑIØ7˜œ×5Ò5Ñ7Ô7¸ÑFÔFˆà˜$œ+Ô0Ô?Ò?×JÒJÈ2ÑNÔN×QÒQÐR_ÔRfÑgÔgð 	ð &×4Ò4Ð5GÐI]Ñ^Ô^ˆàÐr.   r  r:  rg   r;  rh   Úposition_idsrA  r  rY   c	                 óú  — |�w|€u|�|€|                      d¦  «        dz
  }|€|                      |||¦  «        }t          j        |¦  «        dd…dd…df         }
|                      ||||
d¬¦  «        j        }|�|j        dk    rt          d¦  «        ‚dx}}|��t          j        |j	        d         |j	        d         |j
        |j        ¬	¦  «        }t          j        ||gd¬
¦  «        }t          j         |d¬
¦  «        dz
  dd…|j	        d          d…f         }|                     ¦   «         }t          j        t          j        |dz   |j        t          j        ¬¦  «        t          j        |j	        d         dz
  |j        t          j        ¬¦  «        g¦  «        }|ddd…f                              |j	        d         d¦  «        }t#          | j        j        ||||¬¦  «        } | j        d||||dœ|	¤Ž}|S )zû
        action_embeds (`torch.Tensor`, *optional*):
            The embeddings of input actions and robot states.
        pixel_attention_mask (`torch.Tensor`, *optional*):
            The mask indicating padded positions in the input image.
        Nri   r¯   r   T)rA  r;  rD  Útoken_type_idsÚ	use_cacher   z:Only two-dimensional attention masks are accepted for now!©rf   rô   rò   ©rô   rf   )rÊ   rA  r;  r  Úblock_sequence_ids)rA  r;  rD  r  r-   )ÚcumsumrC  r>   Ú
zeros_liker+  r  Úndimrr   Úonesr}   rf   rô   rö   Úget_seq_lengthrz   ÚlongÚrepeatr   rÊ   rœ   r*  )rO   r  r:  rg   r;  rh   rD  rA  r  rS   rF  Údit_position_idsÚdit_attention_maskÚ
noise_maskÚvlm_input_lengthrJ  Úbidirectional_maskÚ
dit_outputs                     r/   rü   zPI0Model.forward�  su  € ð( Ð#¨Ð(?ØÐ)¨lÐ.BØ-×4Ò4°RÑ8Ô8¸1Ñ<�àÐ$Ø $× 1Ò 1°)¸\ÐK_Ñ `Ô `�õ #Ô-¨mÑ<Ô<¸Q¸Q¸QÀÀÀÀ1¸WÔEˆNØ"ŸhšhØ+Ø-Ø)Ø-Øð 'ñ ô ô ð ð Ð%¨.Ô*=ÀÒ*BÐ*BÝÐYÑZÔZÐZð 15Ð4ÐÐ-ØÐ%ÝœØÔ# AÔ&ØÔ# AÔ&Ø$Ô*Ø%Ô,ð	ñ ô ˆJõ "'¤¨N¸JÐ+GÈQÐ!OÑ!OÔ!OÐÝ %¤Ð-?ÀQÐ GÑ GÔ GÈ!Ñ KÈQÈQÈQÐQ^ÔQdÐefÔQgÐPgÐPiÐPiÐMiÔjÐð +×9Ò9Ñ;Ô;ÐÝ"œYå”Ð,¨qÑ0¸Ô9MÕUZÔU_Ð`Ñ`Ô`Ý”
˜=Ô.¨qÔ1°AÑ5¸mÔ>RÕZ_ÔZdÐeÑeÔeðñ
ô 
Ðð 0°°a°a°a°Ô8×?Ò?ÀÔ@SÐTUÔ@VÐXYÑZÔZÐÝ/Ø”;Ô)Ø'Ø-Ø+Ø1ð
ñ 
ô 
Ðð �T”Xð 
Ø'Ø-Ø)Ø+ð	
ð 
ð
 ð
ð 
ˆ
ð Ðr.   r`   )NNNNNNN)r'   r(   r)   r™   rN   r.  r0  rC  r   r   r>   r“   Ú
LongTensorr   r   rü   r•   r–   s   @r/   r'  r'  k  sX  ø€ € € € € ð˜yð ð ð ð ð ð ð/ð /ð /ð-ð -ð -ðð ð ð ð, Øð *.Ø,0Ø.2Ø48Ø04Ø-1Ø(,ðKð Kà”|ðKð ”< $Ñ&ðKð ”l TÑ)ð	Kð
 œ tÑ+ðKð $œl¨TÑ1ðKð Ô&¨Ñ-ðKð ”| dÑ*ðKð  ™ðKð 
!ðKð Kð Kñ „^ñ ÔðKð Kð Kð Kð Kr.   r'  c                   óò  ‡ — e Zd ZdZddiZdefˆ fd„Zee	 	 	 	 	 	 	 	 	 	 dde	j
        de	j
        dz  d	e	j
        dz  d
e	j        dz  de	j        dz  de	j        dz  de	j        dz  de	j        dz  de	j        dz  dedz  de	j
        defd„¦   «         ¦   «         Z e	j        ¦   «         	 	 	 	 dde	j
        d
e	j        de	j
        de	j
        dz  de	j        dz  de	j        dz  dedz  de	j
        fd„¦   «         Zˆ xZS )ÚPI0ForConditionalGenerationz9PI0 model with action projection heads and flow matching.Úaction_out_projÚcolwise_gather_outputrÊ   c                 ó,  •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        j        | _        t          |¦  «        | _        t          j
        | j        |j        ¦  «        | _        |                      ¦   «          d S r`   )rM   rN   r'  r  rœ   r°   Úexpert_hidden_sizerÿ   Úembed_action_timer   r  r�   r[  r,  r  s     €r/   rN   z$PI0ForConditionalGeneration.__init__ä  sx   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý˜fÑ%Ô%ˆŒ
Ø"(Ô"3Ô"?ˆÔÝ!7¸Ñ!?Ô!?ˆÔÝ!œy¨Ô)@À&ÔBWÑXÔXˆÔØ�ŠÑÔÐÐÐr.   NrX   r  r  r:  rg   rh   r;  rD  rA  r  rW   rY   c                 óü  — |j         d         }|€Ðt          j        | j        j        t          j        ¬¦  «        }t          j        | j        j        t          j        ¬¦  «        }t          j                             ||¦  «        }| 	                    |f¦  «         
                    |j        ¦  «        }|| j        j        z  | j        j        z                        ¦   «         }|€7t          j        || j        j        | j        j        |j        |j        ¬¦  «        }|�9|dd…ddf         }||z  d|z
  |z  z    
                    |j        ¦  «        }||z
  }n|}|                      |||¦  «        } | j        d	||||||	||
dœ|¤Ž}|j        dd…| j        j         d…f         }|                      |¦  «        }d}|�!t/          j        ||| j        j        ¬¦  «        }t5          |||j        |j        |j        ¬¦  «        S )
a-  
        state (`torch.Tensor`, *optional*):
            Current robot state.
        noise (`torch.Tensor`, *optional*):
            Random noise at current timestep that needs to be denoised
        timestep (`torch.Tensor`, *optional*):
            Current denoising timestep.
        pixel_attention_mask (`torch.Tensor`, *optional*):
            The mask indicating padded positions in the input image.
        actions (`torch.Tensor`, *optional*):
            Input actions that need to be predicted. Used only when training to compiute loss.
        r   Nre   rI  r¯   )r:  rg   r;  rh   rD  rA  r  r  )Ú	reduction)ÚlossÚlogitsr  Úhidden_statesÚ
attentionsr-   )r}   r>   rL   rÊ   r    rè   r¢   ÚdistributionsÚBetar„   r
  rô   r£   r¤   rÑ   ÚrandnrI   r�   rf   r_  r  Úlast_hidden_stater[  r~   Úmse_lossr¨   r   r  rd  re  )rO   rX   r  r  r:  rg   rh   r;  rD  rA  r  rW   rS   Ú
batch_sizeÚalpha_tÚbeta_tÚdistÚ	time_betaÚtime_expandedÚnoisy_actionsÚtarget_velocityr  ÚoutputsÚlast_hidden_statesÚpredicted_velocityrb  s                             r/   rü   z#PI0ForConditionalGeneration.forwardì  s)  € ð: ”[ ”^ˆ
ð ÐÝ”l 4¤;Ô#GÍuÌ}Ð]Ñ]Ô]ˆGÝ”\ $¤+Ô"EÍUÌ]Ð[Ñ[Ô[ˆFÝÔ&×+Ò+¨G°VÑ<Ô<ˆDØŸš Z MÑ2Ô2×5Ò5°e´lÑCÔCˆIØ! D¤KÔ$CÑCÀdÄkÔFfÑf×mÒmÑoÔoˆHð ˆ=Ý”KØØ”Ô&Ø”Ô*Ø”|Ø”kðñ ô ˆEð ÐØ$ Q Q Q¨¨d ]Ô3ˆMØ*¨UÑ2°a¸-Ñ6GÈ7Ñ5RÑR×VÒVÐW^ÔWdÑeÔeˆMØ# g™oˆOˆOà!ˆMð "×3Ò3°E¸=È(ÑSÔSÐà�$”*ð 

ØØ%Ø)Ø!5Ø%Ø'Ø,Ø+ð

ð 

ð ð

ð 

ˆð %Ô6°q°q°q¸4¼;Ô;QÐ:QÐ:SÐ:SÐ7SÔTÐØ!×1Ò1Ð2DÑEÔEÐàˆØÐå”:˜oÐ/AÈTÌ[ÔMgÐhÑhÔhˆDå%ØØ%Ø#Ô3Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r.   Ú	num_stepsc                 óä  — |p| j         j        }|j        d         }	|j        }
|€5t	          j        dd|	| j         j        | j         j        f|j        |
¬¦  «        }d}|�| 	                    d¦  «        dz
  }| j
                             |||¦  «        }t	          j        |¦  «        dd…dd…df         }| j
                             ||||dd¬	¦  «        j        }|                     ¦   «         }d
|z  }t!          |¦  «        D ]o}d||z  z   }t	          j        |t          j        |
¬¦  «                             |	¦  «        } | d||||||dœ|¤Ž}|                     |¦  «         |||j        z  z   }Œp|S )z0Run flow matching inference to generate actions.r   Nræ   r¡   )r§   Ústdr*   rf   rô   ri   r¯   T)rA  r;  rD  rF  rG  Úreturn_dictg      ð¿rH  )rX   r  r  rh   r;  r  r-   )rÊ   rŸ   r}   rô   r>   ÚnormalrI   r�   rf   rK  r  rC  rL  r+  r  rO  ÚrangerL   rè   ÚexpandÚcroprc  )rO   rX   r:  rg   r  r;  rh   rv  rS   rk  rô   rD  rA  rF  r  Úprefix_lengthÚdtÚsteprù   Útime_tensorÚoutputs                        r/   Úsample_actionsz*PI0ForConditionalGeneration.sample_actionsD  sÒ  € ð Ð@ ¤Ô!@ˆ	Ø”_ QÔ'ˆ
ØÔ!ˆð ˆ=Ý”LØØàØ”KÔ*Ø”KÔ.ðð
 #Ô(Øð
ñ 
ô 
ˆEð ˆØÐ%Ø)×0Ò0°Ñ4Ô4°qÑ8ˆLØœ
×/Ò/°	¸<ÐI]Ñ^Ô^ˆÝÔ)¨-Ñ8Ô8¸¸¸¸A¸A¸A¸q¸ÔAˆØœ*Ÿ.š.Ø'Ø)Ø%Ø)ØØð )ñ 
ô 
ô ð 	ð (×6Ò6Ñ8Ô8ˆð �IÑˆÝ˜)Ñ$Ô$ð 	/ð 	/ˆDØ˜ ™‘?ˆDÝœ, tµ5´=ÈÐPÑPÔP×WÒWÐXbÑcÔcˆKØ�Tð ØØØ$Ø%9Ø-Ø /ðð ð ðð ˆFð × Ò  Ñ/Ô/Ð/Ø˜B ¤Ñ.Ñ.ˆEˆEØˆr.   )
NNNNNNNNNN)NNNN)r'   r(   r)   rÍ   Ú_tp_planr™   rN   r   r   r>   ÚFloatTensorr“   Ú
BoolTensorrX  r   r   rü   Úno_gradrÐ   rƒ  r•   r–   s   @r/   rZ  rZ  ß  s4  ø€ € € € € ØCÐCà!Ð#:Ð;€Hð˜yð ð ð ð ð ð ð Øð +/Ø-1Ø)-Ø,0Ø8<Ø.2Ø04Ø-1Ø(,Ø%)ðT
ð T
àÔ ðT
ð Ô  4Ñ'ðT
ð Ô# dÑ*ð	T
ð
 ”< $Ñ&ðT
ð ”l TÑ)ðT
ð $Ô.°Ñ5ðT
ð œ tÑ+ðT
ð Ô&¨Ñ-ðT
ð ”| dÑ*ðT
ð  ™ðT
ð Ô"ðT
ð 
 ðT
ð T
ð T
ñ „^ñ ÔðT
ðl €U„]�_„_ð +/Ø.2Ø8<Ø $ðAð AàÔ ðAð Ô#ðAð Ô'ð	Að
 Ô  4Ñ'ðAð œ tÑ+ðAð $Ô.°Ñ5ðAð ˜‘:ðAð 
Ô	ðAð Að Añ „_ðAð Að Að Að Ar.   rZ  )r™   r  r'  rZ  rA   r!   )FrÍ   ré   Úcollections.abcr   Únumpyr‘   r>   Útorch.nn.functionalr   Ú
functionalr~   Úhuggingface_hub.dataclassesr   r\   r   r  Úcache_utilsr   Úconfiguration_utilsr	   Úfeature_extraction_utilsr
   Úimage_utilsr   r   Úmasking_utilsr   Úmodeling_outputsr   r   Úmodeling_utilsr   Úprocessing_utilsr   r   Útokenization_utils_baser   r   Úutilsr   r   r   Úutils.genericr   Úutils.import_utilsr   Úautor   r   r   Úpaligemma.processing_paligemmar   Úsiglip.image_processing_siglipr   Ú
get_loggerr'   rn   r!   r1   rA   r™   r“   rÝ   ÚModulerß   rÿ   r  r'  rZ  Ú__all__r-   r.   r/   ú<module>rŸ     sO  ðð [Ð Zà €€€Ø $Ð $Ð $Ð $Ð $Ð $à Ð Ð Ð Ø €€€Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø .Ð .Ð .Ð .Ð .Ð .Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø  Ð  Ð  Ð  Ð  Ð  Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø 4Ð 4Ð 4Ð 4Ð 4Ð 4Ø AÐ AÐ AÐ AÐ AÐ AÐ AÐ AØ /Ð /Ð /Ð /Ð /Ð /Ø OÐ OÐ OÐ OÐ OÐ OÐ OÐ OØ -Ð -Ð -Ð -Ð -Ð -Ø 8Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8Ø CÐ CÐ CÐ CÐ CÐ CÐ CÐ CØ >Ð >Ð >Ð >Ð >Ð >Ð >Ð >Ð >Ð >Ø +Ð +Ð +Ð +Ð +Ð +Ø *Ð *Ð *Ð *Ð *Ð *Ø 8Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8Ø ?Ð ?Ð ?Ð ?Ð ?Ð ?Ø AÐ AÐ AÐ AÐ AÐ Að 
ˆÔ	˜HÑ	%Ô	%€ð ðð ð ð ð Ð,ñ ô ñ „ððð ð ð ð Ð)°ð ñ ô ð ð Ø	€Ð&Ð'Ñ'Ô'ðkDð kDð kDð kDð kDÐ%ñ kDô kDñ (Ô'ñ „ðkDð\ €Ð-Ð.Ñ.Ô.Øðlnð lnð lnð lnð lnÐ ñ lnô lnñ „ñ /Ô.ðlnð^°5´<ð ÀHð ð ð ð ðð ð ð ð ˜BœIñ ô ð ð.$ð $ð $ð $ð $˜RœYñ $ô $ð $ð, ðRð Rð Rð Rð R˜ñ Rô Rñ „ðRð& ðpð pð pð pð pÐ!ñ pô pñ „ðpðfgð gð gð gð gÐ"4ñ gô gð gðTð ð €€€r.   