§
    ‚Štj:8  ã                   ó  — d Z ddlZddlmZ ddlmZmZ ddlZddl	m
Z
 ddlmZmZ ddlmZmZmZmZ dd	lmZmZmZ dd
lmZmZ erddlmZ  ej        e¦  «        Z G d„ ded¬¦  «        Ze G d„ de¦  «        ¦   «         ZdgZ dS )z
Processor class for Idefics3.
é    N)Ú
accumulate)ÚTYPE_CHECKINGÚUnioné   )ÚBatchFeature)Ú
ImageInputÚis_valid_image)ÚMultiModalDataÚProcessingKwargsÚProcessorMixinÚUnpack)Ú
AddedTokenÚBatchEncodingÚ	TextInput)Úauto_docstringÚlogging)ÚPreTokenizedInputc                   ó&   — e Zd ZdddddœddidœZdS )ÚIdefics3ProcessorKwargsTF)Úadd_special_tokensÚpaddingÚis_split_into_wordsÚreturn_mm_token_type_idsÚreturn_row_col_info)Útext_kwargsÚimages_kwargsN)Ú__name__Ú
__module__Ú__qualname__Ú	_defaults© ó    ún/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/idefics3/processing_idefics3.pyr   r   %   s>   € € € € € ð #'ØØ#(Ø(-ð	
ð 
ð " 4ð
ð
ð 
€I€I€Ir"   r   F)Útotalc                   óà  ‡ — e Zd ZeZ	 ddededz  fˆ fd„Ze	 	 	 dde	e
e	         z  e
e
e	                  z  deede
e         e
d         f         dedz  d	ee         d
ef
d„¦   «         Z	 	 dde	dz  deede
e         e
d         f         d	ee         fd„Z	 	 dde	dz  deede
e         e
d         f         d	ee         fˆ fd„Zdeded
efd„Zde
de
e         d
e
e
e                  fd„Zdd„Zˆ xZS )ÚIdefics3ProcessorNé©   Úimage_seq_lenÚchat_templatec                 óÈ  •‡— t          ddd¬¦  «        j        | _        t          ddd¬¦  «        j        | _        t          ddd¬¦  «        j        | _        d| _        || _        ‰                     | j        ¦  «        | _        ‰                     | j        ¦  «        | _	        ‰                     | j        ¦  «        | _
        ˆfd„t          d	¦  «        D ¦   «         | _        t          j        d
¦  «        | _        d| j        | j        | j        gi}‰                     |¦  «         ‰                     | j        ¦  «        | _         t#          ¦   «         j        |‰fd|i|¤Ž dS )aŸ  
        image_seq_len (`int`, *optional*, defaults to 169):
            The length of the image sequence i.e. the number of <image> tokens per image in the input.
            This parameter is used to build the string from the input prompt and image tokens and should match the
            value the model used. It is computed as: image_seq_len = int(((image_size // patch_size) ** 2) / (scale_factor**2))
        z<fake_token_around_image>FT)Ú
normalizedÚspecialz<image>z<end_of_utterance>z<global-img>c           
      óx   •— g | ]6}t          d ¦  «        D ]$}‰                     d|dz   › d|dz   › d�¦  «        ‘Œ%Œ7S )é   ú<row_é   Ú_col_ú>)ÚrangeÚconvert_tokens_to_ids)Ú.0ÚiÚjÚ	tokenizers      €r#   ú
<listcomp>z.Idefics3Processor.__init__.<locals>.<listcomp>H   sn   ø€ ð 
ð 
ð 
ØNOÕejÐklÑemÔemð
ð 
Ø`aˆI×+Ò+Ð,H°A¸±EÐ,HÐ,HÀÀAÁÐ,HÐ,HÐ,HÑIÔIð
ð 
ð 
ð 
r"   r.   z*(\n?<global-img>\n?|<row_\d+_col_\d+>\n?)+Úadditional_special_tokensr)   N)r   ÚcontentÚfake_image_tokenÚimage_tokenÚend_of_utterance_tokenÚglobal_image_tagr(   r4   Úimage_token_idÚfake_image_token_idÚglobal_image_token_idr3   Úrow_col_idsÚreÚcompileÚ%_regex_to_remove_extra_special_tokensr   ÚsuperÚ__init__)ÚselfÚimage_processorr8   r(   r)   ÚkwargsÚtokens_to_addÚ	__class__s     `    €r#   rH   zIdefics3Processor.__init__7   s}  øø€ õ !+Ð+FÐSXÐbfÐ gÑ gÔ gÔ oˆÔÝ% i¸EÈ4ÐPÑPÔPÔXˆÔÝ&0Ð1EÐRWÐaeÐ&fÑ&fÔ&fÔ&nˆÔ#Ø .ˆÔØ*ˆÔØ'×=Ò=¸dÔ>NÑOÔOˆÔØ#,×#BÒ#BÀ4ÔCXÑ#YÔ#YˆÔ Ø%.×%DÒ%DÀTÔEZÑ%[Ô%[ˆÔ"ð
ð 
ð 
ð 
ÝSXÐYZÑS[ÔS[ð
ñ 
ô 
ˆÔõ 68´ZÐ@mÑ5nÔ5nˆÔ2ð (ØÔ%ØÔ ØÔ+ð*ð
ˆð 	×$Ò$ ]Ñ3Ô3Ð3Ø'×=Ò=¸dÔ>NÑOÔOˆÔà�‰ŒÔ˜¨)Ð[Ð[À=Ð[ÐTZÐ[Ð[Ð[Ð[Ð[r"   ÚimagesÚtextr   rK   Úreturnc                 ó$  —  | j         d||dœ|¤Ž\  }} | j        d||dœ|¤Ž  | j        t          fd| j        j        i|¤Ž}|�|n| j        }|d                              dd¦  «        }|d                              dd¦  «        }|d                              dd¦  «        }i x}	}
|��= | j        |fi |d	         ¤Ž\  }	}|	                     d
d¦  «         |	                     dd¦  «         |�÷|  	                    ||¬¦  «        \  }} | j        |fi |d         ¤Ž}
|r||
d<   g }t          |¦  «        D ]v\  }}g }|D ]W}|d         \  }}|
                     ||¦  «        }|
                     ||dz
  ¦  «        }|                     ||z
  dz   ¦  «         ŒX|                     |¦  «         Œw|r|                      |
d         |¦  «        |
d<   |                      ||
dg¬¦  «         n|� | j        dd|i|d         ¤Ž}
t          i |
¥|	¥|¬¦  «        S )a  
        image_seq_len (`int`, *optional*):
            The length of the image sequence. If not provided, the default value of self.image_seq_len is used.
            image_seq_len should be equal to int(((image_size // patch_size) ** 2) / (scale_factor**2))
        )rN   rO   Útokenizer_init_kwargsNr   Úreturn_text_replacement_offsetsFr   Úreturn_tensorsr   ÚrowsÚcols)Úimages_replacementsÚtext_replacement_offsetsÚnew_spanr0   Ú	input_idsÚmm_token_type_idsÚimage)Ú
modalitiesrO   )ÚdataÚtensor_typer!   )Úprepare_inputs_layoutÚvalidate_inputsÚ_merge_kwargsr   r8   Úinit_kwargsr(   ÚpopÚ_process_imagesÚget_text_with_replacementsÚ	enumerateÚchar_to_tokenÚappendÚcreate_mm_token_type_idsÚ_check_special_mm_tokensr   )rI   rN   rO   r(   rK   Úoutput_kwargsrS   r   rT   Úimage_inputsÚtext_inputsrW   rX   Úbatch_image_seq_lengthsÚbatch_idÚtext_replacement_offsetÚimage_seq_lensr^   ÚstartÚendÚstart_id_posÚ
end_id_poss                         r#   Ú__call__zIdefics3Processor.__call__\   sê  € ð 2�tÔ1ÐU¸ÀdÐUÐUÈfÐUÐU‰ˆ�ØˆÔÐ@ F°Ð@Ð@¸Ð@Ð@Ð@à*˜Ô*Ý#ð
ð 
à"&¤.Ô"<ð
ð ð
ð 
ˆð *7Ð)B˜˜ÈÔHZˆØ*7¸Ô*F×*JÒ*JÐKlÐnsÑ*tÔ*tÐ'Ø#0°Ô#?×#CÒ#CÐD^Ð`eÑ#fÔ#fÐ Ø& }Ô5×9Ò9Ð:JÈDÑQÔQˆà%'Ð'ˆ�{ØÑØ0D°Ô0DÀVÐ0nÐ0nÈ}Ð]lÔOmÐ0nÐ0nÑ-ˆLÐ-ð ×Ò˜V TÑ*Ô*Ð*Ø×Ò˜V TÑ*Ô*Ð*àÐØ15×1PÒ1PØÐ.Að 2Qñ 2ô 2Ñ.�Ð.ð -˜dœn¨TÐRÐR°]À=Ô5QÐRÐR�Ø2ð WØ>V�KÐ :Ñ;à*,Ð'Ý9BÐC[Ñ9\Ô9\ð Cð CÑ5�HÐ5Ø%'�NØ 7ð Mð M˜Ø%)¨*Ô%5™
˜˜sØ'2×'@Ò'@ÀÈ5Ñ'QÔ'Q˜Ø%0×%>Ò%>¸xÈÈqÉÑ%QÔ%Q˜
à&×-Ò-¨j¸<Ñ.GÈ!Ñ.KÑLÔLÐLÐLØ+×2Ò2°>ÑBÔBÐBÐBà+ð Ø7;×7TÒ7TØ# KÔ0Ð2Iñ8ô 8�KÐ 3Ñ4ð ×-Ò-¨d°KÈWÈIÐ-ÑVÔVÐVøàÐØ(˜$œ.ÐSÐS¨dÐS°mÀMÔ6RÐSÐSˆKåÐ!@ KÐ!@°<Ð!@ÈnÐ]Ñ]Ô]Ð]r"   c                 óH  ‡ ‡‡— |�,t          |t          ¦  «        r|g}|                     ¦   «         }‰�ì‰ j                             ‰¦  «        Št          ‰¦  «        r‰ggŠn¾t          ‰t          t          f¦  «        r¢t          ‰d         ¦  «        r�|�ˆˆ fd„|D ¦   «         }dgt          t          |¦  «        ¦  «        z   Šˆˆfd„t          t          |¦  «        ¦  «        D ¦   «         }t          ‰¦  «        ‰d         k    r|‰‰d         d …         gz   Šn|Šn‰gŠ‰|fS )Nr   c                 óD   •— g | ]}|                      ‰j        ¦  «        ‘ŒS r!   ©Úcountr=   ©r5   ÚsamplerI   s     €r#   r9   z;Idefics3Processor.prepare_inputs_layout.<locals>.<listcomp>¯   s(   ø€ Ð'ZÐ'ZÐ'ZÈ6¨¯ª°TÔ5EÑ(FÔ(FÐ'ZÐ'ZÐ'Zr"   c                 óB   •— g | ]}‰‰|         ‰|d z            …         ‘ŒS )r0   r!   )r5   r6   Úcumsum_images_in_textrN   s     €€r#   r9   z;Idefics3Processor.prepare_inputs_layout.<locals>.<listcomp>±   sF   ø€ ð $ð $ð $àð Ð4°QÔ7Ð:OÐPQÐTUÑPUÔ:VÐVÔWð$ð $ð $r"   éÿÿÿÿ)Ú
isinstanceÚstrÚcopyrJ   Úfetch_imagesr	   ÚlistÚtupler   r3   Úlen)rI   rN   rO   rK   Ún_images_in_textÚsplit_imagesr   s   ``    @r#   r`   z'Idefics3Processor.prepare_inputs_layout�   sa  øøø€ ð ÐÝ˜$¥Ñ$Ô$ð Ø�v�Ø—9’9‘;”;ˆDàÐØÔ)×6Ò6°vÑ>Ô>ˆFÝ˜fÑ%Ô%ð &Ø!˜(˜��Ý˜F¥T­5 MÑ2Ô2ð &µ~ÀfÈQÄiÑ7PÔ7Pð &ØÐ#à'ZÐ'ZÐ'ZÐ'ZÐUYÐ'ZÑ'ZÔ'ZÐ$Ø-.¨Cµ$µzÐBRÑ7SÔ7SÑ2TÔ2TÑ,TÐ)ð$ð $ð $ð $ð $å!&¥sÐ+;Ñ'<Ô'<Ñ!=Ô!=ð$ñ $ô $�Lõ
 ˜6‘{”{Ð%:¸2Ô%>Ò>Ð>Ø!-°Ð8MÈbÔ8QÐ8SÐ8SÔ1TÐ0UÑ!U˜˜à!-˜˜à$˜X�Fà�tˆ|Ðr"   c                 óx  •‡ —  t          ¦   «         j        ||fi |¤Ž |€|€t          d¦  «        ‚|�ƒˆ fd„|D ¦   «         }|�:d„ |D ¦   «         }||k    r&t          d‰ j        › d|› d‰ j        › d|› d�	¦  «        ‚d S |€9t	          |¦  «        r,t          d	t          |¦  «        › d‰ j        › d
�¦  «        ‚d S d S d S )Nz+You must provide either `text` or `images`.c                 óD   •— g | ]}|                      ‰j        ¦  «        ‘ŒS r!   rz   r|   s     €r#   r9   z5Idefics3Processor.validate_inputs.<locals>.<listcomp>Ë   s(   ø€ ÐRÐRÐRÀ6 §¢¨TÔ-=Ñ >Ô >ÐRÐRÐRr"   c                 ó,   — g | ]}t          |¦  «        ‘ŒS r!   )r‡   )r5   Úsublists     r#   r9   z5Idefics3Processor.validate_inputs.<locals>.<listcomp>Í   s   € Ð%IÐ%IÐ%I°w¥c¨'¡l¤lÐ%IÐ%IÐ%Ir"   zThe total number of zP tokens in the prompts should be the same as the number of images passed. Found ú z tokens and z images per sample.zFound z. tokens in the text but no images were passed.)rG   ra   Ú
ValueErrorr=   ÚanyÚsum)rI   rN   rO   rK   rˆ   Ún_images_in_imagesrM   s   `     €r#   ra   z!Idefics3Processor.validate_inputs¿   sG  øø€ ð 	 �‰ŒÔ ¨Ð7Ð7°Ð7Ð7Ð7àˆ<˜F˜NÝÐJÑKÔKÐKàÐØRÐRÐRÐRÈTÐRÑRÔRÐØÐ!Ø%IÐ%IÀ&Ð%IÑ%IÔ%IÐ"Ø#Ð'9Ò9Ð9Ý$ð{¨tÔ/?ð {ð {Ø"2ð{ð {Ø59Ô5Eð{ð {ØSeð{ð {ð {ñô ð ð :Ð9ð
 �¥CÐ(8Ñ$9Ô$9�Ý Øu�SÐ!1Ñ2Ô2ÐuÐu°TÔ5EÐuÐuÐuñô ð ð Ðð  ���r"   rm   Ú	image_idxc           	      óâ  — d„ |d         D ¦   «         |         }d„ |d         D ¦   «         |         }|dk    r1|dk    r+| j         › | j        › z   | j        › | j        z  z   | j         › z   S d}t	          |¦  «        D ]E}t	          |¦  «        D ].}|| j         › d|dz   › d	|dz   › d
�z   | j        › | j        z  z   z  }Œ/|dz  }ŒF|d| j         › �| j        › z   | j        › | j        z  z   | j         › z   z  }|S )Nc                 ó   — g | ]	}|D ]}|‘ŒŒ
S r!   r!   )r5   Úrow_listÚrows      r#   r9   z9Idefics3Processor.replace_image_token.<locals>.<listcomp>Ù   ó%   € ÐSÐSÐS˜hÈ(ÐSÐSÀ3�cÐSÐSÐSÐSr"   rU   c                 ó   — g | ]	}|D ]}|‘ŒŒ
S r!   r!   )r5   Úcol_listÚcols      r#   r9   z9Idefics3Processor.replace_image_token.<locals>.<listcomp>Ú   r˜   r"   rV   r   Ú r/   r0   r1   r2   ú
)r<   r?   r=   r(   r3   )rI   rm   r“   Ú
image_rowsÚ
image_colsÚtext_split_imagesÚn_hÚn_ws           r#   Úreplace_image_tokenz%Idefics3Processor.replace_image_tokenØ   s}  € ØSÐS¨,°vÔ*>ÐSÑSÔSÐT]Ô^ˆ
ØSÐS¨,°vÔ*>ÐSÑSÔSÐT]Ô^ˆ
Ø˜Š?ˆ?˜z¨Qš˜àÔ(Ð*ØÔ*Ð,ñ-àÔ%Ð'¨$Ô*<Ñ<ñ=ð Ô*Ð,ñ-ðð !#ÐÝ˜ZÑ(Ô(ð *ð *�Ý  Ñ,Ô,ð ð �CØ%ØÔ0Ð2Ø: #¨¡'Ð:Ð:°°a±Ð:Ð:Ð:ñ;à!Ô-Ð/°$Ô2DÑDñEñÐ%Ð%ð
 " TÑ)Ð!Ð!àØ,�TÔ*Ð,Ð,ØÔ*Ð,ñ-àÔ%Ð'¨$Ô*<Ñ<ñ=ð Ô*Ð,ñ-ñÐð %Ð$r"   rZ   ro   c                 ó®  — g }t          |¦  «        D ]Â\  }}t          j        ||         ¦  «        }t          j        |¦  «        }t          j        || j        k    ¦  «        d         }d}	|D ]@}
|	t          |¦  «        k    r n*||	         }||
z   }d|||…<   t          j        ||¦  «        }	ŒA|                     | 	                    ¦   «         ¦  «         ŒÃ|S )Nr   r0   )
rg   ÚnpÚarrayÚ
zeros_likeÚwhererA   r‡   Úsearchsortedri   Útolist)rI   rZ   ro   r[   r6   Úseq_lengthsÚ	array_idsÚmm_token_typesÚimage_start_positionsr7   Úseq_lenrs   rt   s                r#   rj   z*Idefics3Processor.create_mm_token_type_idsõ   só   € ð ÐÝ'Ð(?Ñ@Ô@ð 	>ð 	>‰NˆAˆ{Ýœ ¨1¤Ñ.Ô.ˆIÝœ]¨9Ñ5Ô5ˆNÝ$&¤H¨Y¸$Ô:RÒ-RÑ$SÔ$SÐTUÔ$VÐ!ØˆAØ&ð @ð @�Ø�Ð1Ñ2Ô2Ò2Ð2Ø�EØ-¨aÔ0�Ø˜g‘o�Ø,-�˜u S˜yÑ)Ý”OÐ$9¸3Ñ?Ô?��Ø×$Ò$ ^×%:Ò%:Ñ%<Ô%<Ñ=Ô=Ð=Ð=à Ð r"   c                 ó  ‡ ‡— i }|�ït           j                             di ¦  «        Š‰                     |¦  «         ˆˆ fd„|D ¦   «         }‰ j        dz   }‰ j        dz   }t          ‰                      dd¬¦  «        d	         ¦  «        d
z
  }g }g }	|D ]K\  }
}}||z  d
z   }|dk    r|nd}|                     ||z   ||z  z   ¦  «         |	                     |
¦  «         ŒL|                     ||	dœ¦  «         t          di |¤ŽS )a»  
        Computes the number of placeholder tokens needed for multimodal inputs with the given sizes.

        Args:
            image_sizes (`list[list[int]]`, *optional*):
                The input sizes formatted as (height, width) per each image.

        Returns:
            `MultiModalData`: A `MultiModalData` object holding number of tokens per each of the provided
            input modalities, along with other useful data.
        Nr   c                 ó8   •— g | ]} ‰j         j        g |¢‰‘R Ž ‘ŒS r!   )rJ   Úget_number_of_image_patches)r5   Ú
image_sizer   rI   s     €€r#   r9   z@Idefics3Processor._get_num_multimodal_tokens.<locals>.<listcomp>  sE   ø€ ð "ð "ð "àð A�Ô$Ô@Ð\À*Ð\ÈmÐ\Ð\Ð\ð"ð "ð "r"   r   é   z

F)r   rZ   r0   r   )Únum_image_tokensÚnum_image_patchesr!   )	r   r    ÚgetÚupdater(   r‡   r8   ri   r
   )rI   Úimage_sizesrK   Úvision_dataÚnum_image_row_colsÚbase_image_lengthÚ
col_lengthÚextra_split_newlinerµ   r¶   Únum_patchesÚnum_rowsÚnum_colsÚ
row_lengthÚsplit_extrar   s   `              @r#   Ú_get_num_multimodal_tokensz,Idefics3Processor._get_num_multimodal_tokens
  sh  øø€ ð ˆØÐ"Ý3Ô=×AÒAÀ/ÐSUÑVÔVˆMØ× Ò  Ñ(Ô(Ð(ð"ð "ð "ð "ð "à"-ð"ñ "ô "Ðð
 !%Ô 2°QÑ 6ÐØÔ+¨aÑ/ˆJõ
 #& d§n¢n°VÐPU nÑ&VÔ&VÐWbÔ&cÑ"dÔ"dÐghÑ"hÐØ!ÐØ "Ðà3Eð 6ð 6Ñ/�˜X xØ'¨(Ñ2°QÑ6�
Ø5=À²\°\Ð1Ð1Àq�Ø ×'Ò'Ð(9¸KÑ(GÈ:ÐX`ÑK`Ñ(aÑbÔbÐbØ!×(Ò(¨Ñ5Ô5Ð5Ð5à×ÒÐ4DÐ[lÐmÐmÑnÔnÐnåÐ,Ð, Ð,Ð,Ð,r"   )Nr'   N)NNN)NN)N)r   r   r   r   Úvalid_processor_kwargsÚintr‚   rH   r   r   r…   r   r   r   r   rw   r`   r   ra   Údictr£   rj   rÄ   Ú__classcell__)rM   s   @r#   r&   r&   3   s_  ø€ € € € € à4Ðð fjð#\ð #\Ø>Að#\ØX[Ð^bÑXbð#\ð #\ð #\ð #\ð #\ð #\ðJ ð JNØbfØ$(ð	>^ð >^à˜T *Ô-Ñ-°°T¸*Ô5EÔ0FÑFð>^ð �IÐ2°D¸´OÀTÐJ]ÔE^Ð^Ô_ð>^ð ˜T‘zð	>^ð
 Ð0Ô1ð>^ð 
ð>^ð >^ð >^ñ „^ð>^ðD %)Øbfð ð  à˜TÑ!ð ð �IÐ2°D¸´OÀTÐJ]ÔE^Ð^Ô_ð ð Ð0Ô1ð	 ð  ð  ð  ðH %)Øbfðð à˜TÑ!ðð �IÐ2°D¸´OÀTÐJ]ÔE^Ð^Ô_ðð Ð)Ô*ð	ð ð ð ð ð ð2%°ð %Àð %Èð %ð %ð %ð %ð:!°$ð !ÐQUÐVYÔQZð !Ð_cÐdhÐilÔdmÔ_nð !ð !ð !ð !ð*)-ð )-ð )-ð )-ð )-ð )-ð )-ð )-r"   r&   )!Ú__doc__rD   Ú	itertoolsr   Útypingr   r   Únumpyr¥   Úfeature_extraction_utilsr   Úimage_utilsr   r	   Úprocessing_utilsr
   r   r   r   Útokenization_utils_baser   r   r   Úutilsr   r   r   Ú
get_loggerr   Úloggerr   r&   Ú__all__r!   r"   r#   ú<module>rÕ      s�  ððð ð 
€	€	€	Ø  Ð  Ð  Ð  Ð  Ð  Ø 'Ð 'Ð 'Ð 'Ð 'Ð 'Ð 'Ð 'à Ð Ð Ð à 4Ð 4Ð 4Ð 4Ð 4Ð 4Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø XÐ XÐ XÐ XÐ XÐ XÐ XÐ XÐ XÐ XÐ XÐ XØ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,ð ð =Ø<Ð<Ð<Ð<Ð<Ð<à	ˆÔ	˜HÑ	%Ô	%€ðð ð ð ð Ð.°eð ñ ô ð ð ð-ð -ð -ð -ð -˜ñ -ô -ñ „ð-ðD Ð
€€€r"   