§
    ‚Štj+¾  ã                   óÎ   — d dl mZmZmZmZ d dlmZ ddlmZm	Z	m
Z
mZmZmZmZmZmZmZ ddlmZ ddlmZmZ  ej        e¦  «        Zddd	d
œZdZ G d„ de¦  «        ZeZddgZdS )é    )Ú	TokenizerÚdecodersÚpre_tokenizersÚ
processors)ÚBPEé   )
ÚENCODE_KWARGS_DOCSTRINGÚ
AddedTokenÚBatchEncodingÚEncodedInputÚPaddingStrategyÚPreTokenizedInputÚ
TensorTypeÚ	TextInputÚTextInputPairÚTruncationStrategy)ÚTokenizersBackend)Úadd_end_docstringsÚloggingz
vocab.jsonz
merges.txtztokenizer.json)Ú
vocab_fileÚmerges_fileÚtokenizer_filea<  
            add_special_tokens (`bool`, *optional*, defaults to `True`):
                Whether or not to encode the sequences with the special tokens relative to their model.
            padding (`bool`, `str` or [`~tokenization_utils_base.PaddingStrategy`], *optional*, defaults to `False`):
                Activates and controls padding. Accepts the following values:

                - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
                  sequence if provided).
                - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
                  acceptable input length for the model if that argument is not provided.
                - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
                  lengths).
            truncation (`bool`, `str` or [`~tokenization_utils_base.TruncationStrategy`], *optional*, defaults to `False`):
                Activates and controls truncation. Accepts the following values:

                - `True` or `'longest_first'`: Truncate to a maximum length specified with the argument `max_length` or
                  to the maximum acceptable input length for the model if that argument is not provided. This will
                  truncate token by token, removing a token from the longest sequence in the pair if a pair of
                  sequences (or a batch of pairs) is provided.
                - `'only_first'`: Truncate to a maximum length specified with the argument `max_length` or to the
                  maximum acceptable input length for the model if that argument is not provided. This will only
                  truncate the first sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
                - `'only_second'`: Truncate to a maximum length specified with the argument `max_length` or to the
                  maximum acceptable input length for the model if that argument is not provided. This will only
                  truncate the second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
                - `False` or `'do_not_truncate'` (default): No truncation (i.e., can output batch with sequence lengths
                  greater than the model maximum admissible input size).
            max_length (`int`, *optional*):
                Controls the maximum length to use by one of the truncation/padding parameters. If left unset or set to
                `None`, this will use the predefined model maximum length if a maximum length is required by one of the
                truncation/padding parameters. If the model has no specific maximum input length (like XLNet)
                truncation/padding to a maximum length will be deactivated.
            stride (`int`, *optional*, defaults to 0):
                If set to a number along with `max_length`, the overflowing tokens returned when
                `return_overflowing_tokens=True` will contain some tokens from the end of the truncated sequence
                returned to provide some overlap between truncated and overflowing sequences. The value of this
                argument defines the number of overlapping tokens.
            is_split_into_words (`bool`, *optional*, defaults to `False`):
                Whether or not the input is already pretokenized (e.g. split into words). Set this to `True` if you are
                passing pretokenized inputs to avoid additional tokenization.
            pad_to_multiple_of (`int`, *optional*):
                If set will pad the sequence to a multiple of the provided value. This is especially useful to enable
                the use of Tensor Cores on NVIDIA hardware with compute capability `>= 7.5` (Volta).
            return_tensors (`str` or [`~tokenization_utils_base.TensorType`], *optional*):
                If set, will return tensors instead of list of python integers. Acceptable values are:

                - `'pt'`: Return PyTorch `torch.Tensor` objects.
                - `'np'`: Return Numpy `np.ndarray` objects.
c            ,       óZ  ‡ — e Zd ZdZeZg d¢ZeZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d>de	e
e	ef         z  eee	ef                  z  dz  de	ee	         z  dz  fˆ fd„Zd„ Z eee¦  «        	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d?deez  ee         z  ee         z  deee         z  dz  deee                  eeee                           z  dz  dee         eee                  z  dz  dedee	z  ez  dee	z  ez  dedz  dedededz  d e	dz  d!e	ez  dz  d"edz  d#edz  d$ed%ed&ed'ed(ed)ef*d*„¦   «         Z eee¦  «        	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d@d+ee         ee         z  ee         z  d,edz  deeee                           dz  dee         eee                  z  dz  dedee	z  ez  dee	z  ez  dedz  dededz  d e	dz  d!e	ez  dz  d"edz  d#edz  d$ed%ed&ed'ed(ed)ef(d-„¦   «         ZdAde	d.e	dz  ded)ee	         fd/„Z eee¦  «        	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d@deez  dedz  deee                  dz  dee         dz  dedee	z  ez  dee	z  ez  dedz  dededz  d e	dz  d!e	ez  dz  d"edz  d#edz  d$ed%ed&ed'ed(ed)ef(d0„¦   «         Zddddej         ej!        dddddddd
d
d
d
dfd+ee         ee         z  ee         z  d,edz  deeee                           dz  deee                  dz  ded1ed2ededz  dededz  d e	dz  d!e	dz  d"edz  d#edz  d$ed%ed&ed'ed(ed)ef(d3„Z"ddddej         ej!        dddddddd
d
d
d
dfdeez  dedz  deee                  dz  dee         dz  ded1ed2ededz  dededz  d e	dz  d!edz  d"edz  d#edz  d$ed%ed&ed'ed(ed)ef(d4„Z#dej         dddfd5e
e	e$f         ez  dedz  d1ededz  d e	dz  d#edz  d)e
fd6„Z%	 dBd7ee         d8ee         dz  d)ee         fd9„Z&	 dBd7ee         d8ee         dz  d)ee         fd:„Z'dBd;e	d<e	dz  d)ee	         fd=„Z(ˆ xZ)S )CÚMarkupLMTokenizeraé  
    Construct a MarkupLM tokenizer. Based on byte-level Byte-Pair-Encoding (BPE).

    [`MarkupLMTokenizer`] can be used to turn HTML strings into to token-level `input_ids`, `attention_mask`,
    `token_type_ids`, `xpath_tags_seq` and `xpath_tags_seq`. This tokenizer inherits from [`TokenizersBackend`] which
    contains most of the main methods and ensures a `tokenizers` backend is always instantiated.

    Users should refer to this superclass for more information regarding those methods.

    Args:
        vocab (`str` or `dict[str, int]`, *optional*):
            Custom vocabulary dictionary. If not provided, the vocabulary is loaded from `vocab_file`.
        merges (`str` or `list[str]`, *optional*):
            Custom merges list. If not provided, merges are loaded from `merges_file`.
        errors (`str`, *optional*, defaults to `"replace"`):
            Paradigm to follow when decoding bytes to UTF-8. See
            [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
        bos_token (`str`, *optional*, defaults to `"<s>"`):
            The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.

            <Tip>

            When building a sequence using special tokens, this is not the token that is used for the beginning of
            sequence. The token used is the `cls_token`.

            </Tip>

        eos_token (`str`, *optional*, defaults to `"</s>"`):
            The end of sequence token.

            <Tip>

            When building a sequence using special tokens, this is not the token that is used for the end of sequence.
            The token used is the `sep_token`.

            </Tip>

        sep_token (`str`, *optional*, defaults to `"</s>"`):
            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
            sequence classification or for a text and a question for question answering. It is also used as the last
            token of a sequence built with special tokens.
        cls_token (`str`, *optional*, defaults to `"<s>"`):
            The classifier token which is used when doing sequence classification (classification of the whole sequence
            instead of per-token classification). It is the first token of the sequence when built with special tokens.
        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        pad_token (`str`, *optional*, defaults to `"<pad>"`):
            The token used for padding, for example when batching sequences of different lengths.
        mask_token (`str`, *optional*, defaults to `"<mask>"`):
            The token used for masking values. This is the token used when training this model with masked language
            modeling. This is the token which the model will try to predict.
        add_prefix_space (`bool`, *optional*, defaults to `False`):
            Whether or not to add an initial space to the input. This allows to treat the leading word just as any
            other word. (RoBERTa tokenizer detect beginning of words by the preceding space).
    )Ú	input_idsÚtoken_type_idsÚattention_maskNÚreplaceú<s>ú</s>ú<unk>ú<pad>ú<mask>Fé2   éè  éé  éœÿÿÿTÚvocabÚmergesc                 ó  •— t          |t          ¦  «        rt          |dd¬¦  «        n|}t          |t          ¦  «        rt          |dd¬¦  «        n|}t          |t          ¦  «        rt          |dd¬¦  «        n|}t          |t          ¦  «        rt          |dd¬¦  «        n|}t          |	t          ¦  «        rt          |	dd¬¦  «        n|	}	t          |
t          ¦  «        rt          |
dd¬¦  «        n|
}
t          |t          ¦  «        rt          |dd¬¦  «        n|}|€Mt          |
¦  «        dt          |	¦  «        dt          |¦  «        dt          |¦  «        dt          |¦  «        di}|pg }t          t	          ||d d	d	d¬
¦  «        ¦  «        }t          j        |¬¦  «        |_        t          j        ¦   «         |_	        || _
        || _        || _         t          ¦   «         j        di d|“d|“d|“d|“d|	“d|“d|“d|
“d|“d|“d|“d|“d|“d|“d|“d|“|¤Ž t          |¦  «        }t          |¦  «        }| j        }| j        }t#          j        |› d|› �|› d|› d|› �||f||fg¬¦  «        | j        _        || _        || _        || _        || _        t1          | j        ¦  «        | _        | j        dz   | _        | j        g| j        z  | _        | j        g| j        z  | _        || _        || _        d S ) NF)ÚlstripÚrstripTr   é   é   r   é   Ú )r(   r)   ÚdropoutÚcontinuing_subword_prefixÚend_of_word_suffixÚfuse_unk)Úadd_prefix_spaceÚ	tags_dictÚerrorsÚ	bos_tokenÚ	eos_tokenÚ	unk_tokenÚ	sep_tokenÚ	cls_tokenÚ	pad_tokenÚ
mask_tokenr5   Útrim_offsetsÚ	max_depthÚ	max_widthÚ	pad_widthÚpad_token_labelÚonly_label_first_subwordz $A z $B )ÚsingleÚpairÚspecial_tokens© )Ú
isinstanceÚstrr
   r   r   r   Ú	ByteLevelÚpre_tokenizerr   ÚdecoderÚ_vocabÚ_mergesÚ
_tokenizerÚsuperÚ__init__Úcls_token_idÚsep_token_idr   ÚTemplateProcessingÚpost_processorr6   r@   rA   rB   ÚlenÚ
unk_tag_idÚ
pad_tag_idÚpad_xpath_tags_seqÚpad_xpath_subs_seqrC   rD   )Úselfr6   r(   r)   r7   r8   r9   r;   r<   r:   r=   r>   r5   r@   rA   rB   rC   rD   r?   ÚkwargsÚ	tokenizerÚsep_token_strÚcls_token_strrS   rT   Ú	__class__s                            €úp/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/markuplm/tokenization_markuplm.pyrR   zMarkupLMTokenizer.__init__™   sÚ  ø€ õ, JTÐT]Õ_bÑIcÔIcÐr•J˜y°¸uÐEÑEÔEÐEÐirˆ	ÝISÐT]Õ_bÑIcÔIcÐr•J˜y°¸uÐEÑEÔEÐEÐirˆ	ÝISÐT]Õ_bÑIcÔIcÐr•J˜y°¸uÐEÑEÔEÐEÐirˆ	ÝISÐT]Õ_bÑIcÔIcÐr•J˜y°¸uÐEÑEÔEÐEÐirˆ	ÝISÐT]Õ_bÑIcÔIcÐr•J˜y°¸uÐEÑEÔEÐEÐirˆ	ÝISÐT]Õ_bÑIcÔIcÐr•J˜y°¸uÐEÑEÔEÐEÐirˆ	åJTÐU_ÕadÑJeÔJeÐu•Z 
°4ÀÐFÑFÔFÐFÐkuˆ
àˆ=å�I‘” Ý�I‘” Ý�I‘” Ý�I‘” Ý�J‘” ðˆEð �˜2ˆÝÝØØØØ*,Ø#%Øðñ ô ñ	
ô 	
ˆ	õ #1Ô":ÐL\Ð"]Ñ"]Ô"]ˆ	ÔÝ$Ô.Ñ0Ô0ˆ	ÔØˆŒØˆŒØ#ˆŒØ�‰ŒÔð 	
ð 	
ð 	
Ø�ið	
à�6ð	
ð  �ið	
ð  �ið		
ð
  �ið	
ð  �ið	
ð  �ið	
ð  �ið	
ð "�zð	
ð .Ð-ð	
ð &˜ð	
ð  �ið	
ð  �ið	
ð  �ið	
ð ,˜Oð	
ð  &>Ð%=Øð#	
ð 	
ð 	
õ& ˜I™œˆÝ˜I™œˆØÔ(ˆØÔ(ˆÝ)3Ô)FØ#Ð8Ð8¨Ð8Ð8Ø!ÐIÐI }ÐIÐI¸-ÐIÐIà Ð-Ø Ð-ðð*
ñ *
ô *
ˆŒÔ&ð #ˆŒð #ˆŒØ"ˆŒØ"ˆŒÝ˜dœnÑ-Ô-ˆŒØœ/¨AÑ-ˆŒØ#'¤?Ð"3°d´nÑ"DˆÔØ#'¤>Ð"2°T´^Ñ"CˆÔØ.ˆÔØ(@ˆÔ%Ð%Ð%ó    c                 ó°  — g }g }|                      d¦  «        }|D ]Ó}|                     ¦   «         sŒ|                     ¦   «                               d¦  «        }|d         }t          |¦  «        dk    rdnt          |d         dd…         ¦  «        }|                     | j                             || j        ¦  «        ¦  «         |                     t          | j	        |¦  «        ¦  «         ŒÔ|d| j
        …         }|d| j
        …         }|| j        g| j
        t          |¦  «        z
  z  z  }|| j        g| j
        t          |¦  «        z
  z  z  }||fS )zÌ
        Given the xpath expression of one particular node (like "/html/body/div/li[1]/div/span[2]"), return a list of
        tag IDs and corresponding subscripts, taking into account max depth.
        ú/ú[r   r-   Néÿÿÿÿ)ÚsplitÚstriprW   ÚintÚappendr6   ÚgetrX   ÚminrA   r@   rY   rB   )	r\   ÚxpathÚxpath_tags_listÚxpath_subs_listÚxpath_unitsÚunitÚ	name_subsÚtag_nameÚsubs	            rb   Úget_xpath_seqzMarkupLMTokenizer.get_xpath_seqý   sS  € ð
 ˆØˆà—k’k #Ñ&Ô&ˆØð 	=ð 	=ˆDØ—:’:‘<”<ð ØØŸ
š
™œ×*Ò*¨3Ñ/Ô/ˆIØ  ”|ˆHÝ˜9‘~”~¨Ò*Ð*�!�!µ°I¸a´LÀÀ"ÀÔ4EÑ0FÔ0FˆCØ×"Ò" 4¤>×#5Ò#5°hÀÄÑ#PÔ#PÑQÔQÐQØ×"Ò"¥3 t¤~°sÑ#;Ô#;Ñ<Ô<Ð<Ð<à)Ð*:¨D¬NÐ*:Ô;ˆØ)Ð*:¨D¬NÐ*:Ô;ˆØ˜DœOÐ,°´ÅÀ_ÑAUÔAUÑ0UÑVÑVˆØ˜DœNÐ+¨t¬~ÅÀOÑ@TÔ@TÑ/TÑUÑUˆà Ð/Ð/rc   r   ÚtextÚ	text_pairÚxpathsÚnode_labelsÚadd_special_tokensÚpaddingÚ
truncationÚ
max_lengthÚstrideÚis_split_into_wordsÚpad_to_multiple_ofÚpadding_sideÚreturn_tensorsÚreturn_token_type_idsÚreturn_attention_maskÚreturn_overflowing_tokensÚreturn_special_tokens_maskÚreturn_offsets_mappingÚreturn_lengthÚverboseÚreturnc                 óf  ‡!— dŠ!t          |t          ¦  «        rt          |¦  «        }|�$t          |t          ¦  «        rt          |¦  «        }|�€U|
�sR|€|n|}t          |t          ¦  «        rt          |¦  «        }|}t          |t          ¦  «        r|                     ¦   «         }n†t          |t          ¦  «        rq|rFt          |d         t          ¦  «        r+t          d„ |D ¦   «         ¦  «        }|rd„ |D ¦   «         }n,|}n)|r't          |d         t          ¦  «        rd„ |D ¦   «         }|€|}n|}t          |t          ¦  «        r3|r1t          |d         t          t          f¦  «        rˆ!fd„|D ¦   «         }n't          |d¦  «        rt          |¦  «        nd}‰!g|z  }d	„ }|�b ||¦  «        st          d
¦  «        ‚t          |t          t          f¦  «        st          d¦  «        ‚t          |t          t          f¦  «        }nkt          |t          t          f¦  «        st          d¦  «        ‚t          |t          t          f¦  «        o#|o!t          |d         t          t          f¦  «        }|€|n|}|€
J d¦   «         ‚|rkt          |¦  «        t          |¦  «        k    s
J d¦   «         ‚t          ||¦  «        D ]/\  }}t          |¦  «        t          |¦  «        k    s
J d¦   «         ‚Œ0n*t          |¦  «        t          |¦  «        k    s
J d¦   «         ‚|rÊ|�Pt          |¦  «        t          |¦  «        k    r0t          dt          |¦  «        › dt          |¦  «        › d�¦  «        ‚|�t          t          ||¦  «        ¦  «        n|}t          |du¦  «        }  | j        d'i d|“d| “d|“d|“d|“d|“d|“d|“d|	“d|“d|“d|“d|“d|“d |“d!|“d"|“d#|“d$|“|¤ŽS  | j        d'i d%|“d&|“d|“d|“d|“d|“d|“d|“d|	“d|“d|“d|“d|“d|“d |“d!|“d"|“d#|“d$|“|¤ŽS )(a”  
        Main method to tokenize and prepare for the model one or several sequence(s) or one or several pair(s) of
        sequences with nodes, xpaths and optional labels.

        Args:
            text (`str`, `list[str]`, `list[list[str]]`):
                The sequence or batch of sequences to be encoded. Each sequence can be a string, a list of strings
                (words of a single example or questions of a batch of examples) or a list of list of strings (batch of
                words).
            text_pair (`list[str]`, `list[list[str]]`):
                The sequence or batch of sequences to be encoded. Each sequence should be a list of strings
                (pretokenized string).
            xpaths (`list[list[int]]`, `list[list[list[int]]]`):
                Node-level xpaths. Each bounding box should be normalized to be on a 0-1000 scale.
            node_labels (`list[int]`, `list[list[int]]`, *optional*):
                Node-level integer labels (for token classification tasks).
            is_split_into_words (`bool`, *optional*):
                Set to `True` if the inputs are already provided as pretokenized word lists.
        ú/document/nodeNr   c              3   ó   K  — | ]}d |v V — Œ	dS )ú NrH   ©Ú.0Úentrys     rb   ú	<genexpr>z-MarkupLMTokenizer.__call__.<locals>.<genexpr>T  s&   è è € Ð(PÐ(P¸%¨°¨Ð(PÐ(PÐ(PÐ(PÐ(PÐ(Prc   c                 ó6   — g | ]}|                      ¦   «         ‘ŒS rH   )rh   r�   s     rb   ú
<listcomp>z.MarkupLMTokenizer.__call__.<locals>.<listcomp>V  s    € Ð*SÐ*SÐ*S¸U¨5¯;ª;©=¬=Ð*SÐ*SÐ*Src   c                 ó,   — g | ]}t          |¦  «        ‘ŒS rH   )Úlist)r‘   Úsamples     rb   r•   z.MarkupLMTokenizer.__call__.<locals>.<listcomp>Z  s   € Ð&OÐ&OÐ&O¸¥t¨F¡|¤|Ð&OÐ&OÐ&Orc   c                 ó6   •— g | ]}‰gt          |¦  «        z  ‘ŒS rH   )rW   )r‘   r˜   Úplaceholder_xpaths     €rb   r•   z.MarkupLMTokenizer.__call__.<locals>.<listcomp>b  s(   ø€ ÐZÐZÐZÀÐ,Ð-µ°F±´Ñ;ÐZÐZÐZrc   Ú__len__c                 óˆ  — t          | t          ¦  «        rdS t          | t          t          f¦  «        rŽt	          | ¦  «        dk    rdS t          | d         t          ¦  «        rdS t          | d         t          t          f¦  «        r:t	          | d         ¦  «        dk    p t          | d         d         t          ¦  «        S dS )NTr   F)rI   rJ   r—   ÚtuplerW   )Úts    rb   Ú_is_valid_text_inputz8MarkupLMTokenizer.__call__.<locals>._is_valid_text_inputg  sª   € Ý˜!�SÑ!Ô!ð Ø�tÝ˜!�d¥E˜]Ñ+Ô+ð FÝ�q‘6”6˜Q’;�;Ø˜4Ý˜a œd¥CÑ(Ô(ð  Ø˜4Ý˜a œd¥T­5 MÑ2Ô2ð FÝ˜q œt™9œ9¨š>ÐE­Z¸¸!¼¸Q¼ÅÑ-EÔ-EÐEØ�5rc   zStext input must of type `str` (single example) or `list[str]` (batch of examples). zwNodes must be of type `list[str]` (single pretokenized example), or `list[list[str]]` (batch of pretokenized examples).z%You must provide corresponding xpathszAYou must provide nodes and xpaths for an equal amount of examplesz2You must provide as many nodes as there are xpathszbatch length of `text`: z- does not match batch length of `text_pair`: ú.Úbatch_text_or_text_pairsÚis_pairry   rz   r{   r|   r}   r~   r   r�   r‚   rƒ   r„   r…   r†   r‡   rˆ   r‰   rŠ   rw   rx   rH   )rI   r�   r—   rJ   rh   ÚanyÚhasattrrW   Ú
ValueErrorÚzipÚboolÚbatch_encode_plusÚencode_plus)"r\   rw   rx   ry   rz   r{   r|   r}   r~   r   r€   r�   r‚   rƒ   r„   r…   r†   r‡   rˆ   r‰   rŠ   r]   Únodes_sourceÚprocessed_nodesÚrequires_splitÚlengthrŸ   Ú
is_batchedÚnodesÚnodes_exampleÚxpaths_exampler¡   r¢   rš   s"                                    @rb   Ú__call__zMarkupLMTokenizer.__call__  sQ  ø€ ðZ -Ðå�d�EÑ"Ô"ð 	Ý˜‘:”:ˆDØÐ ¥Z°	½5Ñ%AÔ%AÐ Ý˜Y™œˆIà‰>Ð"5‰>Ø#,Ð#4˜4˜4¸)ˆLÝ˜,­Ñ.Ô.ð 2Ý# LÑ1Ô1�Ø*ˆOå˜,­Ñ,Ô,ð 
PØ".×"4Ò"4Ñ"6Ô"6��Ý˜L­$Ñ/Ô/ð PØð P¥J¨|¸A¬ÅÑ$DÔ$Dð PÝ%(Ð(PÐ(PÀ<Ð(PÑ(PÔ(PÑ%PÔ%P�NØ%ð 7Ø*SÐ*SÀlÐ*SÑ*SÔ*S˜˜à*6˜˜Ø!ð P¥j°¸a´Å%Ñ&HÔ&Hð PØ&OÐ&OÀ,Ð&OÑ&OÔ&O�OàÐ Ø&��à+�	å˜/­4Ñ0Ô0ð 6°_ð 6ÍÐTcÐdeÔTfÕimÕotÐhuÑIvÔIvð 6ØZÐZÐZÐZÈ/ÐZÑZÔZ��å18¸È)Ñ1TÔ1TÐ[�˜_Ñ-Ô-Ð-ÐZ[�Ø+Ð,¨vÑ5�ð
	ð 
	ð 
	ð Ð à'Ð'¨Ñ-Ô-ð xÝ Ð!vÑwÔwÐwÝ˜i­$µ¨Ñ7Ô7ð Ý ðMñô ð õ $ D­4µ¨-Ñ8Ô8ˆJˆJõ ˜d¥T­5 MÑ2Ô2ð Ý ðMñô ð õ $ D­4µ¨-Ñ8Ô8Ðh¸TÐhÅjÐQUÐVWÔQXÕ[_ÕafÐZgÑFhÔFhˆJà!Ð)��¨yˆØÐ!Ð!Ð#JÑ!Ô!Ð!Øð 	cÝ�u‘:”:¥ V¡¤Ò,Ð,Ð,Ð.qÑ,Ô,Ð,Ý14°U¸FÑ1CÔ1Cð wð wÑ-�˜~Ý˜=Ñ)Ô)­S°Ñ-@Ô-@Ò@Ð@Ð@ÐBvÑ@Ô@Ð@Ð@ðwõ �u‘:”:¥ V¡¤Ò,Ð,Ð,Ð.bÑ,Ô,Ð,àð 4	ØÐ$­¨T©¬µc¸)±n´nÒ)DÐ)DÝ ð*­s°4©y¬yð *ð *Ý˜I™œð*ð *ð *ñô ð ð FOÐEZ¥t­C°°iÑ,@Ô,@Ñ'AÔ'AÐ'AÐ`dÐ$Ý˜9¨DÐ0Ñ1Ô1ˆGØ)�4Ô)ð ð ð Ø)AÐ)Aðà˜ðð �vðð (˜Kð	ð
 $6Ð#5ðð  ˜ðð &˜:ðð &˜:ðð �vðð $6Ð#5ðð *˜\ðð  .˜~ðð '<Ð&;ðð '<Ð&;ðð +DÐ*Cðð  ,FÐ+Eð!ð" (>Ð'=ð#ð$ ,˜mð%ð&  ˜Øð)ð ð ð. $�4Ô#ð ð ð Ø�Tðà#˜)ðð �vðð (˜Kð	ð
 $6Ð#5ðð  ˜ðð &˜:ðð &˜:ðð �vðð $6Ð#5ðð *˜\ðð  .˜~ðð '<Ð&;ðð '<Ð&;ðð +DÐ*Cðð  ,FÐ+Eð!ð" (>Ð'=ð#ð$ ,˜mð%ð&  ˜Øð)ð ð rc   r¡   r¢   c           	      ó¾   —  | j         d||||
|dœ|¤Ž\  }}}} | j        di d|“d|“d|“d|“d|“d|“d|“d	|“d
|	“d|
“d|“d|“d|“d|“d|“d|“d|“d|“d|“|¤ŽS )N©r|   r}   r~   r�   rŠ   r¡   r¢   ry   rz   r{   Úpadding_strategyÚtruncation_strategyr~   r   r�   r‚   rƒ   r„   r…   r†   r‡   rˆ   r‰   rŠ   rH   )Ú"_get_padding_truncation_strategiesÚ_batch_encode_plus)r\   r¡   r¢   ry   rz   r{   r|   r}   r~   r   r�   r‚   rƒ   r„   r…   r†   r‡   rˆ   r‰   rŠ   r]   rµ   r¶   s                          rb   r¨   z#MarkupLMTokenizer.batch_encode_plusÅ  s_  € ð2 ElÀDÔDkð E
ØØ!Ø!Ø1ØðE
ð E
ð ðE
ð E
ÑAÐÐ-¨z¸6ð 'ˆtÔ&ð 
ð 
ð 
Ø%=Ð%=ð
à�Gð
ð �6ð
ð $˜ð	
ð
  2Ð1ð
ð .Ð-ð
ð !4Ð 3ð
ð "�zð
ð �6ð
ð  2Ð1ð
ð &˜ð
ð *˜>ð
ð #8Ð"7ð
ð #8Ð"7ð
ð '@Ð&?ð
ð  (BÐ'Að!
ð" $:Ð#9ð#
ð$ (˜-ð%
ð& �GØð)
ð 
ð 	
rc   rF   c                 ó\   — |r||fgn|g} | j         j        |f|ddœ|¤Ž}|d         j        S )NF©r{   Úis_pretokenizedr   )rP   Úencode_batchÚtokens)r\   rw   rF   r{   r]   Úbatched_inputÚ	encodingss          rb   ÚtokenizezMarkupLMTokenizer.tokenizeþ  sX   € Ø*.Ð:˜$ ˜˜˜°T°FˆØ0�D”OÔ0Øð
Ø.@ÐRWð
ð 
Ø[að
ð 
ˆ	ð ˜Œ|Ô"Ð"rc   c           	      ó¾   —  | j         d||||
|dœ|¤Ž\  }}}} | j        di d|“d|“d|“d|“d|“d|“d|“d	|“d
|	“d|
“d|“d|“d|“d|“d|“d|“d|“d|“d|“|¤ŽS )aj  
        Tokenize and prepare for the model a sequence or a pair of sequences. .. warning:: This method is deprecated,
        `__call__` should be used instead.

        Args:
            text (`str`, `list[str]`, `list[list[str]]`):
                The first sequence to be encoded. This can be a string, a list of strings or a list of list of strings.
            text_pair (`list[str]` or `list[int]`, *optional*):
                Optional second sequence to be encoded. This can be a list of strings (words of a single example) or a
                list of list of strings (words of a batch of examples).
        r´   rw   ry   rx   rz   r{   rµ   r¶   r~   r   r�   r‚   rƒ   r„   r…   r†   r‡   rˆ   r‰   rŠ   rH   )r·   Ú_encode_plus)r\   rw   rx   ry   rz   r{   r|   r}   r~   r   r�   r‚   rƒ   r„   r…   r†   r‡   rˆ   r‰   rŠ   r]   rµ   r¶   s                          rb   r©   zMarkupLMTokenizer.encode_plus  s_  € ðJ ElÀDÔDkð E
ØØ!Ø!Ø1ØðE
ð E
ð ðE
ð E
ÑAÐÐ-¨z¸6ð !ˆtÔ ð 
ð 
ð 
Ø�ð
à�6ð
ð  �ið
ð $˜ð	
ð
  2Ð1ð
ð .Ð-ð
ð !4Ð 3ð
ð "�zð
ð �6ð
ð  2Ð1ð
ð &˜ð
ð *˜>ð
ð #8Ð"7ð
ð #8Ð"7ð
ð '@Ð&?ð
ð  (BÐ'Að!
ð" $:Ð#9ð#
ð$ (˜-ð%
ð& �GØð)
ð 
ð 	
rc   rµ   r¶   c           	      óÒ	  ‡ ‡‡‡‡‡‡‡‡‡/— t          |t          ¦  «        s t          dt          |¦  «        › d�¦  «        ‚‰                      ||||	|
|¬¦  «         |rœg }|D ]”\  }}t          |t
          ¦  «        rt          |¦  «        }t          |t          ¦  «        r|g}t          |t
          ¦  «        rt          |¦  «        }t          |t          ¦  «        r|g}|                     ||f¦  «         Œ•|}nZg }|D ]S}t          |t
          ¦  «        rt          |¦  «        }t          |t          ¦  «        r|g}|                     |¦  «         ŒT|}‰ j         	                    ||d¬¦  «        }ˆˆˆˆˆˆˆˆ ˆf	d„|D ¦   «         }i }|d         d         D ]Š/ˆ/fd„|D ¦   «         }||‰/<   Œd	„ |D ¦   «         }‰r;g }t          |¦  «        D ]$\  }\  }}||gt          |d
         ¦  «        z  z  }Œ%||d<   |d
         D ]} ‰                      | |‰¦  «         Œg }!g }"t          t          |d
         ¦  «        ¦  «        D �]g}#‰r|d         |#         }$n|#}$g }%g }&t          |d
         |#         ||#         j        ||#         j        ¦  «        D ]ï\  }'}(})|)�Ž|r;|(dk    r5|%                     ‰ j        ¦  «         |&                     ‰ j        ¦  «         ŒE‰                      ||$         |)         ¦  «        \  }*}+|%                     |*g¦  «         |&                     |+g¦  «         Œ–|'‰ j        ‰ j        ‰ j        fv r5|%                     ‰ j        ¦  «         |&                     ‰ j        ¦  «         Œát1          d¦  «        ‚|!                     |%¦  «         |"                     |&¦  «         �Œi|!|d<   |"|d<   ‰��$g },t          t          |d
         ¦  «        ¦  «        D ]õ}#‰r|d         |#         }$n|#}$g }-t          |d
         |#         |d         |#         ||#         j        ¦  «        D ]”\  }'}.})|)�r‰ j        rI|.d         dk    r"|-                     ‰|$         |)         ¦  «         Œ=|-                     ‰ j        ¦  «         ŒX|-                     ‰|$         |)         ¦  «         Œz|-                     ‰ j        ¦  «         Œ•|,                     |-¦  «         Œö|,|d<   ‰s|d= t7          |||¬¦  «        S )Nz/batch_text_or_text_pairs has to be a list (got ú))rµ   r¶   r~   r   r�   r‚   Trº   c                 óR   •	— g | ]#}‰	                      |‰‰‰‰‰�dn‰‰‰
¬¦  «        ‘Œ$S )NT)Úencodingr„   r…   r†   r‡   rˆ   r‰   rŠ   )Ú_convert_encoding)r‘   rÆ   rz   r…   r‰   rˆ   r†   r‡   r„   r\   rŠ   s     €€€€€€€€€rb   r•   z8MarkupLMTokenizer._batch_encode_plus.<locals>.<listcomp>Ž  se   ø€ ð  
ð  
ð  
ð ð ×"Ò"Ø!Ø&;Ø&;Ø*CØ+EàÐ*ð (, tà+Ø+Øð #ñ ô ð 
ð  
ð  
rc   r   c                 ó0   •— g | ]\  }}|‰         D ]}|‘ŒŒS rH   rH   )r‘   ÚitemÚ_ÚeÚkeys       €rb   r•   z8MarkupLMTokenizer._batch_encode_plus.<locals>.<listcomp>¦  s.   ø€ ÐNÐNÐN™7˜4 ÀDÈÄIÐNÐN¸q�QÐNÐNÐNÐNrc   c                 ó"   — g | ]\  }}|D ]}|‘ŒŒS rH   rH   )r‘   rÊ   rÉ   rË   s       rb   r•   z8MarkupLMTokenizer._batch_encode_plus.<locals>.<listcomp>¨  s)   € ÐSÐSÐS¡W Q¨ÈdÐSÐSÈ˜qÐSÐSÐSÐSrc   r   Úoverflow_to_sample_mappingzId not recognizedÚxpath_tags_seqÚxpath_subs_seqÚoffset_mappingÚlabels)Útensor_type)rI   r—   Ú	TypeErrorÚtypeÚset_truncation_and_paddingr�   rJ   rk   rP   r¼   Ú	enumeraterW   Ú&_eventual_warn_about_too_long_sequenceÚranger¦   Úsequence_idsÚword_idsrZ   r[   rv   ÚextendrS   rT   Úpad_token_idr¥   rD   rC   r   )0r\   r¡   r¢   ry   rz   r{   rµ   r¶   r~   r   r�   r‚   rƒ   r„   r…   r†   r‡   rˆ   r‰   rŠ   Úprocessed_inputsrw   rx   r¿   Útokens_and_encodingsÚsanitized_tokensÚstackÚsanitized_encodingsrÎ   ÚiÚtoksrÊ   r   rÏ   rÐ   Úbatch_indexÚoriginal_indexÚxpath_tags_seq_exampleÚxpath_subs_seq_exampleÚidÚsequence_idÚword_idro   rp   rÒ   Úlabels_exampleÚoffsetrÌ   s0   `   `        ```````                           @rb   r¸   z$MarkupLMTokenizer._batch_encode_plusK  s[  øøøøøøøøøø€ õ, Ð2µDÑ9Ô9ð 	qÝÐoÍdÐSkÑNlÔNlÐoÐoÐoÑpÔpÐpð 	×'Ò'Ø-Ø 3Ø!ØØ1Ø%ð 	(ñ 	
ô 	
ð 	
ð ð 	8Ø!ÐØ#;ð 	;ð 	;‘��iÝ˜d¥EÑ*Ô*ð &Ý ™:œ:�DÝ˜d¥CÑ(Ô(ð "Ø ˜6�DÝ˜i­Ñ/Ô/ð 0Ý $ Y¡¤�IÝ˜i­Ñ-Ô-ð ,Ø!* �IØ ×'Ò'¨¨yÐ(9Ñ:Ô:Ð:Ð:Ø'7Ð$Ð$à!ÐØ0ð .ð .�Ý˜d¥EÑ*Ô*ð &Ý ™:œ:�DÝ˜d¥CÑ(Ô(ð "Ø ˜6�DØ ×'Ò'¨Ñ-Ô-Ð-Ð-Ø'7Ð$à”O×0Ò0Ø$Ø1Ø ð 1ñ 
ô 
ˆ	ð 
ð  
ð  
ð  
ð  
ð  
ð  
ð  
ð  
ð  
ð  
ð  
ð &ð 
ñ  
ô  
Ðð, ÐØ'¨Ô*¨1Ô-ð 	*ð 	*ˆCØNÐNÐNÐNÐ&:ÐNÑNÔNˆEØ$)Ð˜SÑ!Ð!ØSÐSÐ0DÐSÑSÔSÐð %ð 	XØ)+Ð&Ý )Ð*>Ñ ?Ô ?ð Kð K‘�‘9�D˜!Ø*¨q¨cµC¸¸[Ô8IÑ4JÔ4JÑ.JÑJÐ*Ð*Ø=WÐÐ9Ñ:à)¨+Ô6ð 	Xð 	XˆIØ×7Ò7¸	À:ÈwÑWÔWÐWÐWð ˆØˆÝ ¥Ð%5°kÔ%BÑ!CÔ!CÑDÔDð 	:ñ 	:ˆKØ(ð -Ø!1Ð2NÔ!OÐP[Ô!\��à!,�Ø%'Ð"Ø%'Ð"Ý,/Ø  Ô-¨kÔ:Ø# KÔ0Ô=Ø# KÔ0Ô9ñ-ô -ð >ð >Ñ(��K ð
 Ð&Øð I ;°!Ò#3Ð#3Ø.×5Ò5°dÔ6MÑNÔNÐNØ.×5Ò5°dÔ6MÑNÔNÐNÐNà;?×;MÒ;MÈfÐUcÔNdÐelÔNmÑ;nÔ;nÑ8˜¨Ø.×5Ò5°Ð6GÑHÔHÐHØ.×5Ò5°Ð6GÑHÔHÐHÐHà˜dÔ/°Ô1BÀDÔDUÐVÐVÐVØ.×5Ò5°dÔ6MÑNÔNÐNØ.×5Ò5°dÔ6MÑNÔNÐNÐNå(Ð)<Ñ=Ô=Ð=Ø×!Ò!Ð"8Ñ9Ô9Ð9Ø×!Ò!Ð"8Ñ9Ô9Ð9Ñ9à-;ÐÐ)Ñ*Ø-;ÐÐ)Ñ*ð Ñ"ØˆFÝ$¥SÐ)9¸+Ô)FÑ%GÔ%GÑHÔHð .ð .�Ø,ð 1Ø%5Ð6RÔ%SÐT_Ô%`�N�Nà%0�NØ!#�Ý+.Ø$ [Ô1°+Ô>Ø$Ð%5Ô6°{ÔCØ'¨Ô4Ô=ñ,ô ,ð Dð DÑ'�B˜ ð
 Ð*ØÔ8ð XØ% aœy¨Aš~˜~à .× 5Ò 5°kÀ.Ô6QÐRYÔ6ZÑ [Ô [Ð [Ð [à .× 5Ò 5°dÔ6JÑ KÔ KÐ KÐ Kà*×1Ò1°+¸nÔ2MÈgÔ2VÑWÔWÐWÐWà&×-Ò-¨dÔ.BÑCÔCÐCÐCØ—’˜nÑ-Ô-Ð-Ð-à)/Ð˜XÑ&à)ð 7Ø$Ð%5Ð6åÐ-Ð/BÐP^Ð_Ñ_Ô_Ð_rc   c           	      ó<  — d}t          |t          ¦  «        rt          |¦  «        }|�$t          |t          ¦  «        rt          |¦  «        }|€|n|}|}t          |t          ¦  «        r|                     ¦   «         }n4t          |t          ¦  «        r|rt          |d         t          ¦  «        r|}|€|}n|}|€'t          |d¦  «        rt          |¦  «        nd}|g|z  }|r||fgn|g}|g}|�|gnd } | j        |fi dt          |d u¦  «        “d|“d|“d|“d|“d	|“d
|“d|	“d|
“d|“d|“d|“d|“d|“d|“d|“d|“d|“|¤Ž}|€3|s1t          d„ | 
                    ¦   «         D ¦   «         |j        ¦  «        }|                      |d         ||¦  «         |S )Nr�   r   r›   r¢   ry   rz   r{   rµ   r¶   r~   r   r�   r‚   rƒ   r„   r…   r†   r‡   rˆ   r‰   rŠ   c                 ó†   — i | ]>\  }}|t          |¦  «        d k    r#t          |d          t          ¦  «        r|d          n|“Œ?S )r   )rW   rI   r—   )r‘   rÌ   Úvalues      rb   ú
<dictcomp>z2MarkupLMTokenizer._encode_plus.<locals>.<dictcomp>K  sW   € ð ð ð á"˜˜Uð ¥S¨¡Z¤Z°!¢^ ^½
À5ÈÄ8ÍTÑ8RÔ8R ^˜˜qœ˜ÐX]ðð ð rc   r   )rI   r�   r—   rJ   rh   r¤   rW   r¸   r§   r   Úitemsr¿   rØ   )r\   rw   rx   ry   rz   r{   rµ   r¶   r~   r   r�   r‚   rƒ   r„   r…   r†   r‡   rˆ   r‰   rŠ   r]   rš   Únodes_singler«   r­   r¾   Úbatched_xpathsÚbatched_node_labelsÚbatched_outputs                                rb   rÂ   zMarkupLMTokenizer._encode_plusú  sµ  € ð. -Ðå�d�EÑ"Ô"ð 	Ý˜‘:”:ˆDØÐ ¥Z°	½5Ñ%AÔ%AÐ Ý˜Y™œˆIà(Ð0�t�t°iˆØ&ˆå�l¥CÑ(Ô(ð 	+Ø*×0Ò0Ñ2Ô2ˆOˆOÝ˜¥dÑ+Ô+ð 	+°ð 	+ÅÈLÐYZÌOÕ]`ÑAaÔAað 	+Ø*ˆOàÐØ"ˆDˆDà'ˆIàˆ>Ý-4°_ÀiÑ-PÔ-PÐW•S˜Ñ)Ô)Ð)ÐVWˆFØ'Ð(¨6Ñ1ˆFð 09ÐD˜$ 	Ð*Ð+Ð+¸t¸fˆØ ˜ˆØ/:Ð/F˜{˜m˜mÈDÐØ0˜Ô0Øð
ð 
ð 
å˜¨$Ð.Ñ/Ô/Ð/ð
ð "�>ð
ð ,Ð+ð	
ð
  2Ð1ð
ð .Ð-ð
ð !4Ð 3ð
ð "�zð
ð �6ð
ð  2Ð1ð
ð &˜ð
ð *˜>ð
ð #8Ð"7ð
ð #8Ð"7ð
ð '@Ð&?ð
ð  (BÐ'Að!
ð" $:Ð#9ð#
ð$ (˜-ð%
ð& �GØð)
ð 
ˆð2 Ð!Ð*CÐ!Ý*ðð à&4×&:Ò&:Ñ&<Ô&<ðñ ô ð Ô(ñô ˆNð 	×3Ò3°NÀ;Ô4OÐQ[Ð]dÑeÔeÐeàÐrc   Úencoded_inputsc                 ó�  — |€	d| j         v }|| j         d                  }|t          j        k    rt          |¦  «        }|�|�||z  dk    r||z  dz   |z  }|t          j        k    ot          |¦  «        |k    }|rd|vrdgt          |¦  «        z  |d<   |�r®|t          |¦  «        z
  }	|�|n| j        }|dk    r³|r|d         dg|	z  z   |d<   d|v r|d         | j        g|	z  z   |d<   d|v r|d         | j        g|	z  z   |d<   d|v r|d         | j        g|	z  z   |d<   d	|v r|d	         | j	        g|	z  z   |d	<   d
|v r|d
         dg|	z  z   |d
<   || j
        g|	z  z   || j         d         <   nØ|dk    r³|rdg|	z  |d         z   |d<   d|v r| j        g|	z  |d         z   |d<   d|v r| j        g|	z  |d         z   |d<   d|v r| j        g|	z  |d         z   |d<   d	|v r| j	        g|	z  |d	         z   |d	<   d
|v rdg|	z  |d
         z   |d
<   | j
        g|	z  |z   || j         d         <   nt          dt          |¦  «        z   ¦  «        ‚|S )a   
        Args:
        Pad encoded inputs (on left/right and up to predefined length or max length in the batch)
            encoded_inputs:
                Dictionary of tokenized inputs (`list[int]`) or batch of tokenized inputs (`list[list[int]]`).
            max_length: maximum length of the returned list and optionally padding length (see below).
                Will truncate by taking into account the special tokens.
            padding_strategy: PaddingStrategy to use for padding.
                - PaddingStrategy.LONGEST Pad to the longest sequence in the batch
                - PaddingStrategy.MAX_LENGTH: Pad to the max length (default)
                - PaddingStrategy.DO_NOT_PAD: Do not pad
                The tokenizer padding sides are defined in self.padding_side:
                    - 'left': pads on the left of the sequences
                    - 'right': pads on the right of the sequences
            pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value.
                This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability
                `>= 7.5` (Volta).
            padding_side:
                The side on which the model should have padding applied. Should be selected between ['right', 'left'].
                Default value is picked from the class attribute of the same name.
            return_attention_mask:
                (optional) Set to False to avoid returning attention mask (default: set to model specifics)
        Nr   r   r-   Úrightr   rÏ   rÐ   rÒ   Úspecial_tokens_maskÚleftzInvalid padding strategy:)Úmodel_input_namesr   ÚLONGESTrW   Ú
DO_NOT_PADr‚   Úpad_token_type_idrZ   r[   rC   rÝ   r¥   rJ   )
r\   r÷   r~   rµ   r�   r‚   r…   Úrequired_inputÚneeds_to_be_paddedÚ
differences
             rb   Ú_padzMarkupLMTokenizer._padV  s«  € ðB !Ð(Ø$4¸Ô8NÐ$NÐ!à'¨Ô(>¸qÔ(AÔBˆà�Ô6Ò6Ð6Ý˜^Ñ,Ô,ˆJàÐ!Ð&8Ð&DÈ*ÐWiÑJiÐmnÒJnÐJnØ%Ð);Ñ;¸qÑ@ÐDVÑVˆJà-µÔ1KÒKÐqÕPSÐTbÑPcÔPcÐgqÒPqÐð !ð 	IÐ%5¸^Ð%KÐ%KØ01¨sµS¸Ñ5HÔ5HÑ/HˆNÐ+Ñ,àñ ,	RØ#¥c¨.Ñ&9Ô&9Ñ9ˆJØ+7Ð+C˜<˜<ÈÔIZˆLØ˜wÒ&Ð&Ø(ð kØ7EÐFVÔ7WÐ[\ÐZ]Ð`jÑZjÑ7j�NÐ#3Ñ4Ø# ~Ð5Ð5à&Ð'7Ô8¸DÔ<RÐ;SÐV`Ñ;`Ñ`ð #Ð#3Ñ4ð $ ~Ð5Ð5à&Ð'7Ô8¸DÔ<SÐ;TÐWaÑ;aÑað #Ð#3Ñ4ð $ ~Ð5Ð5à&Ð'7Ô8¸DÔ<SÐ;TÐWaÑ;aÑað #Ð#3Ñ4ð ˜~Ð-Ð-Ø/=¸hÔ/GÈ4ÔK_ÐJ`ÐcmÑJmÑ/m�N 8Ñ,Ø(¨NÐ:Ð:Ø<JÐK`Ô<aÐefÐdgÐjtÑdtÑ<t�NÐ#8Ñ9Ø<JÈdÔN_ÐM`ÐcmÑMmÑ<m�˜tÔ5°aÔ8Ñ9Ð9Ø Ò'Ð'Ø(ð kØ89°s¸ZÑ7GÈ.ÐYiÔJjÑ7j�NÐ#3Ñ4Ø# ~Ð5Ð5Ø8<Ô8NÐ7OÐR\Ñ7\Ð_mØ(ô`ñ 8�NÐ#3Ñ4ð $ ~Ð5Ð5Ø8<Ô8OÐ7PÐS]Ñ7]Ð`nØ(ôañ 8�NÐ#3Ñ4ð $ ~Ð5Ð5Ø8<Ô8OÐ7PÐS]Ñ7]Ð`nØ(ôañ 8�NÐ#3Ñ4ð ˜~Ð-Ð-Ø04Ô0DÐ/EÈ
Ñ/RÐUcÐdlÔUmÑ/m�N 8Ñ,Ø(¨NÐ:Ð:Ø=>¸CÀ*Ñ<LÈ~Ð^sÔOtÑ<t�NÐ#8Ñ9Ø=AÔ=NÐ<OÐR\Ñ<\Ð_mÑ<m�˜tÔ5°aÔ8Ñ9Ð9å Ð!<½sÀ<Ñ?PÔ?PÑ!PÑQÔQÐQàÐrc   Útoken_ids_0Útoken_ids_1c                 ój   — |€| j         g|z   | j        gz   S | j         g}| j        g}||z   |z   |z   |z   S )a”  
        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
        adding special tokens. A RoBERTa sequence has the following format:
        - single sequence: `<s> X </s>`
        - pair of sequences: `<s> A </s></s> B </s>`

        Args:
            token_ids_0 (`list[int]`):
                List of IDs to which the special tokens will be added.
            token_ids_1 (`list[int]`, *optional*):
                Optional second list of IDs for sequence pairs.
        Returns:
            `list[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
        )rS   rT   )r\   r  r  ÚclsÚseps        rb   Ú build_inputs_with_special_tokensz2MarkupLMTokenizer.build_inputs_with_special_tokens¸  sV   € ð" ÐØÔ%Ð&¨Ñ4¸Ô8IÐ7JÑJÐJØÔ Ð!ˆØÔ Ð!ˆØ�[Ñ  3Ñ&¨Ñ4°sÑ:Ð:rc   c                 ó–   — | j         g}| j        g}|€t          ||z   |z   ¦  «        dgz  S t          ||z   |z   |z   |z   ¦  «        dgz  S )aË  
        Create a mask from the two sequences passed to be used in a sequence-pair classification task. RoBERTa does not
        make use of token type ids, therefore a list of zeros is returned.

        Args:
            token_ids_0 (`list[int]`):
                List of IDs.
            token_ids_1 (`list[int]`, *optional*):
                Optional second list of IDs for sequence pairs.
        Returns:
            `list[int]`: List of zeros.
        Nr   )rT   rS   rW   )r\   r  r  r  r  s        rb   Ú$create_token_type_ids_from_sequencesz6MarkupLMTokenizer.create_token_type_ids_from_sequencesÏ  sh   € ð Ô Ð!ˆØÔ Ð!ˆàÐÝ�s˜[Ñ(¨3Ñ.Ñ/Ô/°1°#Ñ5Ð5Ý�3˜Ñ$ sÑ*¨[Ñ8¸3Ñ>Ñ?Ô?À1À#ÑEÐErc   Úsave_directoryÚfilename_prefixc                 ób   — | j         j                             ||¬¦  «        }t          |¦  «        S )N)Úname)rP   ÚmodelÚsaver�   )r\   r  r  Úfiless       rb   Úsave_vocabularyz!MarkupLMTokenizer.save_vocabularyå  s+   € Ø”Ô%×*Ò*¨>ÀÐ*ÑPÔPˆÝ�U‰|Œ|Ðrc   )NNr   r   r    r    r   r!   r"   r#   Fr$   r%   r&   r'   TF)NNNTFNNr   FNNNNNFFFFT)NNNTFNNr   NNNNNFFFFT)NF)N)*Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesrü   r   r  rJ   Údictrj   r—   r�   ÚfloatrR   rv   r   r	   Ú0MARKUPLM_ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRINGr   r   r§   r   r   r   r   r²   r   r¨   rÀ   r©   rþ   ÚDO_NOT_TRUNCATEr¸   rÂ   r   r  r	  r  r  Ú__classcell__)ra   s   @rb   r   r   [   s  ø€ € € € € ð7ð 7ðr *ÐØIÐIÐIÐØ€Eð
 HLØ)-ØØØØØØØØØØØØØØ!%Øð'bAð bAð �T˜#˜s˜(”^Ñ# d¨5°°e°Ô+<Ô&=Ñ=ÀÑDðbAð �d˜3”i‘ $Ñ&ð	bAð bAð bAð bAð bAð bAðH0ð 0ð 0ð2 ÐÐ/Ð1aÑbÔbð IMØAEØ:>Ø#'Ø05Ø6:Ø!%ØØ$)Ø)-Ø#'Ø26Ø-1Ø-1Ø*/Ø+0Ø',Ø#Øð+lð làÐ+Ñ+¨d°9¬oÑ=ÀÐEVÔ@WÑWðlð % tÐ,=Ô'>Ñ>ÀÑEðlð �T˜#”Y” $ t¨D°¬I¤Ô"7Ñ7¸$Ñ>ð	lð
 ˜#”Y  d¨3¤i¤Ñ0°4Ñ7ðlð !ðlð ˜‘˜oÑ-ðlð ˜3‘JÐ!3Ñ3ðlð ˜$‘Jðlð ðlð "ðlð   $™Jðlð ˜D‘jðlð ˜jÑ(¨4Ñ/ðlð  $ d™{ðlð   $ d™{ð!lð" $(ð#lð$ %)ð%lð& !%ð'lð( ð)lð* ð+lð. 
ð/lð lð lñ cÔbðlð\ ÐÐ/Ð1aÑbÔbð  $Ø/3Ø:>Ø#'Ø05Ø6:Ø!%ØØ)-Ø#'Ø26Ø-1Ø-1Ø*/Ø+0Ø',Ø#Øð)6
ð 6
à"& y¤/°D¸Ô4GÑ"GÈ$ÐO`ÔJaÑ"að6
ð ˜‘ð6
ð �T˜$˜sœ)”_Ô%¨Ñ,ð	6
ð
 ˜#”Y  d¨3¤i¤Ñ0°4Ñ7ð6
ð !ð6
ð ˜‘˜oÑ-ð6
ð ˜3‘JÐ!3Ñ3ð6
ð ˜$‘Jð6
ð ð6
ð   $™Jð6
ð ˜D‘jð6
ð ˜jÑ(¨4Ñ/ð6
ð  $ d™{ð6
ð  $ d™{ð6
ð  $(ð!6
ð" %)ð#6
ð$ !%ð%6
ð& ð'6
ð( ð)6
ð, 
ð-6
ð 6
ð 6
ñ cÔbð6
ðp#ð #˜Sð #¨¨d©
ð #Ètð #ÐjnÐorÔjsð #ð #ð #ð #ð ÐÐ/Ð1aÑbÔbð /3Ø)-Ø(,Ø#'Ø05Ø6:Ø!%ØØ)-Ø#'Ø26Ø-1Ø-1Ø*/Ø+0Ø',Ø#Øð)B
ð B
àÐ+Ñ+ðB
ð % tÑ+ðB
ð �T˜#”Y” $Ñ&ð	B
ð
 ˜#”Y Ñ%ðB
ð !ðB
ð ˜‘˜oÑ-ðB
ð ˜3‘JÐ!3Ñ3ðB
ð ˜$‘JðB
ð ðB
ð   $™JðB
ð ˜D‘jðB
ð ˜jÑ(¨4Ñ/ðB
ð  $ d™{ðB
ð  $ d™{ðB
ð  $(ð!B
ð" %)ð#B
ð$ !%ð%B
ð& ð'B
ð( ð)B
ð, 
ð-B
ð B
ð B
ñ cÔbðB
ðN  $Ø/3Ø.2Ø#'Ø,;Ô,FØ2DÔ2TØ!%ØØ)-Ø#'Ø%)Ø-1Ø-1Ø*/Ø+0Ø',Ø#Øð)m`ð m`à"& y¤/°D¸Ô4GÑ"GÈ$ÐO`ÔJaÑ"aðm`ð ˜‘ðm`ð �T˜$˜sœ)”_Ô%¨Ñ,ð	m`ð
 ˜$˜sœ)”_ tÑ+ðm`ð !ðm`ð *ðm`ð 0ðm`ð ˜$‘Jðm`ð ðm`ð   $™Jðm`ð ˜D‘jðm`ð ˜d™
ðm`ð  $ d™{ðm`ð  $ d™{ðm`ð  $(ð!m`ð" %)ð#m`ð$ !%ð%m`ð& ð'm`ð( ð)m`ð* 
ð+m`ð m`ð m`ð m`ðd /3Ø)-Ø(,Ø#'Ø,;Ô,FØ2DÔ2TØ!%ØØ)-Ø#'Ø&*Ø-1Ø-1Ø*/Ø+0Ø',Ø#Øð)Zð ZàÐ+Ñ+ðZð % tÑ+ðZð �T˜#”Y” $Ñ&ð	Zð
 ˜#”Y Ñ%ðZð !ðZð *ðZð 0ðZð ˜$‘JðZð ðZð   $™JðZð ˜D‘jðZð ˜t™ðZð  $ d™{ðZð  $ d™{ðZð  $(ð!Zð" %)ð#Zð$ !%ð%Zð& ð'Zð( ð)Zð, 
ð-Zð Zð Zð Zð~ "&Ø,;Ô,FØ)-Ø#'Ø-1ð`ð `à˜S ,Ð.Ô/°-Ñ?ð`ð ˜$‘Jð`ð *ð	`ð
   $™Jð`ð ˜D‘jð`ð  $ d™{ð`ð 
ð`ð `ð `ð `ðF GKð;ð ;Ø œ9ð;Ø37¸´9¸tÑ3Cð;à	ˆcŒð;ð ;ð ;ð ;ð0 GKðFð FØ œ9ðFØ37¸´9¸tÑ3CðFà	ˆcŒðFð Fð Fð Fð,ð ¨cð ÀCÈ$ÁJð ÐZ_Ð`cÔZdð ð ð ð ð ð ð ð rc   r   ÚMarkupLMTokenizerFastN)Ú
tokenizersr   r   r   r   Útokenizers.modelsr   Útokenization_utils_baser	   r
   r   r   r   r   r   r   r   r   Útokenization_utils_tokenizersr   Úutilsr   r   Ú
get_loggerr  Úloggerr  r  r   r  Ú__all__rH   rc   rb   ú<module>r(     s]  ðð  GÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ !Ð !Ð !Ð !Ð !Ð !ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ?Ð >Ð >Ð >Ð >Ð >Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0ð 
ˆÔ	˜HÑ	%Ô	%€à#/ÀÐ`pÐqÐqÐ ð04Ð 0ðfLð Lð Lð Lð LÐ)ñ Lô Lð Lð^ *Ð ð Ð 7Ð
8€€€rc   