§
    ‚Štjßp  ã                   ó¾  — d dl Z d dlZd dlmZ d dlmZmZ d dlmZ d dl	Z
d dlmZ ddlmZ ddlmZmZmZ dd	lmZmZmZ d
dlmZ h d£Z e¦   «         r
d dlZd dlmZ  ej        e¦  «        Zd„ Zd„ Zd„ Z d„ Z!d„ Z"defd„Z#	 	 	 	 d%d„Z$ G d„ de¦  «        Z% G d„ de%¦  «        Z& G d„ de%¦  «        Z' G d„ d ¦  «        Z( G d!„ d"¦  «        Z) G d#„ d$¦  «        Z*dS )&é    N)Úpartial)ÚPoolÚ	cpu_count)Ú
ThreadPool)Útqdmé   )Úwhitespace_tokenize)ÚBatchEncodingÚPreTrainedTokenizerBaseÚTruncationStrategy)Úis_torch_availableÚis_torch_hpu_availableÚloggingé   )ÚDataProcessor>   ÚbartÚmpnetÚrobertaÚ	camembert)ÚTensorDatasetc                 ó  — d                      |                     |¦  «        ¦  «        }t          ||dz   ¦  «        D ]G}t          ||dz
  d¦  «        D ]0}d                      | ||dz   …         ¦  «        }||k    r||fc c S Œ1ŒH||fS )zFReturns tokenized answer spans that better match the annotated answer.ú r   éÿÿÿÿ)ÚjoinÚtokenizeÚrange)	Ú
doc_tokensÚinput_startÚ	input_endÚ	tokenizerÚorig_answer_textÚtok_answer_textÚ	new_startÚnew_endÚ	text_spans	            ú`/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/data/processors/squad.pyÚ_improve_answer_spanr'   *   s¶   € à—h’h˜y×1Ò1Ð2BÑCÔCÑDÔD€Oå˜;¨	°A©Ñ6Ô6ð ,ð ,ˆ	Ý˜Y¨	°A©°rÑ:Ô:ð 	,ð 	,ˆGØŸš ¨I¸À1¹Ð,EÔ!FÑGÔGˆIØ˜OÒ+Ð+Ø! 7Ð+Ð+Ð+Ð+Ð+Ð+ð ,ð	,ð
 ˜Ð#Ð#ó    c                 óö   — d}d}t          | ¦  «        D ]`\  }}|j        |j        z   dz
  }||j        k     rŒ#||k    rŒ*||j        z
  }||z
  }	t          ||	¦  «        d|j        z  z   }
|�|
|k    r|
}|}Œa||k    S )ú:Check if this is the 'max context' doc span for the token.Nr   ç{®Gáz„?)Ú	enumerateÚstartÚlengthÚmin©Ú	doc_spansÚcur_span_indexÚpositionÚ
best_scoreÚbest_span_indexÚ
span_indexÚdoc_spanÚendÚnum_left_contextÚnum_right_contextÚscores              r&   Ú_check_is_max_contextr<   7   s±   € à€JØ€OÝ )¨)Ñ 4Ô 4ð )ð )Ñˆ
�HØŒn˜xœÑ.°Ñ2ˆØ�h”nÒ$Ð$ØØ�cŠ>ˆ>ØØ# h¤nÑ4ÐØ (™NÐÝÐ$Ð&7Ñ8Ô8¸4À(Ä/Ñ;QÑQˆØÐ ¨Ò!3Ð!3ØˆJØ(ˆOøà˜_Ò,Ð,r(   c                 ó   — d}d}t          | ¦  «        D ]e\  }}|d         |d         z   dz
  }||d         k     rŒ&||k    rŒ-||d         z
  }||z
  }	t          ||	¦  «        d|d         z  z   }
|�|
|k    r|
}|}Œf||k    S )r*   Nr-   r.   r   r+   )r,   r/   r0   s              r&   Ú_new_check_is_max_contextr>   K   sÂ   € ð €JØ€OÝ )¨)Ñ 4Ô 4ð )ð )Ñˆ
�HØ�wÔ (¨8Ô"4Ñ4°qÑ8ˆØ�h˜wÔ'Ò'Ð'ØØ�cŠ>ˆ>ØØ# h¨wÔ&7Ñ7ÐØ (™NÐÝÐ$Ð&7Ñ8Ô8¸4À(È8ÔBTÑ;TÑTˆØÐ ¨Ò!3Ð!3ØˆJØ(ˆOøà˜_Ò,Ð,r(   c                 ó`   — | dk    s%| dk    s| dk    s| dk    st          | ¦  «        dk    rdS dS )Nr   ú	úú
i/   TF)Úord)Úcs    r&   Ú_is_whitespacerE   a   s=   € ØˆC‚x€x�1˜’9�9  T¢	 	¨Q°$ªY¨Y½#¸a¹&¼&ÀFÒ:JÐ:JØˆtØˆ5r(   c                 óê  — g }|r�| j         s–| j        }| j        }d                     | j        ||dz   …         ¦  «        }	d                     t          | j        ¦  «        ¦  «        }
|	                     |
¦  «        dk    r#t           	                    d|	› d|
› d�¦  «         g S g }g }g }t          | j        ¦  «        D ] \  }}|                     t          |¦  «        ¦  «         t          j        j        dv rt                               |d¬	¦  «        }nt                               |¦  «        }|D ],}|                     |¦  «         |                     |¦  «         Œ-Œ¡|rz| j         ss|| j                 }| j        t          | j        ¦  «        dz
  k     r|| j        dz            dz
  }nt          |¦  «        dz
  }t#          |||t          | j        ¦  «        \  }}g }t                               | j        d
d|¬¦  «        }t)          t          ¦  «        j                             dd¦  «                             ¦   «         }|t.          v rt          j        t          j        z
  dz   nt          j        t          j        z
  }t          j        t                               d¬¦  «        z
  }t          j        |z
  }|}t          |¦  «        |z  t          |¦  «        k     �rt          j        dk    r|}|}t8          j        j        }n|}|}t8          j        j        }t          |||||d||z
  t          |¦  «        z
  |z
  d¬¦  «        }tA          t          |¦  «        t          |¦  «        |z  z
  |t          |¦  «        z
  |z
  ¦  «        }t          j!        |d         v rŸt          j        dk    r4|d         d |d          "                    t          j!        ¦  «        …         } nct          |d         ¦  «        dz
  |d         d d d…          "                    t          j!        ¦  «        z
  }!|d         |!dz   d …         } n|d         } t           #                    | ¦  «        }"i }#tI          |¦  «        D ]G}t          j        dk    rt          |¦  «        |z   |z   n|}$|t          |¦  «        |z  |z            |#|$<   ŒH||d<   |"|d<   |#|d<   t          |¦  «        |z   |d<   i |d<   t          |¦  «        |z  |d<   ||d<   |                     |¦  «         d|vsd|v rt          |d         ¦  «        dk    rn,|d         }t          |¦  «        |z  t          |¦  «        k     �°tI          t          |¦  «        ¦  «        D ]k}%tI          ||%         d         ¦  «        D ]M}&tK          ||%|%|z  |&z   ¦  «        }'t          j        dk    r|&n||%         d         |&z   }$|'||%         d         |$<   ŒNŒl|D �]}(|(d          "                    t          j&        ¦  «        })tO          j(        |(d         ¦  «        }*t          j        dk    rd|*t          |¦  «        |z   d …<   n,d|*t          |(d         ¦  «         t          |¦  «        |z    …<   tO          j)        tO          j*        |(d         t          j!        k    ¦  «        ¦  «        }+tO          j+        t           ,                    |(d         d¬¦  «        ¦  «         -                    ¦   «         },d|*|+<   d|*|,<   d|*|)<   | j         }-d}d}|rl|-sj|(d         }.|(d         |(d         z   dz
  }/d
}0||.k    r||/k    sd}0|0r|)}|)}d}-n5t          j        dk    rd}1nt          |¦  «        |z   }1||.z
  |1z   }||.z
  |1z   }|                     t]          |(d         |(d         |(d         |)|* /                    ¦   «         dd|(d         |(d         |(d         |(d         |||-| j0        ¬¦  «        ¦  «         �Œ|S ) Nr   r   r   zCould not find answer: 'z' vs. 'ú')ÚRobertaTokenizerÚLongformerTokenizerÚBartTokenizerÚLongformerTokenizerFastÚBartTokenizerFastT)Úadd_prefix_spaceF)Úadd_special_tokensÚ
truncationÚ
max_lengthÚ	TokenizerÚ )ÚpairÚright)rO   ÚpaddingrP   Úreturn_overflowing_tokensÚstrideÚreturn_token_type_idsÚ	input_idsÚparagraph_lenÚtokensÚtoken_to_orig_mapÚ*truncated_query_with_special_tokens_lengthÚtoken_is_max_contextr-   r.   Úoverflowing_tokensr   ÚleftÚtoken_type_ids)Úalready_has_special_tokensÚattention_mask)
Úexample_indexÚ	unique_idrZ   r^   r[   r\   Ústart_positionÚend_positionÚis_impossibleÚqas_id)1rh   rf   rg   r   r   r	   Úanswer_textÚfindÚloggerÚwarningr,   ÚappendÚlenr    Ú	__class__Ú__name__r   r'   ÚencodeÚquestion_textÚtypeÚreplaceÚlowerÚMULTI_SEP_TOKENS_TOKENIZERS_SETÚmodel_max_lengthÚmax_len_single_sentenceÚnum_special_tokens_to_addÚpadding_sider   ÚONLY_SECONDÚvalueÚ
ONLY_FIRSTr/   Úpad_token_idÚindexÚconvert_ids_to_tokensr   r>   Úcls_token_idÚnpÚ	ones_likeÚwhereÚ
atleast_1dÚasarrayÚget_special_tokens_maskÚnonzeroÚSquadFeaturesÚtolistri   )2ÚexampleÚmax_seq_lengthÚ
doc_strideÚmax_query_lengthÚpadding_strategyÚis_trainingÚfeaturesrf   rg   Úactual_textÚcleaned_answer_textÚtok_to_orig_indexÚorig_to_tok_indexÚall_doc_tokensÚiÚtokenÚ
sub_tokensÚ	sub_tokenÚtok_start_positionÚtok_end_positionÚspansÚtruncated_queryÚtokenizer_typeÚsequence_added_tokensÚmax_len_sentences_pairÚsequence_pair_added_tokensÚspan_doc_tokensÚtextsÚpairsrO   Úencoded_dictrZ   Únon_padded_idsÚlast_padding_id_positionr[   r\   r€   Údoc_span_indexÚjÚis_max_contextÚspanÚ	cls_indexÚp_maskÚpad_token_indicesÚspecial_token_indicesÚspan_is_impossibleÚ	doc_startÚdoc_endÚout_of_spanÚ
doc_offsets2                                                     r&   Ú!squad_convert_example_to_featuresr·   g   s	  € ð €HØð 
˜7Ô0ð 
à Ô/ˆØÔ+ˆð —h’h˜wÔ1°.ÀLÐSTÑDTÐ2UÔVÑWÔWˆØ!ŸhšhÕ':¸7Ô;NÑ'OÔ'OÑPÔPÐØ×ÒÐ/Ñ0Ô0°BÒ6Ð6Ý�NŠNÐ`°kÐ`Ð`ÐJ]Ð`Ð`Ð`ÑaÔaÐaØˆIàÐØÐØ€NÝ˜gÔ0Ñ1Ô1ð -ð -‰ˆˆ5Ø× Ò ¥ ^Ñ!4Ô!4Ñ5Ô5Ð5ÝÔÔ'ð ,
ð 
ð 
õ #×+Ò+¨EÀDÐ+ÑIÔIˆJˆJå"×+Ò+¨EÑ2Ô2ˆJØ#ð 	-ð 	-ˆIØ×$Ò$ QÑ'Ô'Ð'Ø×!Ò! )Ñ,Ô,Ð,Ð,ð	-ð ð 	
˜7Ô0ð 	
Ø.¨wÔ/EÔFÐØÔ¥# gÔ&8Ñ"9Ô"9¸AÑ"=Ò=Ð=Ø0°Ô1EÈÑ1IÔJÈQÑNÐÐå" >Ñ2Ô2°QÑ6Ðå1EØÐ.Ð0@Å)ÈWÔM`ñ2
ô 2
Ñ.Ð	Ð-ð €Eå×&Ò&ØÔ°%ÀDÐUeð 'ñ ô €Oõ �)‘_”_Ô-×5Ò5°kÀ2ÑFÔF×LÒLÑNÔN€Nð Õ<Ð<Ð<õ 	Ô"¥YÔ%FÑFÈÑJÐJåÔ'­)Ô*KÑKð õ
 'Ô7½)×:]Ò:]ÐcgÐ:]Ñ:hÔ:hÑhÐÝ!*Ô!;Ð>TÑ!TÐà$€OÝ
ˆe‰*Œ*�zÑ
!¥C¨Ñ$7Ô$7Ò
7Ñ
7åÔ! WÒ,Ð,Ø#ˆEØ#ˆEÝ+Ô7Ô=ˆJˆJà#ˆEØ#ˆEÝ+Ô6Ô<ˆJå ØØØ!Ø$Ø%Ø&*Ø! JÑ.µ°_Ñ1EÔ1EÑEÐHbÑbØ"&ð	
ñ 	
ô 	
ˆõ Ý�ÑÔ¥# e¡*¤*¨zÑ"9Ñ9Ø�S Ñ1Ô1Ñ1Ð4NÑNñ
ô 
ˆõ
 Ô! \°+Ô%>Ð>Ð>ÝÔ%¨Ò0Ð0Ø!-¨kÔ!:Ð;t¸\È+Ô=V×=\Ò=\Õ]fÔ]sÑ=tÔ=tÐ;tÔ!u��õ ˜ [Ô1Ñ2Ô2°QÑ6¸ÀkÔ9RÐSWÐSWÐUWÐSWÔ9X×9^Ò9^Õ_hÔ_uÑ9vÔ9vÑvð )ð ".¨kÔ!:Ð;SÐVWÑ;WÐ;YÐ;YÔ!Z��ð *¨+Ô6ˆNå×0Ò0°Ñ@Ô@ˆàÐÝ�}Ñ%Ô%ð 	Vð 	VˆAÝHQÔH^ÐbiÒHiÐHi•C˜Ñ(Ô(Ð+@Ñ@À1ÑDÐDÐopˆEØ'8½¸U¹¼ÀjÑ9PÐSTÑ9TÔ'UÐ˜eÑ$Ð$à(5ˆ�_Ñ%Ø!'ˆ�XÑØ,=ˆÐ(Ñ)ÝEHÈÑEYÔEYÐ\qÑEqˆÐAÑBØ/1ˆÐ+Ñ,Ý # E¡
¤
¨ZÑ 7ˆ�WÑØ!.ˆ�XÑà�Š�\Ñ"Ô"Ð"à |Ð3Ð3Ø  LÐ0Ð0µS¸ÐFZÔ9[Ñ5\Ô5\Ð`aÒ5aÐ5aàØ&Ð';Ô<ˆõy ˆe‰*Œ*�zÑ
!¥C¨Ñ$7Ô$7Ò
7Ñ
7õ|  ¥ E¡
¤
Ñ+Ô+ð Rð RˆÝ�u˜^Ô,¨_Ô=Ñ>Ô>ð 	Rð 	RˆAÝ6°u¸nÈnÐ_iÑNiÐlmÑNmÑnÔnˆNõ Ô)¨VÒ3Ð3ð �à˜>Ô*Ð+WÔXÐ[\Ñ\ð ð
 DRˆE�.Ô!Ð"8Ô9¸%Ñ@Ð@ð	Rð ð A
ñ A
ˆà˜Ô%×+Ò+­IÔ,BÑCÔCˆ	õ ”˜dÐ#3Ô4Ñ5Ô5ˆÝÔ! WÒ,Ð,ØEFˆF•3�Ñ'Ô'Ð*?Ñ?ÐAÐAÑBÐBà]^ˆF•C˜˜XœÑ'Ô'Ð'­C°Ñ,@Ô,@ÐCXÑ,XÐ*YÐYÑZåœH¥R¤]°4¸Ô3DÍ	ÔH^Ò3^Ñ%_Ô%_Ñ`Ô`ÐÝ "¤
Ý×-Ò-¨d°;Ô.?Ð\`Ð-ÑaÔañ!
ô !
ç
Š'‰)Œ)ð 	ð %&ˆÐ Ñ!Ø()ˆÐ$Ñ%ð ˆˆyÑà$Ô2ÐØˆØˆØð 	IÐ1ð 	Ið ˜WœˆIØ˜7”m d¨8¤nÑ4°qÑ8ˆGØˆKà&¨)Ò3Ð3Ð8HÈGÒ8SÐ8SØ"�àð IØ!*�Ø(�Ø%)Ð"Ð"åÔ)¨VÒ3Ð3Ø!"�J�Jå!$ _Ñ!5Ô!5Ð8MÑ!M�Jà!3°iÑ!?À*Ñ!L�Ø/°)Ñ;¸jÑH�Ø�ŠÝØ�[Ô!ØÐ%Ô&ØÐ%Ô&ØØ—’‘”ØØØ" ?Ô3Ø%)Ð*@Ô%AØ˜H”~Ø"&Ð':Ô";Ø-Ø)Ø0Ø”~ðñ ô ñ	
ô 	
ð 	
ñ 	
ð& €Or(   Útokenizer_for_convertc                 ó
   — | a d S ©N)r    )r¸   s    r&   Ú&squad_convert_example_to_features_initr»   4  s   € à%€I€I€Ir(   rP   FTc
           
      óþ  — t          |t          ¦   «         ¦  «        }t          ¦   «         rt          nt          }
 |
|t
          |f¬¦  «        5 }t          t          |||||¬¦  «        }t          t          | 
                    || d¬¦  «        t          | ¦  «        d|	 ¬¦  «        ¦  «        }ddd¦  «         n# 1 swxY w Y   g }d}d	}t          |t          |¦  «        d
|	 ¬¦  «        D ]7}|sŒ|D ]*}||_        ||_        |                     |¦  «         |dz  }Œ+|dz  }Œ8|}~|dk    �rÑt          ¦   «         st!          d¦  «        ‚t#          j        d„ |D ¦   «         t"          j        ¬¦  «        }t#          j        d„ |D ¦   «         t"          j        ¬¦  «        }t#          j        d„ |D ¦   «         t"          j        ¬¦  «        }t#          j        d„ |D ¦   «         t"          j        ¬¦  «        }t#          j        d„ |D ¦   «         t"          j        ¬¦  «        }t#          j        d„ |D ¦   «         t"          j        ¬¦  «        }|sHt#          j        |                     d	¦  «        t"          j        ¬¦  «        }t/          ||||||¦  «        }njt#          j        d„ |D ¦   «         t"          j        ¬¦  «        }t#          j        d„ |D ¦   «         t"          j        ¬¦  «        }t/          ||||||||¦  «        }||fS |S )aq  
    Converts a list of examples into a list of features that can be directly given as input to a model. It is
    model-dependant and takes advantage of many of the tokenizer's features to create the model's inputs.

    Args:
        examples: list of [`~data.processors.squad.SquadExample`]
        tokenizer: an instance of a child of [`PreTrainedTokenizer`]
        max_seq_length: The maximum sequence length of the inputs.
        doc_stride: The stride used when the context is too large and is split across several features.
        max_query_length: The maximum length of the query.
        is_training: whether to create features for model evaluation or model training.
        padding_strategy: Default to "max_length". Which padding strategy to use
        return_dataset: Default False. Can also be 'pt'.
            if 'pt': returns a torch.data.TensorDataset.
        threads: multiple processing threads.


    Returns:
        list of [`~data.processors.squad.SquadFeatures`]

    Example:

    ```python
    processor = SquadV2Processor()
    examples = processor.get_dev_examples(data_dir)

    features = squad_convert_examples_to_features(
        examples=examples,
        tokenizer=tokenizer,
        max_seq_length=args.max_seq_length,
        doc_stride=args.doc_stride,
        max_query_length=args.max_query_length,
        is_training=not evaluate,
    )
    ```)ÚinitializerÚinitargs)r�   rŽ   r�   r�   r‘   é    )Ú	chunksizez"convert squad examples to features)ÚtotalÚdescÚdisableNi Êš;r   zadd example index and unique idr   Úptz6PyTorch must be installed to return a PyTorch dataset.c                 ó   — g | ]	}|j         ‘Œ
S © )rY   ©Ú.0Úfs     r&   ú
<listcomp>z6squad_convert_examples_to_features.<locals>.<listcomp>’  ó   € Ð%DÐ%DÐ%D°a a¤kÐ%DÐ%DÐ%Dr(   )Údtypec                 ó   — g | ]	}|j         ‘Œ
S rÆ   )rc   rÇ   s     r&   rÊ   z6squad_convert_examples_to_features.<locals>.<listcomp>“  s   € Ð+OÐ+OÐ+OÀ¨AÔ,<Ð+OÐ+OÐ+Or(   c                 ó   — g | ]	}|j         ‘Œ
S rÆ   )ra   rÇ   s     r&   rÊ   z6squad_convert_examples_to_features.<locals>.<listcomp>”  s   € Ð*NÐ*NÐ*NÀ¨1Ô+;Ð*NÐ*NÐ*Nr(   c                 ó   — g | ]	}|j         ‘Œ
S rÆ   )r®   rÇ   s     r&   rÊ   z6squad_convert_examples_to_features.<locals>.<listcomp>•  rË   r(   c                 ó   — g | ]	}|j         ‘Œ
S rÆ   )r¯   rÇ   s     r&   rÊ   z6squad_convert_examples_to_features.<locals>.<listcomp>–  s   € Ð">Ð">Ð">° 1¤8Ð">Ð">Ð">r(   c                 ó   — g | ]	}|j         ‘Œ
S rÆ   )rh   rÇ   s     r&   rÊ   z6squad_convert_examples_to_features.<locals>.<listcomp>—  s   € Ð)LÐ)LÐ)L¸a¨!¬/Ð)LÐ)LÐ)Lr(   c                 ó   — g | ]	}|j         ‘Œ
S rÆ   )rf   rÇ   s     r&   rÊ   z6squad_convert_examples_to_features.<locals>.<listcomp>Ÿ  s   € Ð/SÐ/SÐ/SÀQ°Ô0@Ð/SÐ/SÐ/Sr(   c                 ó   — g | ]	}|j         ‘Œ
S rÆ   )rg   rÇ   s     r&   rÊ   z6squad_convert_examples_to_features.<locals>.<listcomp>   s   € Ð-OÐ-OÐ-OÀ¨a¬nÐ-OÐ-OÐ-Or(   )r/   r   r   r   r   r»   r   r·   Úlistr   Úimapro   rd   re   rn   r   ÚRuntimeErrorÚtorchÚtensorÚlongÚfloatÚarangeÚsizer   )Úexamplesr    r�   rŽ   r�   r‘   r�   Úreturn_datasetÚthreadsÚtqdm_enabledÚpool_clsÚpÚ	annotate_r’   Únew_featuresre   rd   Úexample_featuresÚexample_featureÚall_input_idsÚall_attention_masksÚall_token_type_idsÚall_cls_indexÚ
all_p_maskÚall_is_impossibleÚall_feature_indexÚdatasetÚall_start_positionsÚall_end_positionss                                r&   Ú"squad_convert_examples_to_featuresrñ   9  s›  € õ` �'�9™;œ;Ñ'Ô'€GÝ3Ñ5Ô5Ð?�zˆz½4€HØ	ˆ�'Õ'MÐYbÐXdÐ	eÑ	eÔ	eð 
ÐijÝÝ-Ø)Ø!Ø-Ø-Ø#ð
ñ 
ô 
ˆ	õ ÝØ—’�y (°b�Ñ9Ô9Ý˜(‘m”mØ9Ø(Ð(ð	ñ ô ñ
ô 
ˆð
ð 
ð 
ñ 
ô 
ð 
ð 
ð 
ð 
ð 
ð 
øøøð 
ð 
ð 
ð 
ð$ €LØ€IØ€MÝ Ø�˜H™œÐ,MÐ[gÐWgðñ ô ð 
ð 
Ðð  ð 	ØØ/ð 	ð 	ˆOØ,9ˆOÔ)Ø(1ˆOÔ%Ø×Ò Ñ0Ô0Ð0Ø˜‰NˆIˆIØ˜ÑˆˆØ€HØØ˜ÒÑÝ!Ñ#Ô#ð 	YÝÐWÑXÔXÐXõ œÐ%DÐ%D¸8Ð%DÑ%DÔ%DÍEÌJÐWÑWÔWˆÝ#œlÐ+OÐ+OÀhÐ+OÑ+OÔ+OÕW\ÔWaÐbÑbÔbÐÝ"œ\Ð*NÐ*NÀXÐ*NÑ*NÔ*NÕV[ÔV`ÐaÑaÔaÐÝœÐ%DÐ%D¸8Ð%DÑ%DÔ%DÍEÌJÐWÑWÔWˆÝ”\Ð">Ð">°XÐ">Ñ">Ô">ÅeÄkÐRÑRÔRˆ
Ý!œLÐ)LÐ)LÀ8Ð)LÑ)LÔ)LÕTYÔT_Ð`Ñ`Ô`Ðàð 	Ý %¤¨]×-?Ò-?ÀÑ-BÔ-BÍ%Ì*Ð UÑ UÔ UÐÝ#ØÐ2Ð4FÐHYÐ[hÐjtñô ˆGˆGõ #(¤,Ð/SÐ/SÈ(Ð/SÑ/SÔ/SÕ[`Ô[eÐ"fÑ"fÔ"fÐÝ %¤Ð-OÐ-OÀhÐ-OÑ-OÔ-OÕW\ÔWaÐ bÑ bÔ bÐÝ#ØØ#Ø"Ø#Ø!ØØØ!ñ	ô 	ˆGð ˜Ð Ð àˆs   ÁAB8Â8B<Â?B<c                   ó@   — e Zd ZdZdZdZd	d„Zd	d„Zd
d„Zd
d„Z	d„ Z
dS )ÚSquadProcessorz¤
    Processor for the SQuAD data set. overridden by SquadV1Processor and SquadV2Processor, used by the version 1.1 and
    version 2.0 of SQuAD, respectively.
    NFc           
      óº  — |sb|d         d         d                               ¦   «                              d¦  «        }|d         d         d                               ¦   «         }g }n6d„ t          |d         d         |d         d         ¦  «        D ¦   «         }d }d }t          |d                               ¦   «                              d¦  «        |d                               ¦   «                              d¦  «        |d	                               ¦   «                              d¦  «        |||d
                               ¦   «                              d¦  «        |¬¦  «        S )NÚanswersÚtextr   úutf-8Úanswer_startc                 óŒ   — g | ]A\  }}|                      ¦   «         |                      ¦   «                              d ¦  «        dœ‘ŒBS )r÷   )rø   rö   )ÚnumpyÚdecode)rÈ   r-   rö   s      r&   rÊ   z@SquadProcessor._get_example_from_tensor_dict.<locals>.<listcomp>À  sR   € ð ð ð á�E˜4ð "'§¢¡¤¸¿
º
¹¼×8KÒ8KÈGÑ8TÔ8TÐUÐUðð ð r(   ÚidÚquestionÚcontextÚtitle)ri   rs   Úcontext_textrj   Ústart_position_characterrÿ   rõ   )rú   rû   ÚzipÚSquadExample)ÚselfÚtensor_dictÚevaluateÚanswerrø   rõ   s         r&   Ú_get_example_from_tensor_dictz,SquadProcessor._get_example_from_tensor_dictº  sQ  € Øð 	 Ø  Ô+¨FÔ3°AÔ6×<Ò<Ñ>Ô>×EÒEÀgÑNÔNˆFØ& yÔ1°.ÔAÀ!ÔD×JÒJÑLÔLˆLØˆGˆGðð å#& {°9Ô'=¸nÔ'MÈ{Ð[dÔOeÐflÔOmÑ#nÔ#nðñ ô ˆGð
 ˆFØˆLåØ˜tÔ$×*Ò*Ñ,Ô,×3Ò3°GÑ<Ô<Ø% jÔ1×7Ò7Ñ9Ô9×@Ò@ÀÑIÔIØ$ YÔ/×5Ò5Ñ7Ô7×>Ò>¸wÑGÔGØØ%1Ø˜gÔ&×,Ò,Ñ.Ô.×5Ò5°gÑ>Ô>Øð
ñ 
ô 
ð 	
r(   c                 ó¨   — |r	|d         }n|d         }g }t          |¦  «        D ],}|                     |                      ||¬¦  «        ¦  «         Œ-|S )av  
        Creates a list of [`~data.processors.squad.SquadExample`] using a TFDS dataset.

        Args:
            dataset: The tfds dataset loaded from *tensorflow_datasets.load("squad")*
            evaluate: Boolean specifying if in evaluation mode or in training mode

        Returns:
            List of SquadExample

        Examples:

        ```python
        >>> import tensorflow_datasets as tfds

        >>> dataset = tfds.load("squad")

        >>> training_examples = get_examples_from_dataset(dataset, evaluate=False)
        >>> evaluation_examples = get_examples_from_dataset(dataset, evaluate=True)
        ```Ú
validationÚtrain)r  )r   rn   r  )r  rî   r  rÝ   r  s        r&   Úget_examples_from_datasetz(SquadProcessor.get_examples_from_datasetÒ  sl   € ð, ð 	'Ø˜lÔ+ˆGˆGà˜gÔ&ˆGàˆÝ ™=œ=ð 	`ð 	`ˆKØ�OŠO˜D×>Ò>¸{ÐU]Ð>Ñ^Ô^Ñ_Ô_Ð_Ð_àˆr(   c                 ó8  — |€d}| j         €t          d¦  «        ‚t          t          j                             ||€| j         n|¦  «        dd¬¦  «        5 }t          j        |¦  «        d         }ddd¦  «         n# 1 swxY w Y   |                      |d¦  «        S )	a–  
        Returns the training examples from the data directory.

        Args:
            data_dir: Directory containing the data files used for training and evaluating.
            filename: None by default, specify this if the training file has a different name than the original one
                which is `train-v1.1.json` and `train-v2.0.json` for squad versions 1.1 and 2.0 respectively.

        NrR   úNSquadProcessor should be instantiated via SquadV1Processor or SquadV2ProcessorÚrr÷   ©ÚencodingÚdatar  )	Ú
train_fileÚ
ValueErrorÚopenÚosÚpathr   ÚjsonÚloadÚ_create_examples©r  Údata_dirÚfilenameÚreaderÚ
input_datas        r&   Úget_train_examplesz!SquadProcessor.get_train_examplesó  sÞ   € ð ÐØˆHàŒ?Ð"ÝÐmÑnÔnÐnåÝŒG�LŠL˜°hÐ6F 4¤? ?ÈHÑUÔUÐWZÐelð
ñ 
ô 
ð 	3àÝœ 6Ñ*Ô*¨6Ô2ˆJð	3ð 	3ð 	3ñ 	3ô 	3ð 	3ð 	3ð 	3ð 	3ð 	3ð 	3øøøð 	3ð 	3ð 	3ð 	3ð ×$Ò$ Z°Ñ9Ô9Ð9ó   ÁA;Á;A?ÂA?c                 ó8  — |€d}| j         €t          d¦  «        ‚t          t          j                             ||€| j         n|¦  «        dd¬¦  «        5 }t          j        |¦  «        d         }ddd¦  «         n# 1 swxY w Y   |                      |d¦  «        S )	a”  
        Returns the evaluation example from the data directory.

        Args:
            data_dir: Directory containing the data files used for training and evaluating.
            filename: None by default, specify this if the evaluation file has a different name than the original one
                which is `dev-v1.1.json` and `dev-v2.0.json` for squad versions 1.1 and 2.0 respectively.
        NrR   r  r  r÷   r  r  Údev)	Údev_filer  r  r  r  r   r  r  r  r  s        r&   Úget_dev_exampleszSquadProcessor.get_dev_examples	  sÞ   € ð ÐØˆHàŒ=Ð ÝÐmÑnÔnÐnåÝŒG�LŠL˜°HÐ4D 4¤= =È(ÑSÔSÐUXÐcjð
ñ 
ô 
ð 	3àÝœ 6Ñ*Ô*¨6Ô2ˆJð	3ð 	3ð 	3ñ 	3ô 	3ð 	3ð 	3ð 	3ð 	3ð 	3ð 	3øøøð 	3ð 	3ð 	3ð 	3ð ×$Ò$ Z°Ñ7Ô7Ð7r!  c                 óŒ  — |dk    }g }t          |¦  «        D ]«}|d         }|d         D ]˜}|d         }|d         D ]…}	|	d         }
|	d         }d }d }g }|	                     dd	¦  «        }|s)|r|	d
         d         }|d         }|d         }n|	d
         }t          |
|||||||¬¦  «        }|                     |¦  «         Œ†Œ™Œ¬|S )Nr  rÿ   Ú
paragraphsrþ   Úqasrü   rý   rh   Frõ   r   rö   rø   )ri   rs   r   rj   r  rÿ   rh   rõ   )r   Úgetr  rn   )r  r  Úset_typer‘   rÝ   Úentryrÿ   Ú	paragraphr   Úqari   rs   r  rj   rõ   rh   r  rŒ   s                     r&   r  zSquadProcessor._create_examples  s*  € Ø 'Ò)ˆØˆÝ˜*Ñ%Ô%ð 	-ð 	-ˆEØ˜'”NˆEØ" <Ô0ð -ð -�	Ø(¨Ô3�Ø# EÔ*ð -ð -�BØ œX�FØ$& z¤N�MØ/3Ð,Ø"&�KØ �Gà$&§F¢F¨?¸EÑ$BÔ$B�MØ(ð 4Ø&ð 4Ø%'¨	¤]°1Ô%5˜FØ*0°¬.˜KØ7=¸nÔ7MÐ4Ð4à&(¨¤m˜Gå*Ø%Ø&3Ø%1Ø$/Ø1IØ#Ø&3Ø 'ð	ñ 	ô 	�Gð —O’O GÑ,Ô,Ð,Ð,ð5-ð-ð: ˆr(   )Frº   )rq   Ú
__module__Ú__qualname__Ú__doc__r  r$  r  r  r   r%  r  rÆ   r(   r&   ró   ró   ±  sŠ   € € € € € ðð ð
 €JØ€Hð
ð 
ð 
ð 
ð0ð ð ð ðB:ð :ð :ð :ð,8ð 8ð 8ð 8ð*"ð "ð "ð "ð "r(   ró   c                   ó   — e Zd ZdZdZdS )ÚSquadV1Processorztrain-v1.1.jsonzdev-v1.1.jsonN©rq   r.  r/  r  r$  rÆ   r(   r&   r2  r2  C  ó   € € € € € Ø"€JØ€H€H€Hr(   r2  c                   ó   — e Zd ZdZdZdS )ÚSquadV2Processorztrain-v2.0.jsonzdev-v2.0.jsonNr3  rÆ   r(   r&   r6  r6  H  r4  r(   r6  c                   ó   — e Zd ZdZg dfd„ZdS )r  aT  
    A single training/test example for the Squad dataset, as loaded from disk.

    Args:
        qas_id: The example's unique identifier
        question_text: The question string
        context_text: The context string
        answer_text: The answer string
        start_position_character: The character position of the start of the answer
        title: The title of the example
        answers: None by default, this is used during evaluation. Holds answers as well as their start positions.
        is_impossible: False by default, set to True if the example has no possible answer.
    Fc	                 ó*  — || _         || _        || _        || _        || _        || _        || _        d\  | _        | _        g }	g }
d}| j        D ]c}t          |¦  «        rd}n*|r|	 
                    |¦  «         n|	dxx         |z  cc<   d}|
 
                    t          |	¦  «        dz
  ¦  «         Œd|	| _        |
| _        |�O|sO|
|         | _        |
t          |t          |¦  «        z   dz
  t          |
¦  «        dz
  ¦  «                 | _        d S d S d S )N)r   r   Tr   Fr   )ri   rs   r   rj   rÿ   rh   rõ   rf   rg   rE   rn   ro   r   Úchar_to_word_offsetr/   )r  ri   rs   r   rj   r  rÿ   rõ   rh   r   r9  Úprev_is_whitespacerD   s                r&   Ú__init__zSquadExample.__init__\  s^  € ð ˆŒØ*ˆÔØ(ˆÔØ&ˆÔØˆŒ
Ø*ˆÔØˆŒà15Ñ.ˆÔ˜TÔ.àˆ
Ø ÐØ!Ðð Ô"ð 		<ð 		<ˆAÝ˜aÑ Ô ð +Ø%)Ð"Ð"à%ð (Ø×%Ò% aÑ(Ô(Ð(Ð(à˜r�N�N”N aÑ'�N�N‘NØ%*Ð"Ø×&Ò&¥s¨:¡¤¸Ñ':Ñ;Ô;Ð;Ð;à$ˆŒØ#6ˆÔ ð $Ð/¸Ð/Ø"5Ð6NÔ"OˆDÔØ 3ÝÐ,­s°;Ñ/?Ô/?Ñ?À!ÑCÅSÐI\ÑE]ÔE]Ð`aÑEaÑbÔbô!ˆDÔÐÐð 0Ð/Ð/Ð/r(   N©rq   r.  r/  r0  r;  rÆ   r(   r&   r  r  M  s:   € € € € € ðð ð, Øð-ð -ð -ð -ð -ð -r(   r  c                   ó4   — e Zd ZdZ	 	 ddedz  dedz  fd„ZdS )rŠ   aƒ  
    Single squad example features to be fed to a model. Those features are model-specific and can be crafted from
    [`~data.processors.squad.SquadExample`] using the
    :method:*~transformers.data.processors.squad.squad_convert_examples_to_features* method.

    Args:
        input_ids: Indices of input sequence tokens in the vocabulary.
        attention_mask: Mask to avoid performing attention on padding token indices.
        token_type_ids: Segment token indices to indicate first and second portions of the inputs.
        cls_index: the index of the CLS token.
        p_mask: Mask identifying tokens that can be answers vs. tokens that cannot.
            Mask with 1 for tokens than cannot be in the answer and 0 for token that can be in an answer
        example_index: the index of the example
        unique_id: The unique Feature identifier
        paragraph_len: The length of the context
        token_is_max_context:
            List of booleans identifying which tokens have their maximum context in this feature object. If a token
            does not have their maximum context in this feature object, it means that another feature object has more
            information related to that token and should be prioritized over this feature for that token.
        tokens: list of tokens corresponding to the input ids
        token_to_orig_map: mapping between the tokens and the original text, needed in order to identify the answer.
        start_position: start of the answer token index
        end_position: end of the answer token index
        encoding: optionally store the BatchEncoding with the fast-tokenizer alignment methods.
    Nri   r  c                 óæ   — || _         || _        || _        || _        || _        || _        || _        || _        |	| _        |
| _	        || _
        || _        || _        || _        || _        || _        d S rº   )rY   rc   ra   r®   r¯   rd   re   rZ   r^   r[   r\   rf   rg   rh   ri   r  )r  rY   rc   ra   r®   r¯   rd   re   rZ   r^   r[   r\   rf   rg   rh   ri   r  s                    r&   r;  zSquadFeatures.__init__§  s�   € ð& #ˆŒØ,ˆÔØ,ˆÔØ"ˆŒØˆŒà*ˆÔØ"ˆŒØ*ˆÔØ$8ˆÔ!ØˆŒØ!2ˆÔà,ˆÔØ(ˆÔØ*ˆÔØˆŒà ˆŒˆˆr(   )NN)rq   r.  r/  r0  Ústrr
   r;  rÆ   r(   r&   rŠ   rŠ   Œ  s]   € € € € € ðð ðT "Ø)-ð#%!ð %!ð  �d‘
ð!%!ð"   $Ñ&ð#%!ð %!ð %!ð %!ð %!ð %!r(   rŠ   c                   ó   — e Zd ZdZdd„ZdS )ÚSquadResultaJ  
    Constructs a SquadResult which can be used to evaluate a model's output on the SQuAD dataset.

    Args:
        unique_id: The unique identifier corresponding to that example.
        start_logits: The logits corresponding to the start of the answer
        end_logits: The logits corresponding to the end of the answer
    Nc                 ób   — || _         || _        || _        |r|| _        || _        || _        d S d S rº   )Ústart_logitsÚ
end_logitsre   Ústart_top_indexÚend_top_indexÚ
cls_logits)r  re   rC  rD  rE  rF  rG  s          r&   r;  zSquadResult.__init__Ù  sE   € Ø(ˆÔØ$ˆŒØ"ˆŒàð 	)Ø#2ˆDÔ Ø!.ˆDÔØ(ˆDŒOˆOˆOð	)ð 	)r(   )NNNr<  rÆ   r(   r&   rA  rA  Ï  s2   € € € € € ðð ð)ð )ð )ð )ð )ð )r(   rA  )rP   Fr   T)+r  r  Ú	functoolsr   Úmultiprocessingr   r   Úmultiprocessing.poolr   rú   rƒ   r   Ú$models.bert.tokenization_bert_legacyr	   Útokenization_utils_baser
   r   r   Úutilsr   r   r   r   rw   r×   Útorch.utils.datar   Ú
get_loggerrq   rl   r'   r<   r>   rE   r·   r»   rñ   ró   r2  r6  r  rŠ   rA  rÆ   r(   r&   ú<module>rP     sÉ  ðð €€€Ø 	€	€	€	Ø Ð Ð Ð Ð Ð Ø +Ð +Ð +Ð +Ð +Ð +Ð +Ð +Ø +Ð +Ð +Ð +Ð +Ð +à Ð Ð Ð Ø Ð Ð Ð Ð Ð à GÐ GÐ GÐ GÐ GÐ GØ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aØ HÐ HÐ HÐ HÐ HÐ HÐ HÐ HÐ HÐ HØ  Ð  Ð  Ð  Ð  Ð  ð #LÐ"KÐ"KÐ ð ÐÑÔð /Ø€L€L€LØ.Ð.Ð.Ð.Ð.Ð.ð 
ˆÔ	˜HÑ	%Ô	%€ð
$ð 
$ð 
$ð-ð -ð -ð(-ð -ð -ð,ð ð ðJð Jð JðZ&ÐBYð &ð &ð &ð &ð "ØØØðuð uð uð uðpOð Oð Oð Oð O�]ñ Oô Oð Oðdð ð ð ð �~ñ ô ð ð
ð ð ð ð �~ñ ô ð ð
<ð <ð <ð <ð <ñ <ô <ð <ð~@!ð @!ð @!ð @!ð @!ñ @!ô @!ð @!ðF)ð )ð )ð )ð )ñ )ô )ð )ð )ð )r(   