§
    ‚Štj¹#  ã                   ón  — d dl Z d dlZd dlmZmZ d dlmZ d dlZd dlm	Z	 d dl
mZ ddlmZ ddlmZ dd	lmZmZ d
dlmZmZmZmZ  ej        e¦  «        Z e ej        ¦   «         ¦  «        Z ed„ eD ¦   «         ¦  «        Ze G d„ d¦  «        ¦   «         Z  G d„ de¦  «        Z! G d„ de¦  «        Z"dS )é    N)Ú	dataclassÚfield)ÚEnum)ÚFileLock)ÚDataseté   )Ú$MODEL_FOR_QUESTION_ANSWERING_MAPPING)ÚPreTrainedTokenizer)Úcheck_torch_load_is_safeÚloggingé   )ÚSquadFeaturesÚSquadV1ProcessorÚSquadV2ProcessorÚ"squad_convert_examples_to_featuresc              #   ó$   K  — | ]}|j         V — Œd S ©N)Ú
model_type)Ú.0Úconfs     ú^/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/data/datasets/squad.pyú	<genexpr>r   !   s$   è è € ÐEÐE¨�D”OÐEÐEÐEÐEÐEÐEó    c                   ó   — e Zd ZU dZ edddd                     e¦  «        z   i¬¦  «        Zee	d<    edddi¬¦  «        Z
ee	d	<    ed
ddi¬¦  «        Zee	d<    ed
ddi¬¦  «        Zee	d<    edddi¬¦  «        Zee	d<    edddi¬¦  «        Zee	d<    edddi¬¦  «        Zee	d<    edddi¬¦  «        Zee	d<    edddi¬¦  «        Zee	d<    edddi¬¦  «        Zee	d<    eddd i¬¦  «        Zee	d!<    ed"dd#i¬¦  «        Zee	d$<   dS )%ÚSquadDataTrainingArgumentszb
    Arguments pertaining to what data we are going to input our model for training and eval.
    NÚhelpz!Model type selected in the list: z, )ÚdefaultÚmetadatar   zFThe input data dir. Should contain the .json files for the SQuAD task.Údata_diré€   z‹The maximum total input sequence length after tokenization. Sequences longer than this will be truncated, sequences shorter will be padded.Úmax_seq_lengthzVWhen splitting up a long document into chunks, how much stride to take between chunks.Ú
doc_strideé@   zkThe maximum number of tokens for the question. Questions longer than this will be truncated to this length.Úmax_query_lengthé   z�The maximum length of an answer that can be generated. This is needed because the start and end predictions are not conditioned on one another.Úmax_answer_lengthFz1Overwrite the cached training and evaluation setsÚoverwrite_cachezDIf true, the SQuAD examples contain some that do not have an answer.Úversion_2_with_negativeg        zIIf null_score - best_non_null is greater than the threshold predict null.Únull_score_diff_thresholdé   Ún_best_sizer   zjlanguage id of input for language-specific xlm models (see tokenization_xlm.PRETRAINED_INIT_CONFIGURATION)Úlang_idé   z3multiple threads for converting example to featuresÚthreads)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚjoinÚMODEL_TYPESr   ÚstrÚ__annotations__r   r!   Úintr"   r$   r&   r'   Úboolr(   r)   Úfloatr+   r,   r.   © r   r   r   r   $   s­  € € € € € € ðð ð �eØ Ð(KÈdÏiÊiÐXcÑNdÔNdÑ(dÐeðñ ô €J�ð ð ñ ð �EØ Ð(pÐqðñ ô €Hˆcð ð ñ ð  ˜%ØàðQð
ðñ ô €N�Cð ð ñ ð �eØØÐrÐsðñ ô €J�ð ð ñ ð "˜EØàð/ð
ðñ ô Ð�cð ð ñ ð #˜UØàðJð
ðñ ô Ð�sð ð ñ ð "˜EØ Ð)\Ð ]ðñ ô €O�Tð ð ñ ð %* EØ Ð)oÐ pð%ñ %ô %Ð˜Tð ð ñ ð (- uØ˜vÐ'rÐsð(ñ (ô (Ð˜uð ð ñ ð �uØ˜fÐ&qÐrðñ ô €K�ð ð ñ ð �5ØàðCð
ðñ ô €GˆSð ð ñ ð �5 ¨fÐ6kÐ-lÐmÑmÔm€GˆSÐmÐmÑmÐmÐmr   r   c                   ó   — e Zd ZdZdZdS )ÚSplitÚtrainÚdevN)r/   r0   r1   r=   r>   r:   r   r   r<   r<   g   s   € € € € € Ø€EØ
€C€C€Cr   r<   c                   ó¼   — e Zd ZU eed<   ee         ed<   eed<   eed<   dej	        dddfdede
d	edz  deez  ded
edz  defd„Zd„ Zdeeej        f         fd„ZdS )ÚSquadDatasetÚargsÚfeaturesÚmodeÚis_language_sensitiveNFÚptÚ	tokenizerÚlimit_lengthÚ	cache_dirÚdataset_formatc                 óX  — || _         || _        |j        rt          ¦   «         nt	          ¦   «         | _        t          |t          ¦  «        r,	 t          |         }n# t          $ r t          d¦  «        ‚w xY w|| _
        |j        rdnd}t          j                             |�|n|j        d|j        › d|j        j        › d|j        › d|› �¦  «        }	|	dz   }
t'          |
¦  «        5  t          j                             |	¦  «        rö|j        sït-          j        ¦   «         }t/          ¦   «          t1          j        |	d¬¦  «        | _        | j        d	         | _        | j                             d
d ¦  «        | _        | j                             dd ¦  «        | _        t>                                d|	› d�t-          j        ¦   «         |z
  ¦  «         | j        �| j        €t>           !                    d|	› d�¦  «         �n|t          j"        k    r%| j         #                    |j        ¦  «        | _        n$| j         $                    |j        ¦  «        | _        tK          | j        ||j        |j&        |j'        |t          j(        k    |j)        |¬¦  «        \  | _        | _        t-          j        ¦   «         }t1          j*        | j        | j        | j        dœ|	¦  «         t>                                d|	› dt-          j        ¦   «         |z
  d›d�¦  «         d d d ¦  «         d S # 1 swxY w Y   d S )Nzmode is not a valid split nameÚv2Úv1Úcached_Ú_z.lockT)Úweights_onlyrB   ÚdatasetÚexamplesz"Loading features from cached file z [took %.3f s]zDeleting cached file z; will allow dataset and examples to be cached in future run)rQ   rF   r!   r"   r$   Úis_trainingr.   Úreturn_dataset)rB   rP   rQ   z!Saving features into cached file z [took z.3fz s])+rA   rD   r(   r   r   Ú	processorÚ
isinstancer5   r<   ÚKeyErrorrC   ÚosÚpathr3   r   ÚvalueÚ	__class__r/   r!   r   Úexistsr'   Útimer   ÚtorchÚloadÚold_featuresrB   ÚgetrP   rQ   ÚloggerÚinfoÚwarningr>   Úget_dev_examplesÚget_train_examplesr   r"   r$   r=   r.   Úsave)ÚselfrA   rF   rG   rC   rD   rH   rI   Úversion_tagÚcached_features_fileÚ	lock_pathÚstarts               r   Ú__init__zSquadDataset.__init__r   sƒ  € ð ˆŒ	Ø%:ˆÔ"Ø/3Ô/KÐcÕ)Ñ+Ô+Ð+ÕQaÑQcÔQcˆŒÝ�d�CÑ Ô ð 	AðAÝ˜T”{��øÝð Að Að AÝÐ?Ñ@Ô@Ð@ðAøøøàˆŒ	à"Ô:ÐD�d�dÀˆÝ!œwŸ|š|Ø"Ð.ˆIˆI°D´MØe�d”jÐeÐe 9Ô#6Ô#?ÐeÐeÀ$ÔBUÐeÐeÐXcÐeÐeñ 
ô  
Ðð )¨7Ñ2ˆ	Ý�iÑ Ô ð -	ð -	ÝŒw�~Š~Ð2Ñ3Ô3ð ,¸DÔ<Pð ,Ýœ	™œ�Ý(Ñ*Ô*Ð*Ý$)¤JÐ/CÐRVÐ$WÑ$WÔ$W�Ô!ð !%Ô 1°*Ô =�”Ø#Ô0×4Ò4°YÀÑEÔE�”Ø $Ô 1× 5Ò 5°jÀ$Ñ GÔ G�”Ý—’Ø]Ð9MÐ]Ð]Ð]Õ_cÔ_hÑ_jÔ_jÐmrÑ_rñô ð ð ”<Ð'¨4¬=Ð+@Ý—N’Nð&Ð0Dð &ð &ð &ñô ð ùð
 �5œ9Ò$Ð$Ø$(¤N×$CÒ$CÀDÄMÑ$RÔ$R�D”M�Mà$(¤N×$EÒ$EÀdÄmÑ$TÔ$T�D”Må.PØ!œ]Ø'Ø#'Ô#6Ø#œØ%)Ô%:Ø $­¬Ò 3Ø œLØ#1ð	/ñ 	/ô 	/Ñ+�”˜tœ|õ œ	™œ�Ý”
Ø!%¤¸4¼<ÐUYÔUbÐcÐcØ(ñô ð õ
 —’ØqÐ8LÐqÐqÕUYÔU^ÑU`ÔU`ÐchÑUhÐqÐqÐqÐqñô ð ðW-	ð -	ð -	ñ -	ô -	ð -	ð -	ð -	ð -	ð -	ð -	ð -	øøøð -	ð -	ð -	ð -	ð -	ð -	s   ÁA ÁA5Ã'H+LÌL#Ì&L#c                 ó*   — t          | j        ¦  «        S r   )ÚlenrB   )rg   s    r   Ú__len__zSquadDataset.__len__¾   s   € Ý�4”=Ñ!Ô!Ð!r   Úreturnc                 ó&  — | j         |         }t          j        |j        t          j        ¬¦  «        }t          j        |j        t          j        ¬¦  «        }t          j        |j        t          j        ¬¦  «        }t          j        |j        t          j        ¬¦  «        }t          j        |j        t          j	        ¬¦  «        }t          j        |j
        t          j	        ¬¦  «        }|||dœ}	| j        j        dv r|	d= | j        j        dv r‰|	                     ||dœ¦  «         | j        j        r|	                     d|i¦  «         | j        rG|	                     dt          j        |j        t          j        ¬¦  «        | j        j        z  i¦  «         | j        t*          j        k    rbt          j        |j        t          j        ¬¦  «        }
t          j        |j        t          j        ¬¦  «        }|	                     |
|d	œ¦  «         |	S )
N)Údtype)Ú	input_idsÚattention_maskÚtoken_type_ids)ÚxlmÚrobertaÚ
distilbertÚ	camembertru   )Úxlnetrv   )Ú	cls_indexÚp_maskÚis_impossibleÚlangs)Ústart_positionsÚend_positions)rB   r]   Útensorrs   Úlongrt   ru   r{   r|   r9   r}   rA   r   Úupdater(   rD   ÚonesÚshapeÚint64r,   rC   r<   r=   Ústart_positionÚend_position)rg   ÚiÚfeaturers   rt   ru   r{   r|   r}   Úinputsr   r€   s               r   Ú__getitem__zSquadDataset.__getitem__Á   sÉ  € à”- Ô"ˆå”L Ô!2½%¼*ÐEÑEÔEˆ	Ýœ gÔ&<ÅEÄJÐOÑOÔOˆÝœ gÔ&<ÅEÄJÐOÑOÔOˆÝ”L Ô!2½%¼*ÐEÑEÔEˆ	Ý”˜gœnµE´KÐ@Ñ@Ô@ˆÝœ WÔ%:Å%Ä+ÐNÑNÔNˆð #Ø,Ø,ð
ð 
ˆð Œ9ÔÐ#PÐPÐPØÐ'Ð(àŒ9ÔÐ#3Ð3Ð3Ø�MŠM¨	¸VÐDÐDÑEÔEÐEØŒyÔ0ð @Ø—’˜°Ð>Ñ?Ô?Ð?ØÔ)ð oØ—’˜w­¬°I´OÍ5Ì;Ð)WÑ)WÔ)WÐZ^ÔZcÔZkÑ)kÐmÑnÔnÐnàŒ9�œÒ#Ð#Ý#œl¨7Ô+AÍÌÐTÑTÔTˆOÝ!œL¨Ô)=ÅUÄZÐPÑPÔPˆMØ�MŠM¨oÐP]Ð^Ð^Ñ_Ô_Ð_àˆr   )r/   r0   r1   r   r6   Úlistr   r<   r8   r=   r
   r7   r5   rl   ro   Údictr]   ÚTensorrŒ   r:   r   r   r@   r@   l   s  € € € € € € Ø
$Ð$Ð$Ñ$Ø�=Ô!Ð!Ð!Ñ!Ø
€K€K�KØÐÐÑð $(Ø!œKØ&+Ø $Ø"ðJð Jà(ðJð 'ðJð ˜D‘jð	Jð
 �E‰kðJð  $ðJð ˜‘:ðJð ðJð Jð Jð JðX"ð "ð "ð   S¨%¬,Ð%6Ô 7ð  ð  ð  ð  ð  ð  r   r@   )#rW   r\   Údataclassesr   r   Úenumr   r]   Úfilelockr   Útorch.utils.datar   Úmodels.auto.modeling_autor	   Útokenization_pythonr
   Úutilsr   r   Úprocessors.squadr   r   r   r   Ú
get_loggerr/   ra   r�   ÚkeysÚMODEL_CONFIG_CLASSESÚtupler4   r   r<   r@   r:   r   r   ú<module>rœ      sÅ  ðð 
€	€	€	Ø €€€Ø (Ð (Ð (Ð (Ð (Ð (Ð (Ð (Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð Ø $Ð $Ð $Ð $Ð $Ð $à MÐ MÐ MÐ MÐ MÐ MØ 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø tÐ tÐ tÐ tÐ tÐ tÐ tÐ tÐ tÐ tÐ tÐ tð 
ˆÔ	˜HÑ	%Ô	%€à�tÐEÐ@ÔEÑGÔGÑHÔHÐ ØˆeÐEÐEÐ0DÐEÑEÔEÑEÔE€ð ð?nð ?nð ?nð ?nð ?nñ ?nô ?nñ „ð?nðDð ð ð ð ˆDñ ô ð ð
uð uð uð uð u�7ñ uô uð uð uð ur   