§
    ‚Štjõs  ã                   óÜ   — d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	  e	j
        e¦  «        Zd„ Zd„ Zd„ Zd	„ Zd
„ Zd„ Zdd„Zd„ Zd„ Zd„ Zd„ Zd„ Zdd„Zdd„Zd„ Zd„ Zd„ Zd„ ZdS )a·  
Very heavily inspired by the official evaluation script for SQuAD version 2.0 which was modified by XLNet authors to
update `find_best_threshold` scripts for SQuAD V2.0

In addition to basic functionality, we also compute additional statistics and plot precision-recall curves if an
additional na_prob.json file is provided. This file is expected to map question ID's to the model's predicted
probability that a question is unanswerable.
é    Né   )ÚBasicTokenizer)Úloggingc           	      óf   — d„ }d„ }d„ }d„ } | | | || ¦  «        ¦  «        ¦  «        ¦  «        S )zALower text and remove punctuation, articles and extra whitespace.c                 ól   — t          j        dt           j        ¦  «        }t          j        |d| ¦  «        S )Nz\b(a|an|the)\bú )ÚreÚcompileÚUNICODEÚsub)ÚtextÚregexs     úe/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/data/metrics/squad_metrics.pyÚremove_articlesz)normalize_answer.<locals>.remove_articles'   s*   € Ý”
Ð,­b¬jÑ9Ô9ˆÝŒv�e˜S $Ñ'Ô'Ð'ó    c                 óP   — d                      |                      ¦   «         ¦  «        S )Nr   )ÚjoinÚsplit©r   s    r   Úwhite_space_fixz)normalize_answer.<locals>.white_space_fix+   s   € Ø�xŠx˜Ÿ
š
™œÑ%Ô%Ð%r   c                 óx   ‡— t          t          j        ¦  «        Šd                     ˆfd„| D ¦   «         ¦  «        S )NÚ c              3   ó$   •K  — | ]
}|‰v¯|V — Œd S ©N© )Ú.0ÚchÚexcludes     €r   ú	<genexpr>z8normalize_answer.<locals>.remove_punc.<locals>.<genexpr>0   s-   øè è € Ð>Ð>˜b¨B°gÐ,=Ð,=�rÐ,=Ð,=Ð,=Ð,=Ð>Ð>r   )ÚsetÚstringÚpunctuationr   )r   r   s    @r   Úremove_puncz%normalize_answer.<locals>.remove_punc.   s:   ø€ Ý•fÔ(Ñ)Ô)ˆØ�wŠwÐ>Ð>Ð>Ð> DÐ>Ñ>Ô>Ñ>Ô>Ð>r   c                 ó*   — |                       ¦   «         S r   )Úlowerr   s    r   r%   znormalize_answer.<locals>.lower2   s   € Ø�zŠz‰|Œ|Ðr   r   )Úsr   r   r#   r%   s        r   Únormalize_answerr'   $   sl   € ð(ð (ð (ð&ð &ð &ð?ð ?ð ?ðð ð ð ˆ?˜?˜?¨;¨;°u°u¸Q±x´xÑ+@Ô+@ÑAÔAÑBÔBÐBr   c                 óL   — | sg S t          | ¦  «                             ¦   «         S r   )r'   r   )r&   s    r   Ú
get_tokensr)   8   s)   € Øð Øˆ	Ý˜AÑÔ×$Ò$Ñ&Ô&Ð&r   c                 ó\   — t          t          | ¦  «        t          |¦  «        k    ¦  «        S r   )Úintr'   )Úa_goldÚa_preds     r   Úcompute_exactr.   >   s'   € ÝÕ Ñ'Ô'Õ+;¸FÑ+CÔ+CÒCÑDÔDÐDr   c                 óÈ  — t          | ¦  «        }t          |¦  «        }t          j        |¦  «        t          j        |¦  «        z  }t          |                     ¦   «         ¦  «        }t          |¦  «        dk    st          |¦  «        dk    rt          ||k    ¦  «        S |dk    rdS d|z  t          |¦  «        z  }d|z  t          |¦  «        z  }d|z  |z  ||z   z  }|S )Nr   ç      ð?é   )r)   ÚcollectionsÚCounterÚsumÚvaluesÚlenr+   )	r,   r-   Ú	gold_toksÚ	pred_toksÚcommonÚnum_sameÚ	precisionÚrecallÚf1s	            r   Ú
compute_f1r>   B   sØ   € Ý˜6Ñ"Ô"€IÝ˜6Ñ"Ô"€IÝÔ  Ñ+Ô+­kÔ.AÀ)Ñ.LÔ.LÑL€FÝ�6—=’=‘?”?Ñ#Ô#€HÝ
ˆ9�~„~˜ÒÐ�c )™nœn°Ò1Ð1å�9 	Ò)Ñ*Ô*Ð*Ø�1‚}€}ØˆqØ�h‘¥ Y¡¤Ñ/€IØ�8‰^�c )™nœnÑ,€FØ
ˆi‰-˜&Ñ
  Y°Ñ%7Ñ	8€BØ€Ir   c                 ó  ‡— i }i }| D ]z}|j         }d„ |j        D ¦   «         }|sdg}||vrt          d|› �¦  «         Œ6||         Št          ˆfd„|D ¦   «         ¦  «        ||<   t          ˆfd„|D ¦   «         ¦  «        ||<   Œ{||fS )zV
    Computes the exact and f1 scores from the examples and the model predictions
    c                 óH   — g | ]}t          |d          ¦  «        ¯|d          ‘Œ S r   )r'   )r   Úanswers     r   ú
<listcomp>z"get_raw_scores.<locals>.<listcomp>[   s1   € ÐiÐiÐi¨6ÕHXÐY_Ð`fÔYgÑHhÔHhÐi˜˜vœÐiÐiÐir   r   zMissing prediction for c              3   ó8   •K  — | ]}t          |‰¦  «        V — Œd S r   )r.   ©r   ÚaÚ
predictions     €r   r   z!get_raw_scores.<locals>.<genexpr>f   s-   øè è € Ð"VÐ"VÀA¥=°°JÑ#?Ô#?Ð"VÐ"VÐ"VÐ"VÐ"VÐ"Vr   c              3   ó8   •K  — | ]}t          |‰¦  «        V — Œd S r   )r>   rD   s     €r   r   z!get_raw_scores.<locals>.<genexpr>g   s-   øè è € ÐPÐP¸a¥
¨1¨jÑ 9Ô 9ÐPÐPÐPÐPÐPÐPr   )Úqas_idÚanswersÚprintÚmax)ÚexamplesÚpredsÚexact_scoresÚ	f1_scoresÚexamplerH   Úgold_answersrF   s          @r   Úget_raw_scoresrR   R   sÕ   ø€ ð €LØ€Iàð Qð QˆØ”ˆØiÐi°W´_ÐiÑiÔiˆàð 	 à˜4ˆLà˜ÐÐÝÐ4¨FÐ4Ð4Ñ5Ô5Ð5Øà˜6”]ˆ
Ý"Ð"VÐ"VÐ"VÐ"VÈÐ"VÑ"VÔ"VÑVÔVˆ�VÑÝÐPÐPÐPÐPÀ<ÐPÑPÔPÑPÔPˆ	�&ÑÐà˜Ð"Ð"r   c                 ó˜   — i }|                       ¦   «         D ]2\  }}||         |k    }|rt          ||          ¦  «        ||<   Œ-|||<   Œ3|S r   )ÚitemsÚfloat)ÚscoresÚna_probsÚqid_to_has_ansÚna_prob_threshÚ
new_scoresÚqidr&   Úpred_nas           r   Úapply_no_ans_thresholdr]   l   sd   € Ø€JØ—,’,‘.”.ð  ð  ‰ˆˆQØ˜3”- .Ò0ˆØð 	 Ý#¨°sÔ(;Ð$;Ñ<Ô<ˆJ�s‰OˆOàˆJ�s‰OˆOØÐr   c           	      óÊ  ‡ ‡— |svt          ‰ ¦  «        }t          j        ddt          ‰                      ¦   «         ¦  «        z  |z  fddt          ‰                     ¦   «         ¦  «        z  |z  fd|fg¦  «        S t          |¦  «        }t          j        ddt          ˆ fd„|D ¦   «         ¦  «        z  |z  fddt          ˆfd„|D ¦   «         ¦  «        z  |z  fd|fg¦  «        S )NÚexactç      Y@r=   Útotalc              3   ó(   •K  — | ]}‰|         V — Œd S r   r   )r   ÚkrN   s     €r   r   z!make_eval_dict.<locals>.<genexpr>…   s'   øè è € Ð%HÐ%H¸! l°1¤oÐ%HÐ%HÐ%HÐ%HÐ%HÐ%Hr   c              3   ó(   •K  — | ]}‰|         V — Œd S r   r   )r   rc   rO   s     €r   r   z!make_eval_dict.<locals>.<genexpr>†   s'   øè è € Ð"BÐ"B°A 9¨Q¤<Ð"BÐ"BÐ"BÐ"BÐ"BÐ"Br   )r6   r2   ÚOrderedDictr4   r5   )rN   rO   Úqid_listra   s   ``  r   Úmake_eval_dictrg   w   s  øø€ Øð 
Ý�LÑ!Ô!ˆÝÔ&à˜%¥# l×&9Ò&9Ñ&;Ô&;Ñ"<Ô"<Ñ<¸uÑDÐEØ�u�s 9×#3Ò#3Ñ#5Ô#5Ñ6Ô6Ñ6¸Ñ>Ð?Ø˜%Ð ðñ
ô 
ð 	
õ �H‘”ˆÝÔ&à˜%¥#Ð%HÐ%HÐ%HÐ%H¸xÐ%HÑ%HÔ%HÑ"HÔ"HÑHÈ5ÑPÐQØ�u�sÐ"BÐ"BÐ"BÐ"B¸Ð"BÑ"BÔ"BÑBÔBÑBÀUÑJÐKØ˜%Ð ðñ
ô 
ð 	
r   c                 ó0   — |D ]}||         | |› d|› �<   Œd S )NÚ_r   )Ú	main_evalÚnew_evalÚprefixrc   s       r   Ú
merge_evalrm   Œ   s6   € Øð 1ð 1ˆØ%-¨a¤[ˆ	�V�/�/˜a�/�/Ñ"Ð"ð1ð 1r   c                 óv  ‡‡— t          ˆfd„‰D ¦   «         ¦  «        }|}|}d}t          ‰ˆfd„¬¦  «        }|D ]:}	|	|vrŒ‰|	         r	||	         }
n| |	         rd}
nd}
||
z  }||k    r
|}‰|	         }Œ;d\  }}|D ] }	‰|	         sŒ|dz  }|	|vrŒ|||	         z  }Œ!d	|z  t          |¦  «        z  |d
|z  |z  fS )Nc              3   ó,   •K  — | ]}‰|         °
d V — ŒdS ©é   Nr   ©r   rc   rX   s     €r   r   z&find_best_thresh_v2.<locals>.<genexpr>’   ó-   øè è € ÐHÐH˜1°nÀQÔ6GÐH�QÐHÐHÐHÐHÐHÐHr   ç        c                 ó   •— ‰|          S r   r   ©rc   rW   s    €r   ú<lambda>z%find_best_thresh_v2.<locals>.<lambda>–   ó   ø€ ¨h°q¬k€ r   ©Úkeyéÿÿÿÿr   )r   r   rq   r`   r0   )r4   Úsortedr6   )rM   rV   rW   rX   Ú
num_no_ansÚ	cur_scoreÚ
best_scoreÚbest_threshrf   r[   ÚdiffÚhas_ans_scoreÚhas_ans_cnts     ``         r   Úfind_best_thresh_v2r„   ‘   s;  øø€ ÝÐHÐHÐHÐH ÐHÑHÔHÑHÔH€JØ€IØ€JØ€KÝ�hÐ$9Ð$9Ð$9Ð$9Ð:Ñ:Ô:€HØð (ð (ˆØ�fÐÐØØ˜#Ôð 	Ø˜#”;ˆDˆDà�SŒzð Ø��à�Ø�TÑˆ	Ø�zÒ!Ð!Ø"ˆJØ" 3œ-ˆKøà!%Ñ€M�;Øð %ð %ˆØ˜cÔ"ð 	ØØ�qÑˆà�fÐÐØØ˜ œÑ$ˆˆà�:Ñ¥ F¡¤Ñ+¨[¸#ÀÑ:MÐP[Ñ:[Ð[Ð[r   c                 óš   — t          ||||¦  «        \  }}}t          ||||¦  «        \  }	}
}|| d<   || d<   |	| d<   |
| d<   || d<   || d<   d S )NÚ
best_exactÚbest_exact_threshÚbest_f1Úbest_f1_threshÚhas_ans_exactÚ
has_ans_f1)r„   )rj   rM   Ú	exact_rawÚf1_rawrW   rX   r†   Úexact_threshrŠ   rˆ   Ú	f1_threshr‹   s               r   Úfind_all_best_thresh_v2r�   ³   s}   € Ý.AÀ%ÈÐT\Ð^lÑ.mÔ.mÑ+€J�˜mÝ%8¸ÀÈÐR`Ñ%aÔ%aÑ"€GˆY˜
Ø(€IˆlÑØ%1€IÐ!Ñ"Ø"€IˆiÑØ"+€IÐÑØ!.€IˆoÑØ(€IˆlÑÐÐr   c                 ó8  ‡‡— t          ˆfd„‰D ¦   «         ¦  «        }|}|}d}t          ‰ˆfd„¬¦  «        }t          |¦  «        D ]=\  }	}
|
|vrŒ
‰|
         r	||
         }n| |
         rd}nd}||z  }||k    r
|}‰|
         }Œ>d|z  t          |¦  «        z  |fS )Nc              3   ó,   •K  — | ]}‰|         °
d V — ŒdS rp   r   rr   s     €r   r   z#find_best_thresh.<locals>.<genexpr>¿   rs   r   rt   c                 ó   •— ‰|          S r   r   rv   s    €r   rw   z"find_best_thresh.<locals>.<lambda>Ã   rx   r   ry   r{   r   r`   )r4   r|   Ú	enumerater6   )rM   rV   rW   rX   r}   r~   r   r€   rf   ri   r[   r�   s     ``        r   Úfind_best_threshr•   ¾   sé   øø€ ÝÐHÐHÐHÐH ÐHÑHÔHÑHÔH€JØ€IØ€JØ€KÝ�hÐ$9Ð$9Ð$9Ð$9Ð:Ñ:Ô:€HÝ˜HÑ%Ô%ð (ð (‰ˆˆ3Ø�fÐÐØØ˜#Ôð 	Ø˜#”;ˆDˆDà�SŒzð Ø��à�Ø�TÑˆ	Ø�zÒ!Ð!Ø"ˆJØ" 3œ-ˆKøØ�:Ñ¥ F¡¤Ñ+¨[Ð8Ð8r   c                 ó‚   — t          ||||¦  «        \  }}t          ||||¦  «        \  }}	|| d<   || d<   || d<   |	| d<   d S )Nr†   r‡   rˆ   r‰   )r•   )
rj   rM   rŒ   r�   rW   rX   r†   rŽ   rˆ   r�   s
             r   Úfind_all_best_threshr—   Õ   sb   € Ý/°°yÀ(ÈNÑ[Ô[Ñ€J�Ý)¨%°¸À>ÑRÔRÑ€GˆYà(€IˆlÑØ%1€IÐ!Ñ"Ø"€IˆiÑØ"+€IÐÑÐÐr   r0   c                 ó  — d„ | D ¦   «         }d„ |                      ¦   «         D ¦   «         }d„ |                      ¦   «         D ¦   «         }|€t                               |d¦  «        }t          | |¦  «        \  }}t	          ||||¦  «        }	t	          ||||¦  «        }
t          |	|
¦  «        }|r#t          |	|
|¬¦  «        }t          ||d¦  «         |r#t          |	|
|¬¦  «        }t          ||d¦  «         |rt          ||||||¦  «         |S )Nc                 óB   — i | ]}|j         t          |j        ¦  «        “ŒS r   )rH   ÚboolrI   )r   rP   s     r   ú
<dictcomp>z"squad_evaluate.<locals>.<dictcomp>à   s&   € ÐZÐZÐZÀg˜GœN­D°´Ñ,AÔ,AÐZÐZÐZr   c                 ó   — g | ]	\  }}|¯|‘Œ
S r   r   ©r   rH   Ú
has_answers      r   rB   z"squad_evaluate.<locals>.<listcomp>á   s#   € ÐcÐcÐcÑ"4 &¨*ÐXbÐc�vÐcÐcÐcr   c                 ó   — g | ]	\  }}|°|‘Œ
S r   r   r�   s      r   rB   z"squad_evaluate.<locals>.<listcomp>â   s#   € ÐfÐfÐfÑ!3 ¨Ð[eÐf�fÐfÐfÐfr   rt   )rf   ÚHasAnsÚNoAns)rT   ÚdictÚfromkeysrR   r]   rg   rm   r—   )rL   rM   Úno_answer_probsÚno_answer_probability_thresholdÚqas_id_to_has_answerÚhas_answer_qidsÚno_answer_qidsr_   r=   Úexact_thresholdÚf1_thresholdÚ
evaluationÚhas_ans_evalÚno_ans_evals                 r   Úsquad_evaluater®   ß   sG  € ØZÐZÐQYÐZÑZÔZÐØcÐcÐ8L×8RÒ8RÑ8TÔ8TÐcÑcÔc€OØfÐfÐ7K×7QÒ7QÑ7SÔ7SÐfÑfÔf€NàÐÝŸ-š-¨¨sÑ3Ô3ˆå˜x¨Ñ/Ô/�I€Eˆ2å,ØˆÐ 4Ð6Uñô €Oõ *¨"¨oÐ?SÐUtÑuÔu€Lå °Ñ>Ô>€Jàð 7Ý% o°|ÈoÐ^Ñ^Ô^ˆÝ�:˜|¨XÑ6Ô6Ð6àð 5Ý$ _°lÈ^Ð\Ñ\Ô\ˆÝ�:˜{¨GÑ4Ô4Ð4àð bÝ˜Z¨°°r¸?ÐL`ÑaÔaÐaàÐr   Fc                 ó,  — d„ }t          |¬¦  «        }d                     |                     |¦  «        ¦  «        }|                     | ¦  «        }|dk    r%|r!t                               d| › d|› d�¦  «         |S |t          | ¦  «        z   dz
  } ||¦  «        \  }	}
 ||¦  «        \  }}t          |	¦  «        t          |¦  «        k    r%|r!t                               d	|	› d
|› d�¦  «         |S i }|                     ¦   «         D ]
\  }}|||<   Œd}||v r||         }||
v r|
|         }|€|rt                               d¦  «         |S d}||v r||         }||
v r|
|         }|€|rt                               d¦  «         |S |||dz   …         }|S )z;Project the tokenized prediction back to the original text.c                 óä   — g }t          j        ¦   «         }t          | ¦  «        D ]3\  }}|dk    rŒ||t          |¦  «        <   |                     |¦  «         Œ4d                     |¦  «        }||fS )Nr   r   )r2   re   r”   r6   Úappendr   )r   Úns_charsÚns_to_s_mapÚiÚcÚns_texts         r   Ú_strip_spacesz%get_final_text.<locals>._strip_spaces  s{   € ØˆÝ!Ô-Ñ/Ô/ˆÝ˜d‘O”Oð 	ð 	‰DˆAˆqØ�CŠxˆxØØ)*ˆK�˜H™œÑ&Ø�OŠO˜AÑÔÐÐØ—'’'˜(Ñ#Ô#ˆØ˜Ð%Ð%r   )Údo_lower_caser   r{   zUnable to find text: 'z' in 'ú'rq   z*Length not equal after stripping spaces: 'z' vs 'NzCouldn't map start positionzCouldn't map end position)r   r   ÚtokenizeÚfindÚloggerÚinfor6   rT   )Ú	pred_textÚ	orig_textr¸   Úverbose_loggingr·   Ú	tokenizerÚtok_textÚstart_positionÚend_positionÚorig_ns_textÚorig_ns_to_s_mapÚtok_ns_textÚtok_ns_to_s_mapÚtok_s_to_ns_mapr´   Ú	tok_indexÚorig_start_positionÚns_start_positionÚorig_end_positionÚns_end_positionÚoutput_texts                        r   Úget_final_textrÐ   þ   s%  € ð8	&ð 	&ð 	&õ ¨]Ð;Ñ;Ô;€Ià�xŠx˜	×*Ò*¨9Ñ5Ô5Ñ6Ô6€Hà—]’] 9Ñ-Ô-€NØ˜ÒÐØð 	PÝ�KŠKÐN°ÐNÐNÀ)ÐNÐNÐNÑOÔOÐOØÐØ!¥C¨	¡N¤NÑ2°QÑ6€Là'4 }°YÑ'?Ô'?Ñ$€\Ð#Ø%2 ]°8Ñ%<Ô%<Ñ"€[�/å
ˆ<ÑÔ�C Ñ,Ô,Ò,Ð,Øð 	iÝ�KŠKÐgÀ\ÐgÐgÐYdÐgÐgÐgÑhÔhÐhØÐð €OØ'×-Ò-Ñ/Ô/ð 'ð '‰ˆˆ9Ø%&ˆ˜	Ñ"Ð"àÐØ˜Ð(Ð(Ø+¨NÔ;ÐØÐ 0Ð0Ð0Ø"2Ð3DÔ"EÐàÐ"Øð 	7Ý�KŠKÐ5Ñ6Ô6Ð6ØÐàÐØ�Ð&Ð&Ø)¨,Ô7ˆØÐ.Ð.Ð.Ø 0°Ô AÐàÐ Øð 	5Ý�KŠKÐ3Ñ4Ô4Ð4ØÐàÐ/Ð3DÀqÑ3HÐIÔJ€KØÐr   c                 óÚ   — t          t          | ¦  «        d„ d¬¦  «        }g }t          t          |¦  «        ¦  «        D ]+}||k    r n"|                     ||         d         ¦  «         Œ,|S )z"Get the n-best logits from a list.c                 ó   — | d         S )Nrq   r   ©Úxs    r   rw   z#_get_best_indexes.<locals>.<lambda>^  s
   € ¸aÀ¼d€ r   T©rz   Úreverser   )r|   r”   Úranger6   r±   )ÚlogitsÚn_best_sizeÚindex_and_scoreÚbest_indexesr´   s        r   Ú_get_best_indexesrÜ   \  s{   € å�Y vÑ.Ô.°N°NÈDÐQÑQÔQ€Oà€LÝ•3�Ñ'Ô'Ñ(Ô(ð 3ð 3ˆØ�ÒÐØˆEØ×Ò˜O¨AÔ.¨qÔ1Ñ2Ô2Ð2Ð2ØÐr   c                 óâ   — | sg S d}| D ]}|�||k    r|}Œg }d}| D ]3}t          j        ||z
  ¦  «        }|                     |¦  «         ||z  }Œ4g }|D ]}|                     ||z  ¦  «         Œ|S )z,Compute softmax probability over raw logits.Nrt   )ÚmathÚexpr±   )rV   Ú	max_scoreÚscoreÚ
exp_scoresÚ	total_sumrÔ   Úprobss          r   Ú_compute_softmaxrå   h  s¸   € àð Øˆ	à€IØð ð ˆØÐ ¨	Ò 1Ð 1ØˆIøà€JØ€IØð ð ˆÝŒH�U˜YÑ&Ñ'Ô'ˆØ×Ò˜!ÑÔÐØ�Q‰ˆ	ˆ	à€EØð (ð (ˆØ�Š�U˜YÑ&Ñ'Ô'Ð'Ð'Ø€Lr   c                 óª  — |rt                                d|› �¦  «         |rt                                d|› �¦  «         |r|
rt                                d|› �¦  «         t          j        t          ¦  «        }|D ]"}||j                                      |¦  «         Œ#i }|D ]}|||j        <   Œt          j        dg d¢¦  «        }t          j	        ¦   «         }t          j	        ¦   «         }t          j	        ¦   «         }t          | ¦  «        D �]¿\  }}||         }g }d}d}d}d}t          |¦  «        D �]9\  }}||j                 }t          |j        |¦  «        }t          |j        |¦  «        }|
r?|j        d         |j        d         z   } | |k     r| }|}|j        d         }|j        d         }|D ]¸}!|D ]³}"|!t          |j        ¦  «        k    rŒ|"t          |j        ¦  «        k    rŒ4|!|j        vrŒ>|"|j        vrŒH|j                             |!d¦  «        sŒd|"|!k     rŒk|"|!z
  d	z   }#|#|k    rŒz|                      |||!|"|j        |!         |j        |"         ¬¦  «        ¦  «         Œ´Œ¹�Œ;|
r#|                      ||dd||¬¦  «        ¦  «         t'          |d
„ d¬¦  «        }t          j        dg d¢¦  «        }$i }%g }&|D �]7}'t          |&¦  «        |k    r �n ||'j                 }|'j        dk    rÔ|j        |'j        |'j        d	z   …         }(|j        |'j                 })|j        |'j                 }*|j        |)|*d	z   …         }+|                     |(¦  «        },|,                     ¦   «         },d                     |,                     ¦   «         ¦  «        },d                     |+¦  «        }-t9          |,|-||	¦  «        }.|.|%v rŒÿd|%|.<   nd}.d|%|.<   |&                      |$|.|'j        |'j        ¬¦  «        ¦  «         �Œ9|
rZd|%vr!|&                      |$d||¬¦  «        ¦  «         t          |&¦  «        d	k    r"|&                     d |$ddd¬¦  «        ¦  «         |&s!|&                      |$ddd¬¦  «        ¦  «         t          |&¦  «        d	k     rtA          d¦  «        ‚g }/d}0|&D ]/}1|/                     |1j        |1j        z   ¦  «         |0s	|1j!        r|1}0Œ0tE          |/¦  «        }2g }3t          |&¦  «        D ]V\  }4}1t          j	        ¦   «         }5|1j!        |5d<   |2|4         |5d<   |1j        |5d<   |1j        |5d<   |3                     |5¦  «         ŒWt          |3¦  «        d	k     rtA          d¦  «        ‚|
s|3d         d         ||j#        <   n<||0j        z
  |0j        z
  }6|6||j#        <   |6|k    rd||j#        <   n|0j!        ||j#        <   |3||j#        <   �ŒÁ|rTtI          |d¦  «        5 }7|7 %                    tM          j'        |d¬¦  «        dz   ¦  «         ddd¦  «         n# 1 swxY w Y   |rTtI          |d¦  «        5 }7|7 %                    tM          j'        |d¬¦  «        dz   ¦  «         ddd¦  «         n# 1 swxY w Y   |rV|
rTtI          |d¦  «        5 }7|7 %                    tM          j'        |d¬¦  «        dz   ¦  «         ddd¦  «         n# 1 swxY w Y   |S )zHWrite final predictions to the json file and log-odds of null if needed.úWriting predictions to: zWriting nbest to: zWriting null_log_odds to: ÚPrelimPrediction)Úfeature_indexÚstart_indexÚ	end_indexÚstart_logitÚ	end_logité@B r   Frq   c                 ó    — | j         | j        z   S r   )rì   rí   rÓ   s    r   rw   z,compute_predictions_logits.<locals>.<lambda>â  s   € ÀqÄ}ÐWXÔWbÑGb€ r   TrÕ   ÚNbestPrediction)r   rì   rí   r   r   Úemptyrt   úNo valid predictionsNr   Úprobabilityrì   rí   Úwé   ©Úindentú
)(r¼   r½   r2   ÚdefaultdictÚlistÚexample_indexr±   Ú	unique_idÚ
namedtuplere   r”   rÜ   Ústart_logitsÚ
end_logitsr6   ÚtokensÚtoken_to_orig_mapÚtoken_is_max_contextÚgetr|   ré   rê   rë   Ú
doc_tokensÚconvert_tokens_to_stringÚstripr   r   rÐ   rì   rí   ÚinsertÚ
ValueErrorr   rå   rH   ÚopenÚwriteÚjsonÚdumps)8Úall_examplesÚall_featuresÚall_resultsrÙ   Úmax_answer_lengthr¸   Úoutput_prediction_fileÚoutput_nbest_fileÚoutput_null_log_odds_filerÀ   Úversion_2_with_negativeÚnull_score_diff_thresholdrÁ   Úexample_index_to_featuresÚfeatureÚunique_id_to_resultÚresultÚ_PrelimPredictionÚall_predictionsÚall_nbest_jsonÚscores_diff_jsonrû   rP   ÚfeaturesÚprelim_predictionsÚ
score_nullÚmin_null_feature_indexÚnull_start_logitÚnull_end_logitré   Ústart_indexesÚend_indexesÚfeature_null_scorerê   rë   ÚlengthÚ_NbestPredictionÚseen_predictionsÚnbestÚpredÚ
tok_tokensÚorig_doc_startÚorig_doc_endÚorig_tokensrÂ   r¿   Ú
final_textÚtotal_scoresÚbest_non_null_entryÚentryrä   Ú
nbest_jsonr´   ÚoutputÚ
score_diffÚwriters8                                                           r   Úcompute_predictions_logitsr8    sD	  € ð  ð IÝ�ŠÐGÐ/EÐGÐGÑHÔHÐHØð >Ý�ŠÐ<Ð):Ð<Ð<Ñ=Ô=Ð=Ø ð NÐ%<ð NÝ�ŠÐLÐ1JÐLÐLÑMÔMÐMå +Ô 7½Ñ =Ô =ÐØð Ið IˆØ! 'Ô"7Ô8×?Ò?ÀÑHÔHÐHÐHàÐØð 7ð 7ˆØ06Ð˜FÔ,Ñ-Ð-å#Ô.ØÐeÐeÐeñô Ðõ "Ô-Ñ/Ô/€OÝ Ô,Ñ.Ô.€NÝ"Ô.Ñ0Ô0Ðå"+¨LÑ"9Ô"9ð W4ñ W4Ñˆ�wØ,¨]Ô;ˆàÐàˆ
Ø!"ÐØÐØˆÝ&/°Ñ&9Ô&9ð (	ñ (	Ñ"ˆM˜7Ø(¨Ô):Ô;ˆFÝ-¨fÔ.AÀ;ÑOÔOˆMÝ+¨FÔ,=¸{ÑKÔKˆKà&ð :Ø%+Ô%8¸Ô%;¸fÔ>OÐPQÔ>RÑ%RÐ"Ø%¨
Ò2Ð2Ø!3�JØ-:Ð*Ø'-Ô':¸1Ô'=Ð$Ø%+Ô%6°qÔ%9�NØ,ð ð �Ø!,ð ð �Ið #¥c¨'¬.Ñ&9Ô&9Ò9Ð9Ø Ø ¥C¨¬Ñ$7Ô$7Ò7Ð7Ø Ø"¨'Ô*CÐCÐCØ Ø ¨Ô(AÐAÐAØ Ø"Ô7×;Ò;¸KÈÑOÔOð !Ø Ø  ;Ò.Ð.Ø Ø&¨Ñ4°qÑ8�FØÐ 1Ò1Ð1Ø Ø&×-Ò-Ø)Ð)Ø*7Ø(3Ø&/Ø(.Ô(;¸KÔ(HØ&,Ô&7¸	Ô&Bðñ ô ñô ð ð ð'ñð: #ð 		Ø×%Ò%Ø!Ð!Ø"8Ø !ØØ 0Ø,ðñ ô ñô ð õ $Ð$6Ð<cÐ<cÐmqÐrÑrÔrÐå&Ô1ØÐCÐCÐCñ
ô 
Ðð ÐØˆØ&ð  	tñ  	tˆDÝ�5‰zŒz˜[Ò(Ð(Ø‘Ø˜tÔ1Ô2ˆGØÔ !Ò#Ð#Ø$œ^¨DÔ,<ÀÄÐQRÑ@RÐ,SÔT�
Ø!(Ô!:¸4Ô;KÔ!L�Ø&Ô8¸¼ÔH�Ø%Ô0°À<ÐRSÑCSÐ1TÔU�à$×=Ò=¸jÑIÔI�ð $Ÿ>š>Ñ+Ô+�ØŸ8š8 H§N¢NÑ$4Ô$4Ñ5Ô5�ØŸHšH [Ñ1Ô1�	å+¨H°iÀÐP_Ñ`Ô`�
ØÐ!1Ð1Ð1Øà/3Ð  Ñ,Ð,à�
Ø/3Ð  Ñ,à�LŠLÐ)Ð)¨zÀtÔGWÐcgÔcqÐrÑrÔrÑsÔsÐsÑsà"ð 	`ØÐ)Ð)Ð)Ø—’Ð-Ð-°2ÐCSÐ_mÐnÑnÔnÑoÔoÐoõ �5‰zŒz˜QŠˆØ—’˜QÐ 0Ð 0°gÈ3ÐZ]Ð ^Ñ ^Ô ^Ñ_Ô_Ð_ð ð 	YØ�LŠLÐ)Ð)¨wÀCÐSVÐWÑWÔWÑXÔXÐXåˆu‰:Œ:˜Š>ˆ>ÝÐ3Ñ4Ô4Ð4àˆØ"ÐØð 	0ð 	0ˆEØ×Ò Ô 1°E´OÑ CÑDÔDÐDØ&ð 0Ø”:ð 0Ø*/Ð'øå  Ñ.Ô.ˆàˆ
Ý! %Ñ(Ô(ð 	&ð 	&‰HˆAˆuÝ Ô,Ñ.Ô.ˆFØ"œZˆF�6‰NØ$)¨!¤HˆF�=Ñ!Ø$)Ô$5ˆF�=Ñ!Ø"'¤/ˆF�;ÑØ×Ò˜fÑ%Ô%Ð%Ð%åˆz‰?Œ?˜QÒÐÝÐ3Ñ4Ô4Ð4à&ð 		KØ.8¸¬m¸FÔ.CˆO˜GœNÑ+Ð+ð $Ð&9Ô&EÑEÐI\ÔIfÑgˆJØ/9Ð˜Wœ^Ñ,ØÐ5Ò5Ð5Ø24� ¤Ñ/Ð/à2EÔ2J� ¤Ñ/Ø)3ˆ�w”~Ñ&Ñ&àð GÝÐ(¨#Ñ.Ô.ð 	G°&Ø�LŠL�œ O¸AÐ>Ñ>Ô>ÀÑEÑFÔFÐFð	Gð 	Gð 	Gñ 	Gô 	Gð 	Gð 	Gð 	Gð 	Gð 	Gð 	Gøøøð 	Gð 	Gð 	Gð 	Gð ð FÝÐ# SÑ)Ô)ð 	F¨VØ�LŠL�œ N¸1Ð=Ñ=Ô=ÀÑDÑEÔEÐEð	Fð 	Fð 	Fñ 	Fô 	Fð 	Fð 	Fð 	Fð 	Fð 	Fð 	Føøøð 	Fð 	Fð 	Fð 	Fð !ð HÐ%<ð HÝÐ+¨SÑ1Ô1ð 	H°VØ�LŠL�œÐ$4¸QÐ?Ñ?Ô?À$ÑFÑGÔGÐGð	Hð 	Hð 	Hñ 	Hô 	Hð 	Hð 	Hð 	Hð 	Hð 	Hð 	Høøøð 	Hð 	Hð 	Hð 	Hð Ðs6   ×!-XØXØ!XØ7-Y0Ù0Y4Ù7Y4Ú-[Û[Û[c                 ó  — t          j        dg d¢¦  «        }t          j        dg d¢¦  «        }t                               d|› �¦  «         t          j        t
          ¦  «        }|D ]"}||j                                      |¦  «         Œ#i }|D ]}|||j        <   Œt          j	        ¦   «         }t          j	        ¦   «         }t          j	        ¦   «         }t          | ¦  «        D �]ž\  }}||         }g }d}t          |¦  «        D ]ü\  }}||j                 }|j        }t          ||¦  «        }t          |¦  «        D ]Ã}t          |	¦  «        D ]±}|j        |         }|j        |         } ||	z  |z   }!|j        |!         }"|j        |!         }#| |j        dz
  k    rŒM|#|j        dz
  k    rŒ\|j                             | d¦  «        sŒx|#| k     rŒ|#| z
  dz   }$|$|k    rŒŽ|                      ||| |#||"¬¦  «        ¦  «         Œ²ŒÄŒýt+          |d	„ d
¬¦  «        }i }%g }&|D �]D}'t-          |&¦  «        |k    r �n-||'j                 }|j        |'j        |'j        dz   …         }(|j        |'j                 })|j        |'j                 }*|j        |)|*dz   …         }+|                     |(¦  «        },|,                     ¦   «         },d                     |,                      ¦   «         ¦  «        },d                     |+¦  «        }-tC          |d¦  «        r|j"        }.n|j#        }.tI          |,|-|.|¦  «        }/|/|%v r�Œd
|%|/<   |&                      ||/|'j%        |'j&        ¬¦  «        ¦  «         �ŒF|&s!|&                      |ddd¬¦  «        ¦  «         g }0d}1|&D ](}2|0                     |2j%        |2j&        z   ¦  «         |1s|2}1Œ)tO          |0¦  «        }3g }4t          |&¦  «        D ]V\  }}2t          j	        ¦   «         }5|2j(        |5d<   |3|         |5d<   |2j%        |5d<   |2j&        |5d<   |4                     |5¦  «         ŒWt-          |4¦  «        dk     rtS          d¦  «        ‚|1€tS          d¦  «        ‚|}6|6||j*        <   |1j(        ||j*        <   |4||j*        <   �Œ tW          |d¦  «        5 }7|7 ,                    t[          j.        |d¬¦  «        dz   ¦  «         ddd¦  «         n# 1 swxY w Y   tW          |d¦  «        5 }7|7 ,                    t[          j.        |d¬¦  «        dz   ¦  «         ddd¦  «         n# 1 swxY w Y   |
rTtW          |d¦  «        5 }7|7 ,                    t[          j.        |d¬¦  «        dz   ¦  «         ddd¦  «         n# 1 swxY w Y   |S )zµ
    XLNet write prediction logic (more complex than Bert's). Write final predictions to the json file and log-odds of
    null if needed.

    Requires utils_squad_evaluate.py
    rè   )ré   rê   rë   Ústart_log_probÚend_log_probrð   )r   r:  r;  rç   rî   rq   Fc                 ó    — | j         | j        z   S r   )r:  r;  rÓ   s    r   rw   z/compute_predictions_log_probs.<locals>.<lambda>­  s   € ¨qÔ/?À!Ä.Ñ/P€ r   TrÕ   r   r¸   r   g    €„.ÁNr   ró   r:  r;  rò   rô   rõ   rö   rø   )/r2   rý   r¼   r½   rù   rú   rû   r±   rü   re   r”   Ú
cls_logitsÚminr×   rþ   Ústart_top_indexrÿ   Úend_top_indexÚparagraph_lenr  r  r|   r6   ré   r   rê   rë   r  r  r  r  r   r   Úhasattrr¸   Údo_lowercase_and_remove_accentrÐ   r:  r;  rå   r   r  rH   r	  r
  r  r  )8r  r  r  rÙ   r  r  r  r  Ústart_n_topÚ	end_n_topr  rÁ   rÀ   r  r(  r  r  r  r  r  r  r  rû   rP   r  r  r   ré   Úcur_null_scorer´   Újr:  rê   Új_indexr;  rë   r'  r)  r*  r+  r,  r-  r.  r/  rÂ   r¿   r¸   r0  r1  r2  r3  rä   r4  r5  r6  r7  s8                                                           r   Úcompute_predictions_log_probsrI  N  sr  € õ* $Ô.ØÐkÐkÐkñô Ðõ #Ô-ØÐEÐEÐEñô Ðõ ‡K‚KÐCÐ+AÐCÐCÑDÔDÐDå +Ô 7½Ñ =Ô =ÐØð Ið IˆØ! 'Ô"7Ô8×?Ò?ÀÑHÔHÐHÐHàÐØð 7ð 7ˆØ06Ð˜FÔ,Ñ-Ð-å!Ô-Ñ/Ô/€OÝ Ô,Ñ.Ô.€NÝ"Ô.Ñ0Ô0Ðå"+¨LÑ"9Ô"9ð F4ñ F4Ñˆ�wØ,¨]Ô;ˆàÐàˆ
å&/°Ñ&9Ô&9ð *	ð *	Ñ"ˆM˜7Ø(¨Ô):Ô;ˆFà#Ô.ˆNõ ˜Z¨Ñ8Ô8ˆJå˜;Ñ'Ô'ð "ð "�Ý˜yÑ)Ô)ð !ð !�AØ%+Ô%8¸Ô%;�NØ"(Ô"8¸Ô";�Kà )™m¨aÑ/�Gà#)Ô#4°WÔ#=�LØ &Ô 4°WÔ =�Ið
 # gÔ&;¸aÑ&?Ò?Ð?Ø Ø  GÔ$9¸AÑ$=Ò=Ð=Ø à"Ô7×;Ò;¸KÈÑOÔOð !Ø Ø  ;Ò.Ð.Ø Ø&¨Ñ4°qÑ8�FØÐ 1Ò1Ð1Ø à&×-Ò-Ø)Ð)Ø*7Ø(3Ø&/Ø+9Ø)5ðñ ô ñô ð ð ð3!ð"õH $ØÐ$QÐ$QÐ[_ð
ñ 
ô 
Ðð ÐØˆØ&ð )	ñ )	ˆDÝ�5‰zŒz˜[Ò(Ð(Ø‘Ø˜tÔ1Ô2ˆGð !œ¨Ô(8¸D¼NÈQÑ<NÐ(OÔPˆJØ$Ô6°tÔ7GÔHˆNØ"Ô4°T´^ÔDˆLØ!Ô,¨^¸|ÈaÑ?OÐ-PÔQˆKØ ×9Ò9¸*ÑEÔEˆHð  —~’~Ñ'Ô'ˆHØ—x’x §¢Ñ 0Ô 0Ñ1Ô1ˆHØŸš Ñ-Ô-ˆIå�y /Ñ2Ô2ð IØ )Ô 7��à )Ô H�å'¨°)¸]ÈOÑ\Ô\ˆJàÐ-Ð-Ð-Ùà+/Ð˜ZÑ(à�LŠLØ Ð  jÀÔATÐcgÔctÐuÑuÔuñô ð ñ ð ð 	\Ø�LŠLÐ)Ð)¨rÀ$ÐUYÐZÑZÔZÑ[Ô[Ð[àˆØ"ÐØð 	,ð 	,ˆEØ×Ò Ô 4°uÔ7IÑ IÑJÔJÐJØ&ð ,Ø&+Ð#øå  Ñ.Ô.ˆàˆ
Ý! %Ñ(Ô(ð 	&ð 	&‰HˆAˆuÝ Ô,Ñ.Ô.ˆFØ"œZˆF�6‰NØ$)¨!¤HˆF�=Ñ!Ø',Ô';ˆFÐ#Ñ$Ø%*Ô%7ˆF�>Ñ"Ø×Ò˜fÑ%Ô%Ð%Ð%åˆz‰?Œ?˜QÒÐÝÐ3Ñ4Ô4Ð4ØÐ&ÝÐ3Ñ4Ô4Ð4àˆ
Ø+5Ð˜œÑ(ð +>Ô*Bˆ˜œÑ'à)3ˆ�w”~Ñ&Ñ&å	Ð$ cÑ	*Ô	*ð C¨fØ�Š•T”Z ¸Ð:Ñ:Ô:¸TÑAÑBÔBÐBðCð Cð Cñ Cô Cð Cð Cð Cð Cð Cð Cøøøð Cð Cð Cð Cõ 
Ð Ñ	%Ô	%ð B¨Ø�Š•T”Z °qÐ9Ñ9Ô9¸DÑ@ÑAÔAÐAðBð Bð Bñ Bô Bð Bð Bð Bð Bð Bð Bøøøð Bð Bð Bð Bð ð HÝÐ+¨SÑ1Ô1ð 	H°VØ�LŠL�œÐ$4¸QÐ?Ñ?Ô?À$ÑFÑGÔGÐGð	Hð 	Hð 	Hñ 	Hô 	Hð 	Hð 	Hð 	Hð 	Hð 	Hð 	Høøøð 	Hð 	Hð 	Hð 	Hð Ðs6   Ò-SÓSÓSÓ'-T Ô T$Ô'T$Ô=-U6Õ6U:Õ=U:r   )Nr0   )F)Ú__doc__r2   r  rÞ   r	   r!   Úmodels.bertr   Úutilsr   Ú
get_loggerÚ__name__r¼   r'   r)   r.   r>   rR   r]   rg   rm   r„   r�   r•   r—   r®   rÐ   rÜ   rå   r8  rI  r   r   r   ú<module>rO     s¸  ððð ð Ð Ð Ð Ø €€€Ø €€€Ø 	€	€	€	Ø €€€à )Ð )Ð )Ð )Ð )Ð )Ø Ð Ð Ð Ð Ð ð 
ˆÔ	˜HÑ	%Ô	%€ðCð Cð Cð('ð 'ð 'ðEð Eð Eðð ð ð #ð #ð #ð4ð ð ð
ð 
ð 
ð 
ð*1ð 1ð 1ð
\ð \ð \ðD)ð )ð )ð9ð 9ð 9ð.,ð ,ð ,ðð ð ð ð>[ð [ð [ð [ð|	ð 	ð 	ðð ð ð.Lð Lð Lð^}ð }ð }ð }ð }r   