§
    ™ŠtjJé  ã                  ó\  — d Z ddlmZ ddlZddlZddlZddlZddlZddl	Z	ddl
mZ ddlmZmZ ddlmZmZmZ ddlmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZmZ ddlmZm Z m!Z! ddlm"Z# ddlm$Z% ddl&m'Z'm(Z( ddl)m*Z* ddl+m,Z, ddl-m.Z.m/Z/ ddl0m1Z1m2Z2 ddl0m3Z4 ddl5m6Z6m7Z7 ddl8m9Z9m:Z:m;Z;m<Z<m=Z= ddl>m?Z? ddl@mAZA ddlBmCZC ddlDmEZE ddlFmGZG ddlHmIZImJZJmKZK ddlLmMZN ddlOm"ZP ddlOmQZQmRZR erddlSZT ejU        eV¦  «        ZWeg eEez  f         ez  eeXgef         z  ez  eEz  ZYeg eEez  f         ez  ZZ G d„ d e[¦  «        Z\ G d!„ d"eX¦  «        Z] G d#„ d$eX¦  «        Z^	 d”d•d,„Z_d–d/„Z` G d0„ d1eC¦  «        Zad—d3„Zbd˜d9„Zcd™d<„Zddšd>„Zed›dF„ZfdœdL„Zgd�dN„ZhdždP„ZidŸdZ„Zjd d\„Zkd¡d^„Zlddddd_œd¢dh„Zmddddiœd£dl„Znddmœd¤dp„Zoddddiœd¥dq„Zpddddiœd£dr„Zqddmœd¤ds„Zr	 	 	 d¦d§d{„Zs G d|„ d}eCd~¬¦  «        Zteju         G d€„ d�¦  «        ¦   «         Zvd¨dƒ„Zwd©d†„Zxd‡Zydddˆddd~dd‰œdªd’„Zzdddˆddd~dd‰œdªd“„Z{dS )«z>Utilities for running language models or Chains over datasets.é    )ÚannotationsN)ÚCallable)ÚdatetimeÚtimezone)ÚTYPE_CHECKINGÚAnyÚcast)Úwarn_deprecated)Ú	Callbacks)ÚBaseLanguageModel)ÚBaseMessageÚmessages_from_dict)Ú
ChatResultÚ	LLMResult)ÚRunnableÚRunnableConfigÚRunnableLambda©Úconfig)Úutils)ÚEvaluatorCallbackHandlerÚwait_for_all_evaluators)ÚLangChainTracer)ÚClient)Úget_git_infoÚget_langchain_env_var_metadata)ÚEvaluationResultÚRunEvaluator)Úrun_evaluator)Úas_runnableÚis_traceable_function)ÚDatasetÚDataTypeÚExampleÚRunÚTracerSession)ÚLangSmithError)Ú	HTTPError)Ú	TypedDict)ÚChain)Úload_evaluator)ÚEvaluatorTypeÚPairwiseStringEvaluatorÚStringEvaluator)Ú
evaluation)Úname_generationÚprogressc                  ó   — e Zd ZdZdS )ÚInputFormatErrorz(Raised when the input format is invalid.N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__© ó    úm/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/langchain_classic/smith/evaluation/runner_utils.pyr3   r3   I   s   € € € € € Ø2Ð2Ð2Ð2r9   r3   c                  ó"   — e Zd ZdZdd„Zdd„ZdS )Ú
TestResultz1A dictionary of the results of a single test run.Úreturnúpd.DataFramec                ó¢   — |                       ¦   «         }d„ |j        D ¦   «         }|                     d¬¦  «                             |d¬¦  «        S )zÿReturn quantiles for the feedback scores.

        This method calculates and prints the quantiles for the feedback scores
        across all feedback keys.

        Returns:
            A DataFrame containing the quantiles for each feedback key.
        c                óD   — g | ]}|                      d ¦  «        s|dv ¯|‘ŒS ))úinputs.úoutputs.Ú	reference>   ÚinputÚoutput)Ú
startswith)Ú.0Úcols     r:   ú
<listcomp>z5TestResult.get_aggregate_feedback.<locals>.<listcomp>`   sG   € ð 
ð 
ð 
àØ�~Š~ÐBÑCÔCð
ð Ð)Ð)Ð)ð ð *Ð)Ð)r9   Úall)Úincludeé   )Úaxis)Úto_dataframeÚcolumnsÚdescribeÚdrop)ÚselfÚdfÚto_drops      r:   Úget_aggregate_feedbackz!TestResult.get_aggregate_feedbackS   s^   € ð ×ÒÑ Ô ˆð
ð 
à”zð
ñ 
ô 
ˆð �{Š{ 5ˆ{Ñ)Ô)×.Ò.¨w¸QÐ.Ñ?Ô?Ð?r9   c           	     óˆ  — 	 ddl }n$# t          $ r}d}t          |¦  «        |‚d}~ww xY wg }g }| d                              ¦   «         D �]g\  }}|d         }|                     d¦  «        }	t	          |	t
          ¦  «        rd„ |	                     ¦   «         D ¦   «         }
n	|	€i }
nd|	i}
i d„ |d	                              ¦   «         D ¦   «         ¥|
¥}d
|v r^t	          |d
         t
          ¦  «        r8|                     d„ |d
                              ¦   «         D ¦   «         ¦  «         n|d
         |d
<   |                     i d„ |D ¦   «         ¥|                     d¦  «        |d         |                     d¦  «        dœ¥¦  «         |                     |¦  «         |                     |¦  «         �Œi |j        ||¬¦  «        S )z#Convert the results to a dataframe.r   NzfPandas is required to convert the results to a dataframe. to install pandas, run `pip install pandas`.ÚresultsÚfeedbackrE   c                ó    — i | ]\  }}d |› �|“ŒS )rB   r8   ©rG   ÚkÚvs      r:   ú
<dictcomp>z+TestResult.to_dataframe.<locals>.<dictcomp>y   s$   € ÐHÐHÐH±°°1˜. Q˜.˜.¨!ÐHÐHÐHr9   c                ó    — i | ]\  }}d |› �|“ŒS )rA   r8   rZ   s      r:   r]   z+TestResult.to_dataframe.<locals>.<dictcomp>€   s$   € ÐHÐHÐH©¨¨1�=˜Q�=�= !ÐHÐHÐHr9   rD   rC   c                ó    — i | ]\  }}d |› �|“ŒS )z
reference.r8   rZ   s      r:   r]   z+TestResult.to_dataframe.<locals>.<dictcomp>†   s'   € ÐUÐUÐU±°°AÐ) aÐ)Ð)¨1ÐUÐUÐUr9   c                ó.   — i | ]}d |j         › �|j        “ŒS )z	feedback.)ÚkeyÚscore)rG   Úfs     r:   r]   z+TestResult.to_dataframe.<locals>.<dictcomp>Œ   s'   € ÐFÐFÐF¸Ð* 1¤5Ð*Ð*¨A¬GÐFÐFÐFr9   ÚErrorÚexecution_timeÚrun_id)Úerrorre   rf   )Úindex)	ÚpandasÚImportErrorÚitemsÚgetÚ
isinstanceÚdictÚupdateÚappendÚ	DataFrame)rR   ÚpdÚeÚmsgÚindicesÚrecordsÚ
example_idÚresultrX   Úoutput_rE   Úrs               r:   rN   zTestResult.to_dataframeh   s/  € ð	*ØÐÐÐÐøÝð 	*ð 	*ð 	*ð@ð õ ˜cÑ"Ô"¨Ð)øøøøð	*øøøð ˆØˆØ"& y¤/×"7Ò"7Ñ"9Ô"9ð 	'ñ 	'ÑˆJ˜Ø˜jÔ)ˆHØ—j’j Ñ*Ô*ˆGÝ˜'¥4Ñ(Ô(ð -ØHÐH¸¿º¹¼ÐHÑHÔH��Ø�Ø��à" GÐ,�ðØHÐH°°w´×0EÒ0EÑ0GÔ0GÐHÑHÔHðàðˆAð ˜fÐ$Ð$Ý˜f [Ô1µ4Ñ8Ô8ð 9Ø—H’HØUÐU¸ÀÔ9L×9RÒ9RÑ9TÔ9TÐUÑUÔUñô ð ð ð &,¨KÔ%8�A�k‘NØ�HŠHðØFÐF¸XÐFÑFÔFðà#ŸZšZ¨Ñ0Ô0Ø&,Ð-=Ô&>Ø$Ÿjšj¨Ñ2Ô2ð	ð ð ñô ð ð �NŠN˜1ÑÔÐØ�NŠN˜:Ñ&Ô&Ð&Ñ&àˆrŒ|˜G¨7Ð3Ñ3Ô3Ð3s   ‚ ‡
(‘#£(N)r=   r>   )r4   r5   r6   r7   rU   rN   r8   r9   r:   r<   r<   P   sF   € € € € € Ø;Ð;ð@ð @ð @ð @ð*-4ð -4ð -4ð -4ð -4ð -4r9   r<   c                  ó,   ‡ — e Zd ZdZdˆ fd„Zdd„Zˆ xZS )Ú	EvalErrorz"Your architecture raised an error.rd   ÚBaseExceptionÚkwargsr   r=   ÚNonec                ó>   •—  t          ¦   «         j        dd|i|¤Ž dS )zÌInitialize the `EvalError` with an error and additional attributes.

        Args:
            Error: The error that occurred.
            **kwargs: Additional attributes to include in the error.
        rd   Nr8   )ÚsuperÚ__init__)rR   rd   r~   Ú	__class__s      €r:   r‚   zEvalError.__init__›   s-   ø€ ð 	�‰ŒÔÐ/Ð/˜uÐ/¨Ð/Ð/Ð/Ð/Ð/r9   ÚnameÚstrc                ód   — 	 | |         S # t           $ r}d|› d�}t          |¦  «        |‚d}~ww xY w)zúGet an attribute from the `EvalError`.

        Args:
            name: The name of the attribute to get.

        Returns:
            The value of the attribute.

        Raises:
            AttributeError: If the attribute does not exist.
        z%'EvalError' object has no attribute 'ú'N)ÚKeyErrorÚAttributeError)rR   r„   rs   rt   s       r:   Ú__getattr__zEvalError.__getattr__¤   sP   € ð	-Ø˜”:ÐøÝð 	-ð 	-ð 	-ØA¸$ÐAÐAÐAˆCÝ  Ñ%Ô%¨1Ð,øøøøð	-øøøs   ‚
 Š
/”*ª/)rd   r}   r~   r   r=   r   )r„   r…   r=   r   )r4   r5   r6   r7   r‚   rŠ   Ú__classcell__)rƒ   s   @r:   r|   r|   ˜   sW   ø€ € € € € Ø,Ð,ð0ð 0ð 0ð 0ð 0ð 0ð-ð -ð -ð -ð -ð -ð -ð -r9   r|   ú<my_dataset>Úllm_or_chain_factoryÚMODEL_OR_CHAIN_FACTORYÚdataset_namer…   r=   ÚMCFc                ó�  ‡‡	‡
‡‡— t          | t          ¦  «        rF| Š‰j        j        }| j        �,‰j        j        j        }d|› d|› d|› d�}t          |¦  «        ‚ˆfd„S t          | t          ¦  «        r| S t          | t          ¦  «        r| Š
ˆ
fd„S t          | ¦  «        �r"t          | ¦  «        r"t          t          d| ¦  «        ¦  «        Šˆfd	„S 	  | ¦   «         }nc# t          $ rV t          d| ¦  «        }t          j        |¦  «        }t                               d
|¦  «         t#          |¦  «        Šˆfd„cY S w xY wt          d| ¦  «        Š	t          |t          ¦  «        r|S t          t          d|¦  «        ¦  «        r"t          t          d|¦  «        ¦  «        Šˆfd„S t          |t          ¦  «        sˆ	fd„S ‰	S | S )z½Wrap in a chain factory.

    Forgive the user if they pass in a chain without memory instead of a chain
    factory. It's a common mistake. Raise a more helpful error message as well.
    Na$  Cannot directly evaluate a chain with stateful memory. To evaluate this chain, pass in a chain constructor that initializes fresh memory each time it is called.  This will safeguard against information leakage between dataset examples.
For example:

def chain_constructor():
    new_memory = z(...)
    return z*(memory=new_memory, ...)

run_on_dataset("z", chain_constructor, ...)c                 ó   •— ‰ S ©Nr8   )Úchains   €r:   ú<lambda>z(_wrap_in_chain_factory.<locals>.<lambda>Ó   s   ø€ �u€ r9   c                 ó   •— ‰ S r“   r8   )Úlcfs   €r:   r•   z(_wrap_in_chain_factory.<locals>.<lambda>Ù   s   ø€ �s€ r9   r   c                 ó   •— ‰ S r“   r8   ©Ú	runnable_s   €r:   r•   z(_wrap_in_chain_factory.<locals>.<lambda>Ý   ó   ø€ ˜9€ r9   z'Wrapping function %s as RunnableLambda.c                 ó   •— ‰ S r“   r8   )Úwrappeds   €r:   r•   z(_wrap_in_chain_factory.<locals>.<lambda>æ   s   ø€ ˜7€ r9   c                 ó   •— ‰ S r“   r8   r™   s   €r:   r•   z(_wrap_in_chain_factory.<locals>.<lambda>î   r›   r9   c                 ó"   •— t          ‰ ¦  «        S r“   )r   )Úconstructors   €r:   r•   z(_wrap_in_chain_factory.<locals>.<lambda>ñ   s   ø€ �>¨+Ñ6Ô6€ r9   )rm   r*   rƒ   r4   ÚmemoryÚ
ValueErrorr   r   Úcallabler!   r    r	   Ú	TypeErrorÚinspectÚ	signatureÚloggerÚinfor   )r�   r�   Úchain_classÚmemory_classrt   Ú_modelÚ	user_funcÚsigr”   r    r—   rš   r�   s           @@@@@r:   Ú_wrap_in_chain_factoryr®   ·   sC  øøøøø€ õ Ð&­Ñ.Ô.ð Ø$ˆØ”oÔ.ˆØÔ&Ð2Ø œ<Ô1Ô:ˆLð
Lð %1ð
Lð 
Lð *ð
Lð 
Lð $0ð
Lð 
Lð 
Lð õ ˜S‘/”/Ð!Øˆ}ˆ}ˆ}ÐÝÐ&Õ(9Ñ:Ô:ð $Ø#Ð#ÝÐ&­Ñ1Ô1ð à"ˆØˆ{ˆ{ˆ{ÐÝÐ$Ñ%Ô%ñ Ý Ð!5Ñ6Ô6ð 	%Ý#¥D¨Ð5IÑ$JÔ$JÑKÔKˆIØ$Ð$Ð$Ð$Ð$ð	#Ø)Ð)Ñ+Ô+ˆFˆFøÝð 	#ð 	#ð 	#å˜ZÐ)=Ñ>Ô>ˆIÝÔ# IÑ.Ô.ˆCÝ�KŠKÐAÀ3ÑGÔGÐGÝ$ YÑ/Ô/ˆGØ"�?�?�?Ð"Ð"Ð"ð	#øøøõ ˜:Ð';Ñ<Ô<ˆÝ�fÕ/Ñ0Ô0ð 	ð ˆMÝ ¥ j°&Ñ!9Ô!9Ñ:Ô:ð 	%Ý#¥D¨°VÑ$<Ô$<Ñ=Ô=ˆIØ$Ð$Ð$Ð$Ð$Ý˜&¥(Ñ+Ô+ð 	7à6Ð6Ð6Ð6Ð6àÐØÐs   Ã
C! Ã!AEÅ EÚinputsúdict[str, Any]c                óž  — | sd}t          |¦  «        ‚g }d| v rRt          | d         t          ¦  «        s,dt          | d         ¦  «        j        › �}t          |¦  «        ‚| d         g}�n&d| v rot          | d         t
          ¦  «        rt          d„ | d         D ¦   «         ¦  «        s,dt          | d         ¦  «        j        › �}t          |¦  «        ‚| d         }n³t          | ¦  «        dk    rŒt          t          |  
                    ¦   «         ¦  «        ¦  «        }t          |t          ¦  «        r|g}nYt          |t
          ¦  «        rt          d„ |D ¦   «         ¦  «        r|}n(d	| › �}t          |¦  «        ‚d
| › �}t          |¦  «        ‚t          |¦  «        dk    r|d         S dt          |¦  «        › d�}t          |¦  «        ‚)z»Get prompt from inputs.

    Args:
        inputs: The input dictionary.

    Returns:
        A string prompt.

    Raises:
        InputFormatError: If the input format is invalid.
    úInputs should not be empty.Úpromptz"Expected string for 'prompt', got Úpromptsc              3  ó@   K  — | ]}t          |t          ¦  «        V — Œd S r“   ©rm   r…   ©rG   Úis     r:   ú	<genexpr>z_get_prompt.<locals>.<genexpr>  s=   è è € ð >
ð >
Ø#$�J�q�#ÑÔð>
ð >
ð >
ð >
ð >
ð >
r9   z,Expected list of strings for 'prompts', got rL   c              3  ó@   K  — | ]}t          |t          ¦  «        V — Œd S r“   r¶   r·   s     r:   r¹   z_get_prompt.<locals>.<genexpr>  s,   è è € Ð.SÐ.SÀa­z¸!½SÑ/AÔ/AÐ.SÐ.SÐ.SÐ.SÐ.SÐ.Sr9   z)LLM Run expects string prompt input. Got z5LLM Run expects 'prompt' or 'prompts' in inputs. Got r   z)LLM Run expects single prompt input. Got z	 prompts.)r3   rm   r…   Útyper4   ÚlistrJ   ÚlenÚnextÚiterÚvalues)r¯   rt   r´   Úprompt_s       r:   Ú_get_promptrÂ   ÷   s  € ð ð $Ø+ˆÝ˜sÑ#Ô#Ð#à€GØ�6ÐÐÝ˜& Ô*­CÑ0Ô0ð 	(ØXµt¸FÀ8Ô<LÑ7MÔ7MÔ7VÐXÐXˆCÝ" 3Ñ'Ô'Ð'Ø˜(Ô#Ð$ˆ‰Ø	�fÐ	Ð	Ý˜& Ô+­TÑ2Ô2ð 	(½#ð >
ð >
Ø(.¨yÔ(9ð>
ñ >
ô >
ñ ;
ô ;
ð 	(ð;Ý˜V IÔ.Ñ/Ô/Ô8ð;ð ;ð õ # 3Ñ'Ô'Ð'Ø˜Ô#ˆˆÝ	ˆV‰Œ˜Ò	Ð	Ý•t˜FŸMšM™OœOÑ,Ô,Ñ-Ô-ˆÝ�g�sÑ#Ô#ð 	(Ø�iˆGˆGÝ˜¥Ñ&Ô&ð 	(­3Ð.SÐ.SÈ7Ð.SÑ.SÔ.SÑ+SÔ+Sð 	(ØˆGˆGàF¸fÐFÐFˆCÝ" 3Ñ'Ô'Ð'àNÀfÐNÐNˆÝ˜sÑ#Ô#Ð#Ý
ˆ7�|„|�qÒÐØ�qŒzÐØ
Mµc¸'±l´lÐ
MÐ
MÐ
M€CÝ
˜3Ñ
Ô
Ðr9   c                  ó   — e Zd ZU dZded<   dS )ÚChatModelInputzInput for a chat model.zlist[BaseMessage]ÚmessagesN©r4   r5   r6   r7   Ú__annotations__r8   r9   r:   rÄ   rÄ   )  s$   € € € € € € Ø!Ð!àÐÐÑÐÐr9   rÄ   rn   c                ó2  — | sd}t          |¦  «        ‚|                      ¦   «         }d| v r|                     d¦  «        |d<   nDt          | ¦  «        dk    r1t	          t          |                      ¦   «         ¦  «        ¦  «        |d<   d|v rx|d         }t          |t          ¦  «        rt          d„ |D ¦   «         ¦  «        r|g}t          |¦  «        dk    rt          |d         ¦  «        |d<   nd}t          |¦  «        ‚|S d| › �}t          |¦  «        ‚)	zÊGet Chat Messages from inputs.

    Args:
        inputs: The input dictionary.

    Returns:
        A list of chat messages.

    Raises:
        InputFormatError: If the input format is invalid.
    r²   rÅ   rD   rL   c              3  ó@   K  — | ]}t          |t          ¦  «        V — Œd S r“   )rm   rn   r·   s     r:   r¹   z _get_messages.<locals>.<genexpr>E  s=   è è € ð 2
ð 2
Ø$%�J�q�$ÑÔð2
ð 2
ð 2
ð 2
ð 2
ð 2
r9   r   zGBatch messages not supported. Please provide a single list of messages.zMChat Run expects single List[dict] or List[List[dict]] 'messages' input. Got )r3   ÚcopyÚpopr½   r¾   r¿   rÀ   rm   r¼   rJ   r   )r¯   rt   Ú
input_copyÚraw_messagess       r:   Ú_get_messagesrÎ   /  sS  € ð ð $Ø+ˆÝ˜sÑ#Ô#Ð#Ø—’‘”€JØ�VÐÐØ(Ÿnšn¨ZÑ8Ô8ˆ
�7ÑÐÝ	ˆV‰Œ˜Ò	Ð	Ý"¥4¨¯ª©¬Ñ#8Ô#8Ñ9Ô9ˆ
�7ÑØ�*ÐÐØ! 'Ô*ˆÝ�l¥DÑ)Ô)ð 	*­cð 2
ð 2
Ø)5ð2
ñ 2
ô 2
ñ /
ô /
ð 	*ð )˜>ˆLÝˆ|ÑÔ Ò!Ð!Ý"4°\À!´_Ñ"EÔ"EˆJ�wÑÐð,ð õ # 3Ñ'Ô'Ð'ØÐð	 Øð	 ð 	 ð õ ˜3Ñ
Ô
Ðr9   Úfirst_exampler$   Úinput_mapperúCallable[[dict], Any] | Noner   c                óÞ  — |r~ || j         pi ¦  «        }t          |t          ¦  «        sSt          |t          ¦  «        rt	          d„ |D ¦   «         ¦  «        s'd|› dt          |¦  «        › d�}t          |¦  «        ‚d S d S 	 t          | j         pi ¦  «         d S # t          $ rH 	 t          | j         pi ¦  «         Y d S # t          $ r }d| j         › d�}t          |¦  «        |‚d }~ww xY ww xY w)Nc              3  ó@   K  — | ]}t          |t          ¦  «        V — Œd S r“   ©rm   r   ©rG   rt   s     r:   r¹   z>_validate_example_inputs_for_language_model.<locals>.<genexpr>b  s,   è è € ÐIÐI°S•J˜s¥KÑ0Ô0ÐIÐIÐIÐIÐIÐIr9   z˜When using an input_mapper to prepare dataset example inputs for an LLM or chat model, the output must a single string or a list of chat messages.
Got: ú	 of type ú.zvExample inputs do not match language model input format. Expected a dictionary with messages or a single prompt. Got: z› Please update your dataset OR provide an input_mapper to convert the example.inputs to a compatible format for the llm or chat model you wish to evaluate.)	r¯   rm   r…   r¼   rJ   r»   r3   rÂ   rÎ   )rÏ   rÐ   Úprompt_inputrt   Úerr2s        r:   Ú+_validate_example_inputs_for_language_modelrÚ   Z  sy  € ð ð 6Ø#�| MÔ$8Ð$>¸BÑ?Ô?ˆÝ˜,­Ñ,Ô,ð 
	(Ý�|¥TÑ*Ô*ð
	(åÐIÐI¸LÐIÑIÔIÑIÔIð
	(ð
Gð 'ðGð Gõ 26°lÑ1CÔ1CðGð Gð Gð õ # 3Ñ'Ô'Ð'ð
	(ð 
	(ð 
	(ð 
	(ð	6Ý˜Ô,Ð2°Ñ3Ô3Ð3Ð3Ð3øÝð 	6ð 	6ð 	6ð6Ý˜mÔ2Ð8°bÑ9Ô9Ð9Ð9Ð9Ð9øÝ#ð 	6ð 	6ð 	6ðGà*Ô1ðGð Gð Gð õ ' sÑ+Ô+°Ð5øøøøð	6øøøð	6øøøs*   ÂB Â
C,Â%B>Â>
C(ÃC#Ã#C(Ã(C,r”   r*   c                ón  — |r¥ || j         pi ¦  «        }t          |j        ¦  «                             |¦  «        }t	          |t
          ¦  «        s%d|› dt          |¦  «        › d�}t          |¦  «        ‚|r.d|j        › d|                     ¦   «         › �}t          |¦  «        ‚dS | j         pi }t          |j        ¦  «                             |¦  «        }t          |¦  «        dk    rt          |j        ¦  «        dk    rdS |r.d|j        › d|                     ¦   «         › �}t          |¦  «        ‚dS )	z<Validate that the example inputs match the chain input keys.zvWhen using an input_mapper to prepare dataset example inputs for a chain, the mapped value must be a dictionary.
Got: rÖ   r×   zAMissing keys after loading example using input_mapper.
Expected: z. Got: rL   z³Example inputs missing expected chain input keys. Please provide an input_mapper to convert the example.inputs to a compatible format for the chain you wish to evaluate.Expected: N)
r¯   ÚsetÚ
input_keysÚ
differencerm   rn   r»   r3   Úkeysr½   )rÏ   r”   rÐ   Úfirst_inputsÚmissing_keysrt   s         r:   Ú"_validate_example_inputs_for_chainrâ   }  s©  € ð ð (Ø#�| MÔ$8Ð$>¸BÑ?Ô?ˆÝ˜5Ô+Ñ,Ô,×7Ò7¸ÑEÔEˆÝ˜,­Ñ-Ô-ð 	(ðGà&ðGð Gå15°lÑ1CÔ1CðGð Gð Gð õ
 # 3Ñ'Ô'Ð'Øð 	(ðNØ$Ô/ðNð NØ8D×8IÒ8IÑ8KÔ8KðNð Nð õ # 3Ñ'Ô'Ð'ð	(ð 	(ð %Ô+Ð1¨rˆÝ˜5Ô+Ñ,Ô,×7Ò7¸ÑEÔEˆÝˆ|ÑÔ Ò!Ð!¥c¨%Ô*:Ñ&;Ô&;¸qÒ&@Ð&@ð ˆDØð 	(ð.ð #Ô-ð.ð .ð %×)Ò)Ñ+Ô+ð	.ð .ð õ # 3Ñ'Ô'Ð'ð	(ð 	(r9   Úexamplec                ó  — t          |t          ¦  «        rt          | |¦  «         dS  |¦   «         }t          |t          ¦  «        rt	          | ||¦  «         dS t          |t
          ¦  «        rt                               d|¦  «         dS dS )z9Validate that the example inputs are valid for the model.z Skipping input validation for %sN)rm   r   rÚ   r*   râ   r   r§   Údebug)rã   r�   rÐ   r”   s       r:   Ú_validate_example_inputsræ   ¥  s¢   € õ Ð&Õ(9Ñ:Ô:ð DÝ3°G¸\ÑJÔJÐJÐJÐJà$Ð$Ñ&Ô&ˆÝ�e�UÑ#Ô#ð 	Då.¨w¸¸|ÑLÔLÐLÐLÐLÝ˜�xÑ(Ô(ð 	DÝ�LŠLÐ;¸UÑCÔCÐCÐCÐCð	Dð 	Dr9   Úexamplesúlist[Example]r/   úsmith_eval.RunEvalConfig | NoneÚ	data_typer#   úlist[RunEvaluator] | Nonec           	     óP  — |r¡t          | t          ¦  «        rd\  }}d}nHd} | ¦   «         }t          |t          ¦  «        r|j        nd}t          |t          ¦  «        r|j        nd}t          ||||d         j        rt          |d         j        ¦  «        nd||¦  «        }nd}|S )z<Configure the evaluators to run on the results of the chain.)NNÚllmr”   Nr   )rm   r   r*   rÝ   Úoutput_keysÚ_load_run_evaluatorsÚoutputsr¼   )	r�   rç   r/   rê   Ú
run_inputsÚrun_outputsÚrun_typer”   Úrun_evaluatorss	            r:   Ú_setup_evaluationrõ   ¹  sÍ   € ð ð ÝÐ*Õ,=Ñ>Ô>ð 	RØ&0Ñ#ˆJ˜ØˆHˆHàˆHØ(Ð(Ñ*Ô*ˆEÝ-7¸½uÑ-EÔ-EÐO˜Ô)Ð)È4ˆJÝ/9¸%ÅÑ/GÔ/GÐQ˜%Ô+Ð+ÈTˆKÝ-ØØØØ)1°!¬Ô)<ÐF�D�˜!”Ô$Ñ%Ô%Ð%À$ØØñ
ô 
ˆˆð ˆØÐr9   r   úsmith_eval.RunEvalConfigrñ   úlist[str] | Noneú
str | Nonec                ó  — d }| j         r*| j         }|r ||vrt                               d||¦  «         nN|rt          |¦  «        dk    r	|d         }n0|�.t          |¦  «        dk    rt                               d|¦  «         |S )NzZInput key %s not in chain's specified input keys %s. Evaluation behavior may be undefined.rL   r   zªChain expects multiple input keys: %s, Evaluator is likely to fail. Evaluation behavior may be undefined. Specify an input_key in the RunEvalConfig to avoid this warning.)Ú	input_keyr§   Úwarningr½   )r   rñ   rú   s      r:   Ú_determine_input_keyrü   ×  s´   € ð €IØÔð 
ØÔ$ˆ	Øð 	˜)¨:Ð5Ð5Ý�NŠNð8àØñ	ô ð øð 
ð 
�˜J™œ¨1Ò,Ð,Ø˜q”Mˆ	ˆ	Ø	Ð	¥C¨
¡O¤O°aÒ$7Ð$7Ý�ŠðPð ñ		
ô 	
ð 	
ð Ðr9   rò   c                ó  — d }| j         r*| j         }|r ||vrt                               d||¦  «         nN|rt          |¦  «        dk    r	|d         }n0|�.t          |¦  «        dk    rt                               d|¦  «         |S )Nz`Prediction key %s not in chain's specified output keys %s. Evaluation behavior may be undefined.rL   r   z’Chain expects multiple output keys: %s, Evaluation behavior may be undefined. Specify a prediction_key in the RunEvalConfig to avoid this warning.)Úprediction_keyr§   rû   r½   )r   rò   rþ   s      r:   Ú_determine_prediction_keyrÿ   ò  s·   € ð €NØÔð 
ØÔ.ˆØð 	˜>°Ð<Ð<Ý�NŠNð8àØñ	ô ð øð 
ð 
�˜[Ñ)Ô)¨QÒ.Ð.Ø$ QœˆˆØ	Ð	 ¥S¨Ñ%5Ô%5¸Ò%9Ð%9Ý�Šð;ð ñ		
ô 	
ð 	
ð Ðr9   Úexample_outputsc                óÆ   — | j         r%| j         }|r||vrd|› d|› �}t          |¦  «        ‚n4|r0t          |¦  «        dk    rt          t	          |¦  «        ¦  «        }nd }|S )NzReference key z! not in Dataset example outputs: rL   )Úreference_keyr¢   r½   r¾   r¿   )r   r   r  rt   s       r:   Ú_determine_reference_keyr    s—   € ð Ôð ØÔ,ˆØð 	"˜}°OÐCÐCð7 ð 7ð 7Ø%4ð7ð 7ð õ ˜S‘/”/Ð!øØ	ð �S Ñ1Ô1°QÒ6Ð6Ý�T /Ñ2Ô2Ñ3Ô3ˆˆàˆØÐr9   Úeval_configúSsmith_eval_config.SINGLE_EVAL_CONFIG_TYPE | smith_eval_config.CUSTOM_EVALUATOR_TYPEÚeval_llmúBaseLanguageModel | Noneró   r  rú   rþ   r   c           	     óª  — t          | t          ¦  «        r| S t          | t          t          f¦  «        r=t          | t          ¦  «        st          | ¦  «        } t	          | |¬¦  «        }| j        }	nÅt          | t          j        ¦  «        rld|i|                      ¦   «         ¥}
t	          | j	        fi |
¤Ž}| j	        j        }	t          | t          j
        ¦  «        r| j        p|}| j        p|}| j        p|}n?t          | ¦  «        rt          | ¦  «        S dt!          | ¦  «        › �}t#          |¦  «        ‚t          |t$          ¦  «        rI|j        r|€d|	› d|› d�}t#          |¦  «        ‚t(          j                             |||||||	g¬¦  «        }n?t          |t.          ¦  «        rd|	› d	�}t1          |¦  «        ‚d|	› d
�}t1          |¦  «        ‚|S )N)rí   rí   zUnknown evaluator type: zPMust specify reference_key in smith_eval.RunEvalConfig to use evaluator of type z) with dataset with multiple output keys: r×   )rú   rþ   r  ÚtagszRun evaluator for zó is not implemented. PairwiseStringEvaluators compare the outputs of two different models rather than the output of a single model. Did you mean to use a StringEvaluator instead?
See: https://python.langchain.com/docs/guides/evaluation/string/z is not implemented)rm   r   r,   r…   r+   ÚvalueÚsmith_eval_configÚ
EvalConfigÚ
get_kwargsÚevaluator_typeÚSingleKeyEvalConfigrú   rþ   r  r£   Úrun_evaluator_decr»   r¢   r.   Úrequires_referenceÚ
smith_evalÚStringRunEvaluatorChainÚfrom_run_and_data_typer-   ÚNotImplementedError)r  r  ró   rê   r   r  rú   rþ   Ú
evaluator_Úeval_type_tagr~   rt   r   s                r:   Ú_construct_run_evaluatorr    sA  € õ �+�|Ñ,Ô,ð ØÐÝ�+¥­sÐ3Ñ4Ô4ð Ý˜+¥}Ñ5Ô5ð 	5Ý'¨Ñ4Ô4ˆKÝ# K°XÐ>Ñ>Ô>ˆ
Ø#Ô)ˆˆÝ	�KÕ!2Ô!=Ñ	>Ô	>ð Ø˜Ð> [×%;Ò%;Ñ%=Ô%=Ð>ˆÝ# KÔ$>ÐIÐIÀ&ÐIÐIˆ
Ø#Ô2Ô8ˆå�kÕ#4Ô#HÑIÔIð 	GØ#Ô-Ð:°ˆIØ(Ô7ÐI¸>ˆNØ'Ô5ÐF¸ˆMøÝ	�+Ñ	Ô	ð å  Ñ-Ô-Ð-à<­¨kÑ):Ô):Ð<Ð<ˆÝ˜‰oŒoÐå�*�oÑ.Ô.ð 'ØÔ(ð 	"¨]Ð-BðJØ&3ðJð Jà7FðJð Jð Jð õ
 ˜S‘/”/Ð!Ý"Ô:×QÒQØØØØØ)Ø'Ø�ð Rñ 
ô 
ˆˆõ 
�JÕ 7Ñ	8Ô	8ð 'ðQ ð Qð Qð Qð 	õ " #Ñ&Ô&Ð&ð F =ÐEÐEÐEˆÝ! #Ñ&Ô&Ð&ØÐr9   ú)tuple[str | None, str | None, str | None]c                ól   — t          | |¦  «        }t          | |¦  «        }t          | |¦  «        }|||fS r“   )rü   rÿ   r  )r   rñ   rò   r   rú   rþ   r  s          r:   Ú	_get_keysr  b  s>   € õ % V¨ZÑ8Ô8€IÝ.¨v°{ÑCÔC€NÝ,¨V°_ÑEÔE€MØ�n mÐ3Ð3r9   úlist[RunEvaluator]c                ó¨  — g }d\  }}}	| j         s%| j        r4t          d„ | j        D ¦   «         ¦  «        rt          | |||¦  «        \  }}}	| j         D ]2}
t	          |
| j        ||||	||¦  «        }|                     |¦  «         Œ3| j        pg }|D ]Á}t          |t          ¦  «        r|                     |¦  «         Œ-t          |t          ¦  «        r9|                     t          j                             ||||||	¬¦  «        ¦  «         Œ{t          |¦  «        r#|                     t          |¦  «        ¦  «         Œ­d|› d�}t          |¦  «        ‚|S )aŠ  Load run evaluators from a configuration.

    Args:
        config: Configuration for the run evaluators.
        run_type: The type of run.
        data_type: The type of dataset used in the run.
        example_outputs: The example outputs.
        run_inputs: The input keys for the run.
        run_outputs: The output keys for the run.

    Returns:
        A list of run evaluators.
    ©NNNc              3  ó@   K  — | ]}t          |t          ¦  «        V — Œd S r“   )rm   r.   )rG   rs   s     r:   r¹   z'_load_run_evaluators.<locals>.<genexpr>‡  s,   è è € ÐQÐQ°1•
˜1�oÑ.Ô.ÐQÐQÐQÐQÐQÐQr9   )rú   rþ   r  zUnsupported custom evaluator: z+. Expected RunEvaluator or StringEvaluator.)Ú
evaluatorsÚcustom_evaluatorsÚanyr  r  r  rp   rm   r   r.   r  r  r  r£   r  r¢   )r   ró   rê   r   rñ   rò   rô   rú   rþ   r  r  r   r!  Úcustom_evaluatorrt   s                  r:   rï   rï   n  sÒ  € ð* €NØ/?Ñ,€Iˆ~˜}ØÔð 	
ØÔ ð	
åÐQÐQ¸Ô8PÐQÑQÔQÑQÔQð	
õ 4=ØØØØñ	4
ô 4
Ñ0ˆ	�> =ð Ô(ð -ð -ˆÝ0ØØŒOØØØØØØñ	
ô 	
ˆð 	×Ò˜mÑ,Ô,Ð,Ð,ØÔ0Ð6°BÐØ-ð "ð "ÐÝÐ&­Ñ5Ô5ð 	"Ø×!Ò!Ð"2Ñ3Ô3Ð3Ð3ÝÐ(­/Ñ:Ô:ð 	"Ø×!Ò!ÝÔ2×IÒIØ$ØØØ'Ø#1Ø"/ð Jñ ô ñ	ô 	ð 	ð 	õ Ð&Ñ'Ô'ð 	"Ø×!Ò!Õ"3Ð4DÑ"EÔ"EÑFÔFÐFÐFð>Ð1Að >ð >ð >ð õ ˜S‘/”/Ð!àÐr9   ©r	  Ú	callbacksrÐ   Úmetadatarí   r   r	  r%  r   r&  údict[str, Any] | Noneústr | BaseMessagec             ƒ  óN  K  — |�” ||¦  «        }t          |t          ¦  «        s.t          |t          ¦  «        rJt          d„ |D ¦   «         ¦  «        r1|                      |t          ||pg |pi ¬¦  «        ¬¦  «        ƒ d{V —†S d|› d�}t          |¦  «        ‚	 t          |¦  «        }|                      |t          ||pg |pi ¬¦  «        ¬¦  «        ƒ d{V —†}	nJ# t          $ r= t          |¦  «        }
 | j        di |
¤dt          ||pg |pi ¬¦  «        i¤Žƒ d{V —†}	Y nw xY w|	S )	a#  Asynchronously run the language model.

    Args:
        llm: The language model to run.
        inputs: The input dictionary.
        tags: Optional tags to add to the run.
        callbacks: Optional callbacks to use during the run.
        input_mapper: Optional function to map inputs to the expected format.
        metadata: Optional metadata to add to the run.

    Returns:
        The LLMResult or ChatResult.

    Raises:
        ValueError: If the LLM type is unsupported.
        InputFormatError: If the input format is invalid.
    Nc              3  ó@   K  — | ]}t          |t          ¦  «        V — Œd S r“   rÔ   rÕ   s     r:   r¹   z_arun_llm.<locals>.<genexpr>×  ó,   è è € ÐOÐO°S•J˜s¥KÑ0Ô0ÐOÐOÐOÐOÐOÐOr9   ©r%  r	  r&  r   z%Input mapper returned invalid format ú3
Expected a single string or list of chat messages.r   r8   )	rm   r…   r¼   rJ   Úainvoker   r3   rÂ   rÎ   )rí   r¯   r	  r%  rÐ   r&  Úprompt_or_messagesrt   r³   Ú
llm_outputÚ
llm_inputss              r:   Ú	_arun_llmr2  ¹  s  è è € ð4 ÐØ)˜\¨&Ñ1Ô1ÐÝÐ(­#Ñ.Ô.ð 	ÝÐ)­4Ñ0Ô0ð	åÐOÐOÐ<NÐOÑOÔOÑOÔOð	ð ŸšØ"Ý%Ø'Ø˜ Ø%˜^¨ðñ ô ð %ñ ô ð ð ð ð ð ð ð ðCØ"ðCð Cð Cð 	õ
 ˜sÑ#Ô#Ð#ð
Ý˜VÑ$Ô$ˆØ.1¯kªkØÝ!Ø#Ø�Z˜RØ!˜ Rðñ ô ð /:ñ /
ô /
ð )
ð )
ð )
ð )
ð )
ð )
ˆ
ˆ
øõ ð 	
ð 	
ð 	
Ý" 6Ñ*Ô*ˆ
Ø&˜3œ;ð 
ð 
Øð
ð 
å!Ø#Ø�Z˜RØ!˜ Rðñ ô ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ˆ
ˆ
ˆ
ð	
øøøð Ðs   ÂA C ÃAD"Ä!D"©r	  rÐ   r&  úChain | Runnableú
dict | strc             ƒ  óÖ  K  — |€|n
 ||¦  «        }t          | t          ¦  «        r�t          |t          ¦  «        rzt          |¦  «        dk    rg| j        r`t          t          |                     ¦   «         ¦  «        ¦  «        }|                      |t          ||pg |pi ¬¦  «        ¬¦  «        ƒ d{V —†}n3t          |pg ||pi ¬¦  «        }	|                      ||	¬¦  «        ƒ d{V —†}|S )z%Run a chain asynchronously on inputs.NrL   r,  r   ©r	  r%  r&  )
rm   r*   rn   r½   rÝ   r¾   r¿   rÀ   r.  r   ©
r”   r¯   r%  r	  rÐ   r&  Úinputs_ÚvalrE   Úrunnable_configs
             r:   Ú_arun_chainr<  ÿ  s8  è è € ð %Ð,ˆfˆf°,°,¸vÑ2FÔ2F€Gå�5�%Ñ Ô ðFå�w¥Ñ%Ô%ðFõ �‰LŒL˜AÒÐØÔð õ •4˜ŸšÑ(Ô(Ñ)Ô)Ñ*Ô*ˆØ—}’}ØÝ!Ø#Ø�Z˜RØ!˜ Rðñ ô ð %ñ 
ô 
ð 
ð 
ð 
ð 
ð 
ð 
ˆˆõ )Ø�˜ØØ�^ ð
ñ 
ô 
ˆð
 —}’} W°_�}ÑEÔEÐEÐEÐEÐEÐEÐEˆØ€Mr9   )rÐ   r   ú#dict | str | LLMResult | ChatResultc          
   ƒ  ó*  K  — t          |t          ¦  «        rdnd}d}	 t          |t          ¦  «        rBt          || j        pi |d         |d         ||                     d¦  «        ¬¦  «        ƒ d{V —†}nK |¦   «         }t          || j        pi |d         |d         ||                     d¦  «        ¬¦  «        ƒ d{V —†}|}nO# t          $ rB}t                               d|| j	        | j        |¦  «         t          |¬	¦  «        }Y d}~nd}~ww xY w|S )
a[  Asynchronously run the Chain or language model.

    Args:
        example: The example to run.
        config: The configuration for the run.
        llm_or_chain_factory: The Chain or language model constructor to run.
        input_mapper: Optional function to map the input to the expected format.

    Returns:
        A list of outputs.
    ÚLLMr*   Nr	  r%  r&  r$  z*%s failed for example %s with inputs %s
%s©rd   )rm   r   r2  r¯   rl   r<  Ú	Exceptionr§   rû   Úidr|   )	rã   r   r�   rÐ   Úchain_or_llmrx   rE   r”   rs   s	            r:   Ú_arun_llm_or_chainrD  #  s‹  è è € õ& Ð0Õ2CÑDÔDÐQˆˆÈ'ð ð €Fð$ÝÐ*Õ,=Ñ>Ô>ð 	Ý )Ø$Ø”Ð$ "Ø˜F”^Ø  Ô-Ø)ØŸš JÑ/Ô/ð!ñ !ô !ð ð ð ð ð ð ˆFˆFð )Ð(Ñ*Ô*ˆEÝ&ØØ”Ð$ "Ø˜F”^Ø  Ô-Ø)ØŸš JÑ/Ô/ðñ ô ð ð ð ð ð ð ˆFð ˆˆøÝð $ð $ð $Ý�ŠØ9ØØŒJØŒNØñ	
ô 	
ð 	
õ  Ð#Ñ#Ô#ˆˆˆˆˆˆøøøøð$øøøð €Ms   ŸB$C Ã
DÃ8DÄDc          
     ó"  — |�� ||¦  «        }t          |t          ¦  «        s.t          |t          ¦  «        rEt          d„ |D ¦   «         ¦  «        r,|                      |t          ||pg |pi ¬¦  «        ¬¦  «        }n’d|› d�}t          |¦  «        ‚	 t          |¦  «        }	|                      |	t          ||pg |pi ¬¦  «        ¬¦  «        }nA# t          $ r4 t          |¦  «        }
 | j        d	i |
¤dt          ||pi ¬¦  «        i¤Ž}Y nw xY w|S )
a  Run the language model on the example.

    Args:
        llm: The language model to run.
        inputs: The input dictionary.
        callbacks: The callbacks to use during the run.
        tags: Optional tags to add to the run.
        input_mapper: function to map to the inputs dictionary from an Example
        metadata: Optional metadata to add to the run.

    Returns:
        The LLMResult or ChatResult.

    Raises:
        ValueError: If the LLM type is unsupported.
        InputFormatError: If the input format is invalid.
    Nc              3  ó@   K  — | ]}t          |t          ¦  «        V — Œd S r“   rÔ   rÕ   s     r:   r¹   z_run_llm.<locals>.<genexpr>|  r+  r9   r,  r   z'Input mapper returned invalid format:  r-  r   )r%  r&  r8   )	rm   r…   r¼   rJ   Úinvoker   r3   rÂ   rÎ   )rí   r¯   r%  r	  rÐ   r&  r/  r0  rt   Úllm_promptsr1  s              r:   Ú_run_llmrI  ]  sž  € ð6 ÐØ)˜\¨&Ñ1Ô1ÐÝÐ(­#Ñ.Ô.ð 	(ÝÐ)­4Ñ0Ô0ð	(åÐOÐOÐ<NÐOÑOÔOÑOÔOð	(ð -0¯JªJØ"Ý%Ø'Ø˜ Ø%˜^¨ðñ ô ð -7ñ -ô -ˆJˆJðGØ&ðGð Gð Gð õ
 # 3Ñ'Ô'Ð'ð	Ý% fÑ-Ô-ˆKØŸšØÝ%Ø'Ø˜ Ø%˜^¨ðñ ô ð $ñ ô ˆJˆJøõ  ð 	ð 	ð 	Ý& vÑ.Ô.ˆJØ#˜œð ð Øðð å%°	ÀHÀNÐPRÐSÑSÔSðð ð ˆJˆJˆJð	øøøð Ðs   Â:C Ã;DÄDc               óº  — |€|n
 ||¦  «        }t          | t          ¦  «        r‰t          |t          ¦  «        rtt          |¦  «        dk    ra| j        rZt          t          |                     ¦   «         ¦  «        ¦  «        }|                      |t          ||pg |pi ¬¦  «        ¬¦  «        }n-t          |pg ||pi ¬¦  «        }	|                      ||	¬¦  «        }|S )zRun a chain on inputs.NrL   r,  r   r7  )
rm   r*   rn   r½   rÝ   r¾   r¿   rÀ   rG  r   r8  s
             r:   Ú
_run_chainrK  ¡  s  € ð %Ð,ˆfˆf°,°,¸vÑ2FÔ2F€Gå�5�%Ñ Ô ð?å�w¥Ñ%Ô%ð?õ �‰LŒL˜AÒÐØÔð õ •4˜ŸšÑ(Ô(Ñ)Ô)Ñ*Ô*ˆØ—’ØÝ!Ø#Ø�Z˜RØ!˜ Rðñ ô ð ñ 
ô 
ˆˆõ )Ø�˜ØØ�^ ð
ñ 
ô 
ˆð
 —’˜g¨o�Ñ>Ô>ˆØ€Mr9   c          
     ó8  — t          |t          ¦  «        rdnd}d}	 t          |t          ¦  «        r<t          || j        pi |d         |d         ||                     d¦  «        ¬¦  «        }nE |¦   «         }t          || j        pi |d         |d         ||                     d¦  «        ¬¦  «        }|}nd# t          $ rW}t          |¦  «        j        }	t           
                    d|| j        | j        |	|¦  «         t          |¬	¦  «        }Y d}~nd}~ww xY w|S )
aj  Run the Chain or language model synchronously.

    Args:
        example: The example to run.
        config: The configuration for the run.
        llm_or_chain_factory: The Chain or language model constructor to run.
        input_mapper: Optional function to map the input to the expected format.

    Returns:
        The outputs of the model or chain.
    r?  r*   Nr%  r	  r&  r3  zC%s failed for example %s with inputs %s
Error Type: %s, Message: %sr@  )rm   r   rI  r¯   rl   rK  rA  r»   r4   r§   rû   rB  r|   )
rã   r   r�   rÐ   rC  rx   rE   r”   rs   Ú
error_types
             r:   Ú_run_llm_or_chainrN  Å  s\  € õ& Ð0Õ2CÑDÔDÐQˆˆÈ'ð ð €Fð$ÝÐ*Õ,=Ñ>Ô>ð 	Ý"Ø$Ø”Ð$ "Ø�{Ô#Ø˜F”^Ø)ØŸš JÑ/Ô/ðñ ô ˆFˆFð )Ð(Ñ*Ô*ˆEÝØØ”Ð$ "Ø�{Ô#Ø˜F”^Ø)ØŸš JÑ/Ô/ðñ ô ˆFð ˆˆøÝð 
$ð 
$ð 
$Ý˜!‘W”WÔ%ˆ
Ý�ŠØRØØŒJØŒNØØñ	
ô 	
ð 	
õ  Ð#Ñ#Ô#ˆˆˆˆˆˆøøøøð
$øøøð €Ms   �BB6 Â6
DÃ ADÄDÚclientr   Úproject_nameÚproject_metadataÚdataset_versionústr | datetime | Noneú1tuple[MCF, TracerSession, Dataset, list[Example]]c           
     ó  — t          ||¦  «        }|                      |¬¦  «        }t          |                      |j        |¬¦  «        ¦  «        }	|	sd|› d�}
t          |
¦  «        ‚d„ |	D ¦   «         }|rt          |¦  «        nd }|r|                     ¦   «         nd }	 |pi }t          ¦   «         }|ri |¥d|i¥}||d<   |  	                    ||j        |rd|ini |¬	¦  «        }nh# t          t
          t          f$ rN}d
t          |¦  «        vr‚ t          j        ¦   «         }d|› d|› d|› d�}d|› d|› �}
t          |
¦  «        |‚d }~ww xY w|j        d|j        › �z   }t!          d|› d|› d|› d|j        › �d¬¦  «         ||||	fS )N)r�   )Ú
dataset_idÚas_ofzDataset z has no example rows.c                ó*   — g | ]}|j         ¯	|j         ‘ŒS r8   )Úmodified_at)rG   Úexs     r:   rI   z%_prepare_eval_run.<locals>.<listcomp>  s!   € ÐGÐGÐG b¸¼ÐG�2”>ÐGÐGÐGr9   ÚgitrR  r	  )Úreference_dataset_idÚproject_extrar&  zalready exists z+
run_on_dataset(
    ...
    project_name="z - z", # Update since z already exists
)
zTest project z/ already exists. Please use a different name:

z/compare?selectedSessions=z)View the evaluation results for project 'z' at:
z

View all tests for Dataset z at:
T)Úflush)r®   Úread_datasetr¼   Úlist_examplesrB  r¢   ÚmaxÚ	isoformatr   Úcreate_projectr(   r'   r…   ÚuuidÚuuid4ÚurlÚprint)rO  r�   r�   rP  rQ  r	  rR  Úwrapped_modelÚdatasetrç   rt   rY  Úmax_modified_atÚinferred_versionÚgit_infoÚprojectrs   ÚuidÚexample_msgÚcomparison_urls                       r:   Ú_prepare_eval_runrq  þ  sz  € õ +Ð+?ÀÑNÔN€MØ×!Ò!¨|Ð!Ñ<Ô<€Gå�F×(Ò(°G´JÀoÐ(ÑVÔVÑWÔW€HØð Ø<˜Ð<Ð<Ð<ˆÝ˜‰oŒoÐØGÐG¨HÐGÑGÔG€Kð +6Ð?•c˜+Ñ&Ô&Ð&¸4€OØ6EÐO�×0Ò0Ñ2Ô2Ð2È4Ðð%Ø+Ð1¨rÐÝ‘>”>ˆØð 	ð Ø"ð à�xð ð  Ðð
 /?ÐÐ*Ñ+Ø×'Ò'ØØ!(¤Ø,0Ð8˜6 4˜.˜.°bØ%ð	 (ñ 
ô 
ˆˆøõ •z¥>Ð2ð %ð %ð %Ø¥C¨¡F¤FÐ*Ð*ØÝŒj‰lŒlˆðð  ðð ð %(ðð ð <Hðð ð ˆð!˜Lð !ð !Øð!ð !ð 	õ ˜‰oŒo 1Ð$øøøøð%øøøð ”[Ð#LÀÄ
Ð#LÐ#LÑL€NÝ	ð	H°Lð 	Hð 	HØð	Hð 	Hà&2ð	Hð 	Hà:A¼+ð	Hð 	Hð ð	ñ ô ð ð ˜' 7¨HÐ4Ð4s   ÂAC% Ã%E
Ã<A	EÅE
c                  ó2   — e Zd ZU dZded<   ded<   ded<   dS )	Ú
_RowResultz5A dictionary of the results for a single example row.zlist[EvaluationResult] | NonerX   zfloat | Nonere   rø   rf   NrÆ   r8   r9   r:   rs  rs  =  s<   € € € € € € Ø?Ð?à+Ð+Ð+Ñ+Ø Ð Ð Ñ ØÐÐÑÐÐr9   rs  F)Útotalc                  ó¨   — e Zd ZU dZded<   ded<   ded<   ded	<   d
ed<   dZded<   d8d„Zd9d„Zd:d„Zd;d„Z		 d<d=d"„Z
e	 	 	 	 	 	 	 d>d?d7„¦   «         ZdS )@Ú_DatasetRunContainerz3A container to help manage the state of a eval run.r   rO  r&   rm  r�   rh  rè   rç   zlist[RunnableConfig]ÚconfigsNz3list[smith_eval_config.BATCH_EVALUATOR_LIKE] | NoneÚbatch_evaluatorsÚbatch_resultsr¼   Úall_eval_resultsúdict[str, _RowResult]r=   rn   c                óJ  — i }t          | j        |d¬¦  «        D �]\  }}|                     t          |j        ¦  «        i ¦  «        }|j        |                     dg ¦  «        |                     d¦  «        |                     d¦  «        dœ|t          |j        ¦  «        <   t          |t          ¦  «        r#|j        |t          |j        ¦  «                 d<   n||t          |j        ¦  «                 d<   |j	        r"|j	        |t          |j        ¦  «                 d	<   �Œ	|S )
NF©ÚstrictrX   re   rf   )rD   rX   re   rf   rd   rE   rC   )
Úziprç   rl   r…   rB  r¯   rm   r|   rd   rð   )rR   ry  rz  rW   rã   rE   Ú
row_results          r:   Ú_merge_test_outputsz(_DatasetRunContainer._merge_test_outputsP  s  € ð
 ˆÝ" 4¤=°-ÈÐNÑNÔNð 	Hñ 	H‰OˆG�VØ)×-Ò-­c°'´*©o¬o¸rÑBÔBˆJà œØ&ŸNšN¨:°rÑ:Ô:Ø",§.¢.Ð1AÑ"BÔ"BØ$Ÿ.š.¨Ñ2Ô2ð	(ð (ˆG•C˜œ
‘O”OÑ$õ ˜&¥)Ñ,Ô,ð <Ø4:´L��˜GœJ™œÔ(¨Ñ1Ð1à5;��˜GœJ™œÔ(¨Ñ2ØŒð HØ8?¼��˜GœJ™œÔ(¨Ñ5ùØˆr9   Úrunsúdict[str, Run]ú
list[dict]c           	     ó:  ‡— | j         }|sg S ˆfd„| j        D ¦   «         }g }t          j                             ¦   «         5 }|D ]¿}	  ||| j        ¦  «        }t          |t          ¦  «        r|                     ¦   «         }|                     t          d|¦  «        ¦  «          |j
        | j        j        fi |¤d | j        j        dœ¤Ž Œˆ# t          $ r+ t                                dt%          |¦  «        ¦  «         Y Œ¼w xY w	 d d d ¦  «         n# 1 swxY w Y   |S )Nc                óD   •— g | ]}‰t          |j        ¦  «                 ‘ŒS r8   )r…   rB  )rG   rã   r‚  s     €r:   rI   z>_DatasetRunContainer._run_batch_evaluators.<locals>.<listcomp>j  s&   ø€ ÐHÐHÐH¨w�T�#˜gœj™/œ/Ô*ÐHÐHÐHr9   rn   )rf   Ú
project_idz Error running batch evaluator %s)rx  rç   Ú
concurrentÚfuturesÚThreadPoolExecutorrm   r   Ú
model_dumprp   r	   ÚsubmitrO  Úcreate_feedbackrm  rB  rA  r§   Ú	exceptionÚrepr)rR   r‚  r   Ú	runs_listÚaggregate_feedbackÚexecutorÚ	evaluatorrx   s    `      r:   Ú_run_batch_evaluatorsz*_DatasetRunContainer._run_batch_evaluatorsf  s«  ø€ ØÔ*ˆ
Øð 	ØˆIØHÐHÐHÐH¸$¼-ÐHÑHÔHˆ	ØÐÝÔ×2Ò2Ñ4Ô4ð 	¸Ø'ð ð �	ðØ&˜Y y°$´-Ñ@Ô@�FÝ! &Õ*:Ñ;Ô;ð 5Ø!'×!2Ò!2Ñ!4Ô!4˜Ø&×-Ò-­d°6¸6Ñ.BÔ.BÑCÔCÐCØ#�H”OØœÔ3ðð à ðð  $Ø#'¤<¤?ð	ð ð ð ð ð øõ !ð ð ð Ý×$Ò$Ø:½DÀ¹O¼Oñô ð ð ð ðøøøðð	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	ð" "Ð!s7   Á DÁBCÃ
DÃ2D Ã=DÃ?D Ä DÄDÄDú,tuple[dict[str, _RowResult], dict[str, Run]]c                óÌ  — i }i }| j         D �]F}t          d|d         ¦  «        D �]+}t          |t          ¦  «        r]|j        }|                     ¦   «         D ]@\  \  }}}|                     t          |¦  «        i ¦  «                             d|i¦  «         ŒAŒut          |t          ¦  «        r¡|j
        }	|	r(|	j        r!|	j        |	j        z
                       ¦   «         nd }
|	rt          |	j        ¦  «        nd }|                     t          |j        ¦  «        i ¦  «                             |
||	dœ¦  «         |	|t          |j        ¦  «        <   �Œ-�ŒHt          d|¦  «        |fS )Nr¼   r%  rX   )re   rf   Úrunr{  )rw  r	   rm   r   Úlogged_eval_resultsrk   Ú
setdefaultr…   ro   r   Ú
latest_runÚend_timeÚ
start_timeÚtotal_secondsrB  rw   )rR   rz  Úall_runsÚcÚcallbackÚeval_resultsÚ_rw   r\   r—  re   rf   s               r:   Ú_collect_metricsz%_DatasetRunContainer._collect_metrics  sŸ  € Ø!#ÐØˆØ”ð 	=ñ 	=ˆAÝ  ¨¨;¬Ñ8Ô8ð =ñ =�Ý˜hÕ(@ÑAÔAð =Ø#+Ô#?�LØ.:×.@Ò.@Ñ.BÔ.Bð ð Ñ*™˜˜J¨Ø(×3Ò3µC¸
±O´OÀRÑHÔH×OÒOØ'¨˜Oñô ð ð ðõ   ­/Ñ:Ô:ð =Ø"Ô-�Cð ð"Ø#&¤<ð"˜œ¨¬Ñ6×EÒEÑGÔGÐGà!ð #ð
 -0Ð9�S ¤™[œ[˜[°T�FØ$×/Ò/µ°HÔ4GÑ0HÔ0HÈ"ÑMÔM×TÒTà.<Ø&,Ø#&ðð ñô ð ð :=�H�S Ô!4Ñ5Ô5Ñ6ùñ-=õ. Ð+Ð-=Ñ>Ô>ÀÐHÐHr9   ú)list[dict | str | LLMResult | ChatResult]r<   c                óT  — t                                d¦  «         t          ¦   «          |                      ¦   «         \  }}d }| j        r/t                                d¦  «         |                      |¦  «        }|                      ||¦  «        }t          | j        j	        ||¬¦  «        S )Nz#Waiting for evaluators to complete.zRunning session evaluators.)rP  rW   Úaggregate_metrics)
r§   r¨   r   r£  rx  r”  r�  r<   rm  r„   )rR   ry  rz  rž  r‘  rW   s         r:   Ú_collect_test_resultsz*_DatasetRunContainer._collect_test_resultsœ  s®   € õ 	�ŠÐ9Ñ:Ô:Ð:ÝÑ!Ô!Ð!Ø%)×%:Ò%:Ñ%<Ô%<Ñ"Ð˜(Ø!ÐØÔ ð 	FÝ�KŠKÐ5Ñ6Ô6Ð6Ø!%×!;Ò!;¸HÑ!EÔ!EÐØ×*Ò*¨=Ð:JÑKÔKˆÝØœÔ*ØØ0ð
ñ 
ô 
ð 	
r9   FÚverboseÚboolc                óÖ  — |                       |¦  «        }|rY	 |                     ¦   «         }t          |¦  «         n4# t          $ r'}t                               d|d¬¦  «         Y d }~nd }~ww xY w	 | j                             | j        j	        t          j        t          j        ¦  «        ¬¦  «         n4# t          $ r'}t                               d|d¬¦  «         Y d }~nd }~ww xY w|S )Nz&Failed to print aggregate feedback: %sT)Úexc_info)r›  zFailed to close project: %s)r§  rU   Ú_display_aggregate_resultsrA  r§   rå   rO  Úupdate_projectrm  rB  r   Únowr   Úutc)rR   ry  r¨  rW   Úagg_feedbackrs   s         r:   Úfinishz_DatasetRunContainer.finish®  s&  € ð
 ×,Ò,¨]Ñ;Ô;ˆØð 	YðYØ&×=Ò=Ñ?Ô?�Ý*¨<Ñ8Ô8Ð8Ð8øÝð Yð Yð YÝ—’ÐEÀqÐSW�ÑXÔXÐXÐXÐXÐXÐXÐXøøøøðYøøøð	JàŒK×&Ò&Ø””Ý!œ¥h¤lÑ3Ô3ð 'ñ ô ð ð øõ ð 	Jð 	Jð 	JÝ�LŠLÐ6¸ÀDˆLÑIÔIÐIÐIÐIÐIÐIÐIøøøøð	Jøøøàˆs.   ™#= ½
A.ÁA)Á)A.Á2AB5 Â5
C&Â?C!Ã!C&é   r�   r…   r�   rŽ   rP  rø   r/   ré   r	  r÷   rÐ   rÑ   Úconcurrency_levelÚintrQ  r'  Úrevision_idrR  údatetime | str | Nonec           	     ó¬  ‡‡‡‡‡‡‡— |pt          j        ¦   «         }|
r|	si }	|	                     d|
i¦  «         t          ‰||||	‰|¬¦  «        \  }Š}}‰pg Š‰j                             d¦  «        pi                      ¦   «         D ] \  }}‰                     d|› d|› �¦  «         Œ!d‰j        d         iŠ|
r|
‰d<   t          |¦  «        }t          ||||j
        pt          j        ¦  «        Št          |d         ||¦  «         t          j        t!          |¦  «        ¦  «        Šˆˆˆˆˆˆˆfd„|D ¦   «         } | ‰‰||||r|j        nd ¬	¦  «        S )
Nrµ  )rQ  r	  rR  r[  zgit:ú=rR  r   c                ó    •— g | ]J}t          t          ‰j        ‰|j        ¬ ¦  «        t	          ‰pg ‰|j        d¬¦  «        ‰g‰‰‰¬¦  «        ‘ŒKS ))rP  rO  rw   r   )r   rO  rw   Úmax_concurrency)r%  r	  rº  r&  )r   r   r„   rB  r   )	rG   rã   rO  r³  Úprogress_barrm  rô   Úrun_metadatar	  s	     €€€€€€€r:   rI   z0_DatasetRunContainer.prepare.<locals>.<listcomp>ð  s•   ø€ ð 
ð 
ð 
ð( õ' å#Ø%,¤\Ø%Ø#*¤:ðñ ô õ
 -Ø#1Ð#7°RØ%Ø#*¤:Ø()ð	ñ ô ð !ðð Ø 1Ø%ð#ñ ô ð
ð 
ð 
r9   )rO  rm  rh  rç   rw  rx  )r0   Úrandom_namero   rq  r&  rl   rk   rp   r®   rõ   rê   r#   Úkvræ   r1   ÚProgressBarCallbackr½   rx  )ÚclsrO  r�   r�   rP  r/   r	  rÐ   r³  rQ  rµ  rR  rh  ri  rç   r[   r\   rw  r»  rm  rô   r¼  s    `    ` `         @@@@r:   Úpreparez_DatasetRunContainer.prepareÄ  sé  øøøøøøø€ ð $ÐD¥Ô'BÑ'DÔ'DˆØð 	BØ#ð &Ø#%Ð Ø×#Ò# ]°KÐ$@ÑAÔAÐAÝ4EØØØ ØØ-ØØ+ð5
ñ 5
ô 5
Ñ1ˆ�w ¨ð ˆz�rˆØÔ%×)Ò)¨%Ñ0Ô0Ð6°B×=Ò=Ñ?Ô?ð 	(ð 	(‰DˆAˆqØ�KŠK˜˜q˜˜ 1˜˜Ñ'Ô'Ð'Ð'Ø)¨7Ô+;Ð<MÔ+NÐOˆØð 	6Ø*5ˆL˜Ñ'Ý.Ð/CÑDÔDˆÝ*ØØØØÔÐ,¥¤ñ	
ô 
ˆõ 	! ¨!¤¨m¸\ÑJÔJÐJÝÔ3µC¸±M´MÑBÔBˆð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð( $ð)
ñ 
ô 
ˆð, ˆsØØØ'ØØØ<FÐP˜ZÔ8Ð8ÈDð
ñ 
ô 
ð 	
r9   )ry  r¼   rz  r{  r=   rn   )r‚  rƒ  r=   r„  )r=   r•  )ry  r¤  r=   r<   )F)ry  r¼   r¨  r©  r=   r<   )NNNr²  NNN)rO  r   r�   r…   r�   rŽ   rP  rø   r/   ré   r	  r÷   rÐ   rÑ   r³  r´  rQ  r'  rµ  rø   rR  r¶  r=   rv  )r4   r5   r6   r7   rÇ   rx  r�  r”  r£  r§  r±  ÚclassmethodrÁ  r8   r9   r:   rv  rv  E  s  € € € € € € à=Ð=à€N€N�NØÐÐÑØÐÐÑØÐÐÑØ!Ð!Ð!Ñ!ØLPÐÐPÐPÐPÑPðð ð ð ð,"ð "ð "ð "ð2Ið Ið Ið Ið:
ð 
ð 
ð 
ð* ðð ð ð ð ð, ð 7;Ø!%Ø59Ø!"Ø26Ø"&Ø15ðH
ð H
ð H
ð H
ñ „[ðH
ð H
ð H
r9   rv  r©  c                 óŠ   — 	 ddl m}   | ¦   «         }|d uodt          t          |¦  «        ¦  «        v S # t          $ r Y dS w xY w)Nr   )Úget_ipythonÚzmqshellF)ÚIPython.core.getipythonrÄ  r…   r»   rj   )rÄ  Úress     r:   Ú_is_jupyter_environmentrÈ    sf   € ðØ7Ð7Ð7Ð7Ð7Ð7àˆk‰mŒmˆØ˜$ˆÐ? :µµT¸#±Y´Y±´Ð#?Ð?øÝð ð ð Øˆuˆuðøøøs   ‚14 ´
AÁAÚaggregate_resultsr>   c                óà   — t          ¦   «         r)ddlm}m}  | |d¦  «        ¦  «          || ¦  «         d S |                      d„ d¬¦  «        }t          d¦  «         t          |¦  «         d S )Nr   )ÚHTMLÚdisplayz<h3>Experiment Results:</h3>c                ó
   — | d›S )Nz.2fr8   )Úxs    r:   r•   z,_display_aggregate_results.<locals>.<lambda>"  s
   €  a : :€ r9   Úright)Úfloat_formatÚjustifyz
 Experiment Results:)rÈ  ÚIPython.displayrË  rÌ  Ú	to_stringrg  )rÉ  rË  rÌ  Úformatted_strings       r:   r¬  r¬    s¤   € ÝÑ Ô ð  Ø1Ð1Ð1Ð1Ð1Ð1Ð1Ð1àˆ��Ð3Ñ4Ô4Ñ5Ô5Ð5ØˆÐ!Ñ"Ô"Ð"Ð"Ð"à,×6Ò6Ø-Ð-Øð 7ñ 
ô 
Ðõ 	Ð&Ñ'Ô'Ð'ÝÐÑÔÐÐÐr9   aè  The input_mapper argument is deprecated and will be removed in a future release. Please add a  RunnableLambda to your chain to map inputs to the expected format instead. Example:
def construct_chain():
    my_chain = ...
    input_mapper = {'other_key': 'MyOtherInput', 'my_input_key': x}
    return input_mapper | my_chain
run_on_dataset(..., llm_or_chain_factory=construct_chain)
(See https://api.python.langchain.com/en/latest/schema/langchain.schema.runnable.base.RunnableLambda.html)r²  )r/   rR  r³  rP  rQ  r¨  rµ  úClient | Noner¶  r³  r´  r¨  rµ  r~   r   c             ‹  óÆ  K  — |
                      dd¦  «        }|rt          dt          d¬¦  «         |	€!t          ¦   «                              d¦  «        }	|
                      dd¦  «        }|rt          dd	d¬¦  «         |
r(t          dd
|
                     ¦   «         › d�d¬¦  «         | pt          ¦   «         } t                               | |||||||||	|¬¦  «        }t          j
        |j        d                              d¦  «        gt          t          j        t          |j        |¬¦  «        |j        |j        ¦  «        ¢R Ž ƒ d{V —†}|                     ||¬¦  «        S )a  Run on dataset.

    Run the Chain or language model on a dataset and store traces
    to the specified project name.

    For the (usually faster) async version of this function,
    see `arun_on_dataset`.

    Args:
        dataset_name: Name of the dataset to run the chain on.
        llm_or_chain_factory: Language model or Chain constructor to run
            over the dataset. The Chain constructor is used to permit
            independent calls on each example without carrying over state.
        evaluation: Configuration for evaluators to run on the
            results of the chain.
        dataset_version: Optional version of the dataset.
        concurrency_level: The number of async tasks to run concurrently.
        project_name: Name of the project to store the traces in.
            Defaults to `{dataset_name}-{chain class name}-{datetime}`.
        project_metadata: Optional metadata to add to the project.
            Useful for storing information the test variant.
            (prompt version, model version, etc.)
        client: LangSmith client to use to access the dataset and to
            log feedback and run traces.
        verbose: Whether to print progress.
        revision_id: Optional revision identifier to assign this test run to
            track the performance of different versions of your system.
        **kwargs: Should not be used, but is provided for backwards compatibility.

    Returns:
        `dict` containing the run's project name and the resulting model outputs.

    Examples:
    ```python
    from langsmith import Client
    from langchain_openai import ChatOpenAI
    from langchain_classic.chains import LLMChain
    from langchain_classic.smith import smith_eval.RunEvalConfig, run_on_dataset

    # Chains may have memory. Passing in a constructor function lets the
    # evaluation framework avoid cross-contamination between runs.
    def construct_chain():
        model = ChatOpenAI(temperature=0)
        chain = LLMChain.from_string(
            model,
            "What's the answer to {your_input_key}"
        )
        return chain

    # Load off-the-shelf evaluators via config or the EvaluatorType (string or enum)
    evaluation_config = smith_eval.RunEvalConfig(
        evaluators=[
            "qa",  # "Correctness" against a reference answer
            "embedding_distance",
            smith_eval.RunEvalConfig.Criteria("helpfulness"),
            smith_eval.RunEvalConfig.Criteria({
                "fifth-grader-score": "Do you have to be smarter than a fifth "
                "grader to answer this question?"
            }),
        ]
    )

    client = Client()
    await arun_on_dataset(
        client,
        dataset_name="<my_dataset_name>",
        llm_or_chain_factory=construct_chain,
        evaluation=evaluation_config,
    )
    ```
    You can also create custom evaluators by subclassing the `StringEvaluator or
    LangSmith's `RunEvaluator` classes.

    ```python
    from typing import Optional
    from langchain_classic.evaluation import StringEvaluator


    class MyStringEvaluator(StringEvaluator):
        @property
        def requires_input(self) -> bool:
            return False

        @property
        def requires_reference(self) -> bool:
            return True

        @property
        def evaluation_name(self) -> str:
            return "exact_match"

        def _evaluate_strings(
            self, prediction, reference=None, input=None, **kwargs
        ) -> dict:
            return {"score": prediction == reference}


    evaluation_config = smith_eval.RunEvalConfig(
        custom_evaluators=[MyStringEvaluator()],
    )

    await arun_on_dataset(
        client,
        dataset_name="<my_dataset_name>",
        llm_or_chain_factory=construct_chain,
        evaluation=evaluation_config,
    )
    ```
    rÐ   Nú0.0.305T©ÚmessageÚpendingrµ  r	  ú0.1.9úqThe tags argument is deprecated and will be removed in a future release. Please specify project_metadata instead.úPThe following arguments are deprecated and will be removed in a future release: r×   ©rÙ  Úremoval©rQ  rµ  rR  r   rº  ©r�   rÐ   ©r¨  )rË   r
   Ú_INPUT_MAPPER_DEP_WARNINGr   rl   rß   r   rv  rÁ  Úrunnable_utilsÚgather_with_concurrencyrw  ÚmapÚ	functoolsÚpartialrD  rh  rç   r±  )rO  r�   r�   r/   rR  r³  rP  rQ  r¨  rµ  r~   rÐ   r	  Ú	containerry  s                  r:   Úarun_on_datasetrê  :  sØ  è è € ðv —:’:˜n¨dÑ3Ô3€LØð TÝ˜	Õ+DÈdÐSÑSÔSÐSØÐÝ4Ñ6Ô6×:Ò:¸=ÑIÔIˆØ�:Š:�f˜dÑ#Ô#€DØð 
ÝØðUàð		
ñ 	
ô 	
ð 	
ð ð 
ÝØð à�{Š{‰}Œ}ð ð  ð  ð ð	
ñ 	
ô 	
ð 	
ð Ð•v‘x”x€FÝ$×,Ò,ØØØØØØØØØ)ØØ'ð -ñ ô €Iõ )Ô@ØÔ˜!Ô× Ò Ð!2Ñ3Ô3ðå	ÝÔÝ"Ø%.Ô%<Ø)ðñ ô ð
 ÔØÔñ

ô 

ðð ð ð ð ð ð ð ð €Mð ×Ò˜M°7ÐÑ;Ô;Ð;r9   c               óR  ‡‡— |
                      dd¦  «        Š‰rt          dt          d¬¦  «         |
                      dd¦  «        }|rt          ddd¬¦  «         |	€!t          ¦   «                              d	¦  «        }	|
r(t          dd
|
                     ¦   «         › d�d¬¦  «         | pt          ¦   «         } t                               | |||||‰|||	|¬¦  «        Š|dk    r*ˆˆfd„t          ‰j
        ‰j        d¬¦  «        D ¦   «         }n„t          j        ‰j        d         ¦  «        5 }t          |                     t!          j        t$          ‰j        ‰¬¦  «        ‰j
        ‰j        ¦  «        ¦  «        }ddd¦  «         n# 1 swxY w Y   ‰                     ||¬¦  «        S )a  Run on dataset.

    Run the Chain or language model on a dataset and store traces
    to the specified project name.

    For the (usually faster) async version of this function,
    see `arun_on_dataset`.

    Args:
        dataset_name: Name of the dataset to run the chain on.
        llm_or_chain_factory: Language model or Chain constructor to run
            over the dataset. The Chain constructor is used to permit
            independent calls on each example without carrying over state.
        evaluation: Configuration for evaluators to run on the
            results of the chain.
        dataset_version: Optional version of the dataset.
        concurrency_level: The number of async tasks to run concurrently.
        project_name: Name of the project to store the traces in.
            Defaults to `{dataset_name}-{chain class name}-{datetime}`.
        project_metadata: Optional metadata to add to the project.
            Useful for storing information the test variant.
            (prompt version, model version, etc.)
        client: LangSmith client to use to access the dataset and to
            log feedback and run traces.
        verbose: Whether to print progress.
        revision_id: Optional revision identifier to assign this test run to
            track the performance of different versions of your system.
        **kwargs: Should not be used, but is provided for backwards compatibility.

    Returns:
        `dict` containing the run's project name and the resulting model outputs.

    Examples:
    ```python
    from langsmith import Client
    from langchain_openai import ChatOpenAI
    from langchain_classic.chains import LLMChain
    from langchain_classic.smith import smith_eval.RunEvalConfig, run_on_dataset

    # Chains may have memory. Passing in a constructor function lets the
    # evaluation framework avoid cross-contamination between runs.
    def construct_chain():
        model = ChatOpenAI(temperature=0)
        chain = LLMChain.from_string(
            model,
            "What's the answer to {your_input_key}"
        )
        return chain

    # Load off-the-shelf evaluators via config or the EvaluatorType (string or enum)
    evaluation_config = smith_eval.RunEvalConfig(
        evaluators=[
            "qa",  # "Correctness" against a reference answer
            "embedding_distance",
            smith_eval.RunEvalConfig.Criteria("helpfulness"),
            smith_eval.RunEvalConfig.Criteria({
                "fifth-grader-score": "Do you have to be smarter than a fifth "
                "grader to answer this question?"
            }),
        ]
    )

    client = Client()
    run_on_dataset(
        client,
        dataset_name="<my_dataset_name>",
        llm_or_chain_factory=construct_chain,
        evaluation=evaluation_config,
    )
    ```

    You can also create custom evaluators by subclassing the `StringEvaluator` or
    LangSmith's `RunEvaluator` classes.

    ```python
    from typing import Optional
    from langchain_classic.evaluation import StringEvaluator


    class MyStringEvaluator(StringEvaluator):
        @property
        def requires_input(self) -> bool:
            return False

        @property
        def requires_reference(self) -> bool:
            return True

        @property
        def evaluation_name(self) -> str:
            return "exact_match"

        def _evaluate_strings(
            self, prediction, reference=None, input=None, **kwargs
        ) -> dict:
            return {"score": prediction == reference}


    evaluation_config = smith_eval.RunEvalConfig(
        custom_evaluators=[MyStringEvaluator()],
    )

    run_on_dataset(
        client,
        dataset_name="<my_dataset_name>",
        llm_or_chain_factory=construct_chain,
        evaluation=evaluation_config,
    )
    ```
    rÐ   Nr×  TrØ  r	  rÛ  rÜ  rµ  rÝ  r×   rÞ  rà  r   c                óF   •— g | ]\  }}t          ||‰j        ‰¬ ¦  «        ‘ŒS )rá  )rN  rh  )rG   rã   r   ré  rÐ   s      €€r:   rI   z"run_on_dataset.<locals>.<listcomp>‰  sM   ø€ ð 

ð 

ð 

ñ  �˜õ ØØØ%.Ô%<Ø)ð	ñ ô ð

ð 

ð 

r9   Fr}  rá  râ  )rË   r
   rã  r   rl   rß   r   rv  rÁ  r  rç   rw  r;  Úget_executor_for_configr¼   ræ  rç  rè  rN  rh  r±  )rO  r�   r�   r/   rR  r³  rP  rQ  r¨  rµ  r~   r	  ry  r’  ré  rÐ   s                 @@r:   Úrun_on_datasetrî  è  sZ  øø€ ðx —:’:˜n¨dÑ3Ô3€LØð TÝ˜	Õ+DÈdÐSÑSÔSÐSØ�:Š:�f˜dÑ#Ô#€DØð 
ÝØðUàð		
ñ 	
ô 	
ð 	
ð ÐÝ4Ñ6Ô6×:Ò:¸=ÑIÔIˆàð 
ÝØð à�{Š{‰}Œ}ð ð  ð  ð ð	
ñ 	
ô 	
ð 	
ð Ð•v‘x”x€FÝ$×,Ò,ØØØØØØØØØ)ØØ'ð -ñ ô €Ið ˜AÒÐð

ð 

ð 

ð 

ð 

õ $'ØÔ" IÔ$5¸eð$ñ $ô $ð

ñ 

ô 

ˆˆõ Ô4°YÔ5FÀqÔ5IÑJÔJð 	ÈhÝ Ø—’ÝÔ%Ý)Ø-6Ô-DØ%1ðñ ô ð
 Ô&ØÔ%ñô ñ
ô 
ˆMð	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	ð ×Ò˜M°7ÐÑ;Ô;Ð;s   Ä-AFÆFÆF)rŒ   )r�   rŽ   r�   r…   r=   r�   )r¯   r°   r=   r…   )r¯   r°   r=   rn   )rÏ   r$   rÐ   rÑ   r=   r   )rÏ   r$   r”   r*   rÐ   rÑ   r=   r   )rã   r$   r�   r�   rÐ   rÑ   r=   r   )
r�   r�   rç   rè   r/   ré   rê   r#   r=   rë   )r   rö   rñ   r÷   r=   rø   )r   rö   rò   r÷   r=   rø   )r   rö   r   r÷   r=   rø   )r  r  r  r  ró   r…   rê   r#   r   r÷   r  rø   rú   rø   rþ   rø   r=   r   )
r   rö   rñ   r÷   rò   r÷   r   r÷   r=   r  )r   rö   ró   r…   rê   r#   r   r÷   rñ   r÷   rò   r÷   r=   r  )rí   r   r¯   r°   r	  r÷   r%  r   rÐ   rÑ   r&  r'  r=   r(  )r”   r4  r¯   r°   r%  r   r	  r÷   rÐ   rÑ   r&  r'  r=   r5  )
rã   r$   r   r   r�   r�   rÐ   rÑ   r=   r=  )rí   r   r¯   r°   r%  r   r	  r÷   rÐ   rÑ   r&  r'  r=   r(  r  )rO  r   r�   r…   r�   rŽ   rP  r…   rQ  r'  r	  r÷   rR  rS  r=   rT  )r=   r©  )rÉ  r>   r=   r   )rO  rÕ  r�   r…   r�   rŽ   r/   ré   rR  r¶  r³  r´  rP  rø   rQ  r'  r¨  r©  rµ  rø   r~   r   r=   r°   )|r7   Ú
__future__r   Úconcurrent.futuresrˆ  Údataclassesrç  r¥   Úloggingrd  Úcollections.abcr   r   r   Útypingr   r   r	   Úlangchain_core._apir
   Úlangchain_core.callbacksr   Úlangchain_core.language_modelsr   Úlangchain_core.messagesr   r   Úlangchain_core.outputsr   r   Úlangchain_core.runnablesr   r   r   r   r;  r   rä  Ú!langchain_core.tracers.evaluationr   r   Ú langchain_core.tracers.langchainr   Úlangsmith.clientr   Úlangsmith.envr   r   Úlangsmith.evaluationr   r   r   r  Úlangsmith.run_helpersr    r!   Úlangsmith.schemasr"   r#   r$   r%   r&   Úlangsmith.utilsr'   Úrequestsr(   Útyping_extensionsr)   Úlangchain_classic.chains.baser*   Ú$langchain_classic.evaluation.loadingr+   Ú#langchain_classic.evaluation.schemar,   r-   r.   Úlangchain_classic.smithr/   r  Ú"langchain_classic.smith.evaluationr  r0   r1   ri   rr   Ú	getLoggerr4   r§   rn   rŽ   r�   rA  r3   r<   r|   r®   rÂ   rÄ   rÎ   rÚ   râ   ræ   rõ   rü   rÿ   r  r  r  rï   r2  r<  rD  rI  rK  rN  rq  rs  Ú	dataclassrv  rÈ  r¬  rã  rê  rî  r8   r9   r:   ú<module>r     sÅ  ðØ DÐ Dà "Ð "Ð "Ð "Ð "Ð "à Ð Ð Ð Ø Ð Ð Ð Ø Ð Ð Ð Ø €€€Ø €€€Ø €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø 'Ð 'Ð 'Ð 'Ð 'Ð 'Ð 'Ð 'ðð ð ð ð ð ð ð ð ð ð 0Ð /Ð /Ð /Ð /Ð /Ø .Ð .Ð .Ð .Ð .Ð .Ø <Ð <Ð <Ð <Ð <Ð <Ø CÐ CÐ CÐ CÐ CÐ CÐ CÐ CØ 8Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8Ø MÐ MÐ MÐ MÐ MÐ MÐ MÐ MÐ MÐ MØ >Ð >Ð >Ð >Ð >Ð >Ø <Ð <Ð <Ð <Ð <Ð <ðð ð ð ð ð ð ð ð =Ð <Ð <Ð <Ð <Ð <Ø #Ð #Ð #Ð #Ð #Ð #Ø FÐ FÐ FÐ FÐ FÐ FÐ FÐ Fðð ð ð ð ð ð ð ðð ð ð ð ð ð EÐ DÐ DÐ DÐ DÐ DÐ DÐ DØ LÐ LÐ LÐ LÐ LÐ LÐ LÐ LÐ LÐ LÐ LÐ LÐ LÐ LØ *Ð *Ð *Ð *Ð *Ð *Ø Ð Ð Ð Ð Ð Ø 'Ð 'Ð 'Ð 'Ð 'Ð 'à /Ð /Ð /Ð /Ð /Ð /Ø ?Ð ?Ð ?Ð ?Ð ?Ð ?ðð ð ð ð ð ð ð ð ð ð
 =Ð <Ð <Ð <Ð <Ð <Ø JÐ JÐ JÐ JÐ JÐ JØ HÐ HÐ HÐ HÐ HÐ HÐ HÐ Hàð ØÐÐÐà	ˆÔ	˜8Ñ	$Ô	$€ð ˆR�˜Ñ!Ð!Ô"Øñà�ˆv�sˆ{Ôñð ñð ñ	ð ð ˆr�5˜8Ñ#Ð#Ô$Ð'8Ñ8€ð3ð 3ð 3ð 3ð 3�yñ 3ô 3ð 3ðE4ð E4ð E4ð E4ð E4�ñ E4ô E4ð E4ðP-ð -ð -ð -ð -�ñ -ô -ð -ðB 'ð= ð = ð = ð = ð = ð@/ ð / ð / ð / ðd ð  ð  ð  ð  �Yñ  ô  ð  ð' ð ' ð ' ð ' ðV 6ð  6ð  6ð  6ðF%(ð %(ð %(ð %(ðPDð Dð Dð Dð(ð ð ð ð<ð ð ð ð6ð ð ð ð4ð ð ð ð&@ð @ð @ð @ðF	4ð 	4ð 	4ð 	4ðEð Eð Eð Eð^ "ØØ15Ø&*ðCð Cð Cð Cð Cð CðV "Ø15Ø&*ð!ð !ð !ð !ð !ð !ðR 26ð4ð 4ð 4ð 4ð 4ð 4ð~ "Ø15Ø&*ðAð Að Að Að Að AðR "Ø15Ø&*ð!ð !ð !ð !ð !ð !ðR 26ð6ð 6ð 6ð 6ð 6ð 6ð| /3Ø!Ø-1ð<5ð <5ð <5ð <5ð <5ð~ð ð ð ð � %ð ñ ô ð ð ÔðG
ð G
ð G
ð G
ð G
ñ G
ô G
ñ ÔðG
ðTð ð ð ð ð  ð  ð  ð 
:ð ð, 37Ø-1ØØ#Ø.2ØØ"ðk<ð k<ð k<ð k<ð k<ð k<ðf 37Ø-1ØØ#Ø.2ØØ"ðz<ð z<ð z<ð z<ð z<ð z<ð z<ð z<r9   