ó
    üÞ j–  ã                  óü  • S r SSKJr  SSKrSSKrSSKrSSKrSSKJr  SSK	J
r
JrJr  SSKJr  SSKJrJrJrJrJrJr  SSKJrJrJrJrJr  SS	KJr  SS
KJr   SSKJ!r!  SSK"J#r#  SSK$J%r%J&r&J'r'J(r(  \RR                  " \*5      r+        S)S jr, " S S\5      r- " S S\SS9r. " S S\5      r/ " S S\SS9r0 " S S5      r1\\/\0\24   r3 " S S\5      r4\\4\24   r5 " S S\15      r6  S*S jr7S r8S+S! jr9 " S" S#5      r:    S,S$ jr;    S-S% jr<    S.S& jr=    S/S' jr>\\\\!RP                     \\!RN                     /\\/\04   4   \\?\!RP                     \?\!RN                     /\\/\04   4   4   r@S0S( jrAg)1z?This module contains the evaluator classes for evaluating runs.é    )ÚannotationsN)Úabstractmethod)Ú	AwaitableÚ	CoroutineÚSequence)Úwraps)ÚAnyÚCallableÚLiteralÚOptionalÚUnionÚcast)Ú	BaseModelÚ
ConfigDictÚFieldÚValidationErrorÚmodel_validator)Ú	TypedDict©Úrun_helpers)Úschemas)Ú_safe_extract_feedback_keys)Ú
SCORE_TYPEÚ
VALUE_TYPEÚExampleÚRunc                ó¨   •  [         R                  " 5         [        SU SU S35      e! [         a    [         R                  " U " 5       5      s $ f = f)NzCannot call `zC` on an async run evaluator from within a running event loop. Use `z
` instead.)ÚasyncioÚget_running_loopÚRuntimeErrorÚrun)Úawaitable_factoryÚmethod_nameÚasync_method_names      ÚX/var/www/html/gaurav/venv/lib/python3.13/site-packages/langsmith/evaluation/evaluator.pyÚ_run_async_evaluator_syncr&       s`   € ð
0Ü× Ò Ô"ô Ø
˜�}ð %$Ø$5Ð#6°jð	Bóð øô ó 0Ü�{Š{Ñ,Ó.Ó/Ò/ð0ús   ‚) ©%AÁAc                  ó0   • \ rS rSr% SrS\S'    S\S'   Srg)	ÚCategoryé/   z$A category for categorical feedback.úOptional[Union[float, int]]ÚvalueÚstrÚlabel© N©Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú__annotations__Ú__static_attributes__r.   ó    r%   r(   r(   /   s   ‡ Ù.à&Ó&ØCØƒJÚ&r7   r(   c                  óH   • \ rS rSr% SrS\S'    S\S'    S\S'    S\S	'   S
rg)ÚFeedbackConfigé8   zcConfiguration to define a type of feedback.

Applied on on the first creation of a `feedback_key`.
z0Literal['continuous', 'categorical', 'freeform']Útyper*   ÚminÚmaxz%Optional[list[Union[Category, dict]]]Ú
categoriesr.   Nr/   r.   r7   r%   r9   r9   8   s*   ‡ ñð
 ;Ó:ØØ	$Ó$Ø;Ø	$Ó$ØAØ5Ö5r7   r9   F)Útotalc                  óö   • \ rS rSr% SrS\S'    SrS\S'    SrS\S	'    SrS
\S'    Sr	S\S'    Sr
S
\S'    \" \S9rS\S'    SrS\S'    SrS\S'    SrS\S'    SrS
\S'    \" SS9r\" SS9SS j5       rSrg)ÚEvaluationResultéG   zEvaluation result.r,   ÚkeyNr   Úscorer   r+   zOptional[dict]ÚmetadatazOptional[str]ÚcommentÚ
correction)Údefault_factoryÚdictÚevaluator_infoz%Optional[Union[FeedbackConfig, dict]]Úfeedback_configúOptional[Union[uuid.UUID, str]]Úsource_run_idÚtarget_run_idÚextraÚforbid)rO   Úafter)Úmodec                ó®   • U R                   cG  [        U R                  [        [        45      (       a"  [
        R                  SU R                   35        U $ )z:Warn when numeric values are passed via the `value` field.zJNumeric values should be provided in the 'score' field, not 'value'. Got: )rD   Ú
isinstancer+   ÚintÚfloatÚloggerÚwarning©Úselfs    r%   Úcheck_value_non_numericÚ(EvaluationResult.check_value_non_numericf   sH   € ð �:‰:Ñ¤*¨T¯Z©Z¼#¼u¸×"FÑ"FÜ�N‰NðØŸ™˜ð&ôð ˆr7   r.   )ÚreturnrA   )r0   r1   r2   r3   r4   r5   rD   r+   rE   rF   rG   r   rI   rJ   rK   rM   rN   rO   r   Úmodel_configr   r[   r6   r.   r7   r%   rA   rA   G   s¼   ‡ Ùà	ƒHØ@Ø€Eˆ:ÓØ0Ø€Eˆ:ÓØ8Ø#€HˆnÓ#Ø8Ø!€Gˆ]Ó!Ø2Ø!%€J�Ó%Ø:Ù °Ñ6€N�DÓ6Ø5Ø=A€OÐ:ÓAØ;Ø59€MÐ2Ó9Ø6Ø59€MÐ2Ó9ðð !€Eˆ>Ó Ø)á HÑ-€Lá˜'Ñ"óó #ór7   rA   c                  ó$   • \ rS rSr% SrS\S'   Srg)ÚEvaluationResultséq   zeBatch evaluation results.

This makes it easy for your evaluator to return multiple
metrics at once.
zlist[EvaluationResult]Úresultsr.   Nr/   r.   r7   r%   r`   r`   q   s   ‡ ñð $Ó#Ú!r7   r`   c                  óv   • \ rS rSrSr\  S       S	S jj5       r  S       S	S jjr\S
S j5       r	Sr
g)ÚRunEvaluatoré|   zEvaluator interface class.Nc                ó   • g)zEvaluate an example.Nr.   )rZ   r!   ÚexampleÚevaluator_run_ids       r%   Úevaluate_runÚRunEvaluator.evaluate_run   s   � r7   c              ƒ  ó°   ^ ^^^^#   • [         R                  " 5       mUUUUU 4S jn[        R                  " 5       R	                  SU5      I Sh  v•N $  N7f)z#Evaluate an example asynchronously.c                 óŒ   >• [         R                  " S0 T D6   TR                  TTT5      sS S S 5        $ ! , (       d  f       g = f)Nr.   )ÚrhÚtracing_contextri   )Úcurrent_contextrh   rg   r!   rZ   s   €€€€€r%   Ú_run_with_contextÚ5RunEvaluator.aevaluate_run.<locals>._run_with_context‘   s3   ø€ Ü×#Ò#Ñ6 oÓ6Ø×(Ñ(¨¨gÐ7GÓH÷ 7×6×6ús	   ˜5µ
AN)rm   Úget_tracing_contextr   r   Úrun_in_executor)rZ   r!   rg   rh   rp   ro   s   ```` @r%   Úaevaluate_runÚRunEvaluator.aevaluate_runˆ   sG   üé € ô ×0Ò0Ó2ˆ÷	Iñ 	Iô ×-Ò-Ó/×?Ñ?ÀÐFWÓX×XÐXÑXùs   ‡AAÁAÁAc                óN   • [        U SS5      n[        USS5      nU(       a  U/$ / $ )aÎ  Feedback keys this evaluator emits, when statically known.

Used to render per-evaluator experiment progress and to synthesize error
feedback when the evaluator raises. Best-effort: empty when the keys can't
be determined without running the evaluator. Subclasses that know their
keys should override this. The default also resolves evaluators that wrap
a single named inner evaluator exposed as ``.evaluator`` (e.g. the
LangChain string-evaluator integration).
Ú	evaluatorNÚevaluation_name)Úgetattr)rZ   ÚinnerÚnames      r%   Úfeedback_keysÚRunEvaluator.feedback_keys—   s0   € ô ˜˜k¨4Ó0ˆÜ�uÐ/°Ó6ˆÞ�ˆvÐ% 2Ð%r7   r.   ©NN©r!   r   rg   úOptional[Example]rh   úOptional[uuid.UUID]r]   ú*Union[EvaluationResult, EvaluationResults]©r]   ú	list[str])r0   r1   r2   r3   r4   r   ri   rt   Úpropertyr|   r6   r.   r7   r%   rd   rd   |   s‘   † Ù$àð &*Ø04ð	#àð#ð #ð#ð .ð	#ð
 
4ô#ó ð#ð &*Ø04ð	YàðYð #ðYð .ð	Yð
 
4õYð ó&ó ó&r7   rd   c                  óP   • \ rS rSr% SrS\S'    S\S'    SrS\S	'    SrS
\S'   Srg)ÚComparisonEvaluationResultéª   z¤Feedback scores for the results of comparative evaluations.

These are generated by functions that compare two or more runs,
returning a ranking or other feedback.
r,   rC   z'dict[Union[uuid.UUID, str], SCORE_TYPE]ÚscoresNrL   rM   z6Optional[Union[str, dict[Union[uuid.UUID, str], str]]]rF   r.   )	r0   r1   r2   r3   r4   r5   rM   rF   r6   r.   r7   r%   r‡   r‡   ª   s8   ‡ ñð 
ƒHØ@Ø3Ó3Ø4Ø59€MÐ2Ó9Ø6ØFJ€GÐCÓJòr7   r‡   c                  ó  ^ • \ rS rSrSr S   SS jjr S       SS jjr      SS jr      SS jr\	SS j5       r
\	SS j5       r  S       SS	 jjr  S     SU 4S
 jjjr S     SS jjrSS jrSrU =r$ )ÚDynamicRunEvaluatoréÁ   až  A dynamic evaluator that wraps a function and transforms it into a `RunEvaluator`.

This class is designed to be used with the `@run_evaluator` decorator, allowing
functions that take a `Run` and an optional `Example` as arguments, and return
an `EvaluationResult` or `EvaluationResults`, to be used as instances of `RunEvaluator`.

Attributes:
    func (Callable): The function that is wrapped by this evaluator.
c                ó:  ^• [        U5      U l        [        U5      u  nmU(       a  [        U5      u  nmS	U4S jjn[        U5      " U 5        SSKJn  Ub&  UR                  X#S9U l        [        USS5      U l	        [        R                  " U5      (       a5  Ub  [        S5      eUR                  XS9U l        [        USS5      U l	        gUR                  [        [        [        [         ["           /[$        4   U5      US9U l        [        USS5      U l	        g)
zÖInitialize the `DynamicRunEvaluator` with a given function.

Args:
    func (Callable): A function that takes a `Run` and an optional `Example` as
    arguments, and returns a dict or `ComparisonEvaluationResult`.
c                óf   >• Tc  U $ T" U R                  S5      U R                  S5      5      u    pU$ )Nr!   rg   ©Úget©ÚinputsÚ_Útraced_inputsÚprepare_inputss      €r%   Úprocess_inputsÚ4DynamicRunEvaluator.__init__.<locals>.process_inputså   s<   ø€ ØÑ%Ø�Ù$2Ø—
‘
˜5Ó! 6§:¡:¨iÓ#8ó%Ñ!ˆQ�ð !Ð r7   r   r   N©r–   r0   r‹   ú�Func was provided as a coroutine function, but afunc was also provided. If providing both, func should be a regular function to avoid ambiguity.©r’   rI   r]   rI   )r   Ú_feedback_keysÚ_normalize_evaluator_funcr   Ú	langsmithr   Úensure_traceableÚafuncry   Ú_nameÚinspectÚiscoroutinefunctionÚ	TypeErrorr   r
   r   r   r   Ú_RUNNABLE_OUTPUTÚfunc©rZ   r¥   rŸ   r–   r   r•   s        @r%   Ú__init__ÚDynamicRunEvaluator.__init__Ì   s   ø€ ô( :¸$Ó?ˆÔÜ!:¸4Ó!@Ñˆˆ~ÞÜ&?ÀÓ&FÑ#ˆU�N÷	!ô 	ˆdŒ�DÔÝ)àÑØ$×5Ñ5Øð 6ð ˆDŒJô ! ¨
Ð4IÓJˆDŒJÜ×&Ò& t×,Ñ,ØÑ Üð3óð ð
 %×5Ñ5Øð 6ð ˆDŒJô !  zÐ3HÓIˆD�Jà#×4Ñ4Ü”Xœs¤H¬WÑ$5Ð6Ô8HÐHÑIÈ4ÓPØ-ð 5ð ˆDŒIô !  zÐ3HÓIˆD�Jr7   c                óv  ^• [        T[        5      (       a  TR                  (       d  UTl        T$  T(       d  [        ST 35      eST;  a  U(       a  U R                  TS'   [        U4S jS 5       5      (       a  [        ST 35      e[        S0 SU0TED6$ ! [         a  n[        ST 35      UeS nAff = f)	NziExpected an EvaluationResult object, or dict with a metric 'key' and optional 'score'; got empty result: rC   c              3  ó*   >#   • U H	  oT;  v •  M     g 7f©Nr.   )Ú.0ÚkÚresults     €r%   Ú	<genexpr>Ú@DynamicRunEvaluator._coerce_evaluation_result.<locals>.<genexpr>  s   øé € ÐJÑ,I q˜F–?Ò,Iùó   ƒ)rD   r+   rF   zrExpected an EvaluationResult object, or dict with a metric 'key' and optional 'score' or categorical 'value'; got rM   z[Expected an EvaluationResult object, or dict with a metric 'key' and optional 'score'; got r.   )rT   rA   rM   Ú
ValueErrorr    Úallr   )rZ   r®   rM   Úallow_no_keyÚes    `   r%   Ú_coerce_evaluation_resultÚ-DynamicRunEvaluator._coerce_evaluation_result  sã   ø€ ô �fÔ.×/Ñ/Ø×'×'Ø'4�Ô$ØˆMð	ÞÜ ðFØFLÀXðOóð ð ˜FÓ"¦|Ø $§
¡
��u‘ÜÔJÑ,IÓJ×JÑJÜ ðOØOUÈhðXóð ô $ÑQ ¸Ð&PÈÐ&PÑQÐQøÜó 	Üð4Ø4:°8ð=óð ðûð	ús   ²A'B Â
B8Â$B3Â3B8c                óÖ   • SU;   a@  UR                  5       nUS    Vs/ sH  nU R                  XBS9PM     snUS'   [        S0 UD6$ U R                  [        [        U5      USS9$ s  snf )Nrb   )rM   T)rM   r´   r.   )Úcopyr¶   r`   r   rI   )rZ   rb   rM   ÚcpÚrs        r%   Ú_coerce_evaluation_resultsÚ.DynamicRunEvaluator._coerce_evaluation_results%  sˆ   € ð
 ˜ÓØ—‘“ˆBð ! Ò+óá+�Að ×.Ñ.¨qÐ.ÓNÙ+ñˆBˆy‰Mô %Ñ* rÑ*Ð*à×-Ñ-Ü”�wÓ¨}È4ð .ð 
ð 	
ùòs   žA&c                ó–   • [        U[        5      (       a  UR                  (       d  X!l        U$ [        U5      nU R	                  X5      $ r«   )rT   rA   rM   Ú_format_evaluator_resultr¼   )rZ   r®   rM   s      r%   Ú_format_resultÚ"DynamicRunEvaluator._format_result6  sA   € ô �fÔ.×/Ñ/Ø×'×'Ø'4Ô$ØˆMÜ)¨&Ó1ˆØ×.Ñ.¨vÓEÐEr7   c                ó   • [        U S5      $ ©zŠCheck if the evaluator function is asynchronous.

Returns:
    bool: `True` if the evaluator function is asynchronous, `False` otherwise.
rŸ   ©ÚhasattrrY   s    r%   Úis_asyncÚDynamicRunEvaluator.is_asyncD  ó   € ô �t˜WÓ%Ð%r7   c                ó   • U R                   $ )zÑFeedback keys this evaluator emits, statically inferred from its source.

Best-effort: empty when keys can't be determined without running the
function (e.g. a dynamically computed key or unavailable source).
)r›   rY   s    r%   r|   Ú!DynamicRunEvaluator.feedback_keysM  s   € ð ×"Ñ"Ð"r7   c                ó8  ^ ^^• [        T S5      (       d  [        UUU 4S jSS5      $ Uc  [        R                  " 5       nSTR                  0n[        TSS5      (       a  [        TR                  5      US'   T R                  TTX4S	.S
9nT R                  XS5      $ )aU  Evaluate a run using the wrapped function.

This method directly invokes the wrapped function with the provided arguments.

Args:
    run (Run): The run to be evaluated.
    example (Optional[Example]): An optional example to be used in the evaluation.

Returns:
    Union[EvaluationResult, EvaluationResults]: The result of the evaluation.
r¥   c                 ó(   >• TR                  TT 5      $ r«   )rt   )rg   r!   rZ   s   €€€r%   Ú<lambda>Ú2DynamicRunEvaluator.evaluate_run.<locals>.<lambda>i  s   ø€ ˜×*Ñ*¨3°Ô8r7   ri   rt   NrN   Ú
session_idÚ
experiment©Úrun_idrE   ©Úlangsmith_extra)
rÅ   r&   ÚuuidÚuuid4Úidry   r,   rÏ   r¥   rÀ   )rZ   r!   rg   rh   rE   r®   s   ```   r%   ri   Ú DynamicRunEvaluator.evaluate_runV  sž   ú€ ô" �t˜V×$Ñ$Ü,Þ8ØØóð ð
 Ñ#Ü#Ÿzšz›|ÐØ$3°S·V±VÐ#<ˆÜ�3˜ d×+Ñ+Ü%(¨¯©Ó%8ˆH�\Ñ"Ø—‘ØØØ'7ÑNð ð 
ˆð
 ×"Ñ" 6Ó<Ð<r7   c              ƒ  ó\  >#   • [        U S5      (       d  [        TU ]	  X5      I Sh  v•N $ Uc  [        R                  " 5       nSUR
                  0n[        USS5      (       a  [        UR                  5      US'   U R                  UUX4S.S9I Sh  v•N nU R                  XS5      $  N€ N7f)a|  Evaluate a run asynchronously using the wrapped async function.

This method directly invokes the wrapped async function with the
    provided arguments.

Args:
    run (Run): The run to be evaluated.
    example (Optional[Example]): An optional example to be used
        in the evaluation.

Returns:
    Union[EvaluationResult, EvaluationResults]: The result of the evaluation.
rŸ   NrN   rÏ   rÐ   rÑ   rÓ   )rÅ   Úsuperrt   rÕ   rÖ   r×   ry   r,   rÏ   rŸ   rÀ   )rZ   r!   rg   rh   rE   r®   Ú	__class__s         €r%   rt   Ú!DynamicRunEvaluator.aevaluate_runy  s«   øé € ô& �t˜W×%Ñ%Ü™Ñ.¨sÓ<×<Ð<ØÑ#Ü#Ÿzšz›|ÐØ$3°S·V±VÐ#<ˆÜ�3˜ d×+Ñ+Ü%(¨¯©Ó%8ˆH�\Ñ"Ø—z‘zØØØ'7ÑNð "ð 
÷ 
ˆð
 ×"Ñ" 6Ó<Ð<ñ =ñ
ùs"   ƒ$B,§B(¨A*B,ÂB*ÂB,Â*B,c                ó$   • U R                  X5      $ )a‰  Make the evaluator callable, allowing it to be used like a function.

This method enables the evaluator instance to be called directly, forwarding the
call to `evaluate_run`.

Args:
    run (Run): The run to be evaluated.
    example (Optional[Example]): An optional example to be used in the evaluation.

Returns:
    Union[EvaluationResult, EvaluationResults]: The result of the evaluation.
)ri   )rZ   r!   rg   s      r%   Ú__call__ÚDynamicRunEvaluator.__call__š  s   € ð × Ñ  Ó.Ð.r7   c                ó"   • SU R                    S3$ )ú)Represent the DynamicRunEvaluator object.z<DynamicRunEvaluator Ú>©r    rY   s    r%   Ú__repr__ÚDynamicRunEvaluator.__repr__«  s   € à& t§z¡z l°!Ð4Ð4r7   )r›   r    rŸ   r¥   r«   )r¥   úXCallable[[Run, Optional[Example]], Union[_RUNNABLE_OUTPUT, Awaitable[_RUNNABLE_OUTPUT]]]rŸ   zIOptional[Callable[[Run, Optional[Example]], Awaitable[_RUNNABLE_OUTPUT]]])F)r®   zUnion[EvaluationResult, dict]rM   ú	uuid.UUIDr´   Úboolr]   rA   )rb   zUnion[dict, EvaluationResults]rM   rç   r]   r‚   )r®   zMUnion[EvaluationResult, EvaluationResults, dict, str, int, bool, float, list]rM   rç   r]   r‚   ©r]   rè   rƒ   r~   r   )r!   r   rg   r€   rh   r�   )r!   r   rg   r€   r]   r‚   ©r]   r,   )r0   r1   r2   r3   r4   r§   r¶   r¼   rÀ   r…   rÆ   r|   ri   rt   rÞ   rä   r6   Ú__classcell__)rÛ   s   @r%   r‹   r‹   Á   sY  ø† ñð, ð9Jð
ð9Jð
õ9Jð~ #ð	à-ðð !ðð ð	ð
 
õð<
à/ð
ð !ð
ð 
4ô	
ð"Fð
ðFð
 !ðFð 
4ôFð ó&ó ð&ð ó#ó ð#ð &*Ø04ð	!=àð!=ð #ð!=ð .ð	!=ð
 
4õ!=ðL &*Ø04ð	=àð=ð #ð=ð .÷	=ð =ðD 6:ð/Øð/Ø!2ð/à	3õ/÷"5ò 5r7   r‹   c                ó   • [        U 5      $ )zeCreate a run evaluator from a function.

Decorator that transforms a function into a `RunEvaluator`.
)r‹   ©r¥   s    r%   Úrun_evaluatorrî   °  s   € ô ˜tÓ$Ð$r7   i'  c                ó`   • [        U 5      n[        U5      [        :”  a  US [        S-
   S-   nU$ )Né   z...))ÚreprÚlenÚ_MAXSIZE)ÚobjÚss     r%   Ú_maxsize_reprrö   ¿  s1   € ÜˆS‹	€AÜ
ˆ1ƒv”ÓØˆn”˜1‘Ð Ñ&ˆØ€Hr7   c                  óÈ   • \ rS rSrSr S   SS jjr\SS j5       r S     SS jjr S     SS jjr	 S     SS jjr
SS	 jr\SS
 j5       r        SS jrSrg)ÚDynamicComparisonRunEvaluatoriÆ  z4Compare predictions (as traces) from 2 or more runs.Nc                ó(  ^• [        U5      u  nmU(       a  [        U5      u  nmS	U4S jjn[        U5      " U 5        SSKJn  Ub&  UR	                  X#S9U l        [        USS5      U l        [        R                  " U5      (       a5  Ub  [        S5      eUR	                  XS9U l        [        USS5      U l        gUR	                  [        [        [        [           [        [            /["        4   U5      US9U l        [        USS5      U l        g)
zÜInitialize the `DynamicRunEvaluator` with a given function.

Args:
    func (Callable): A function that takes a `Run` and an optional `Example` as
    arguments, and returns an `EvaluationResult` or `EvaluationResults`.
c                óf   >• Tc  U $ T" U R                  S5      U R                  S5      5      u    pU$ )NÚrunsrg   r�   r‘   s      €r%   r–   Ú>DynamicComparisonRunEvaluator.__init__.<locals>.process_inputsá  s<   ø€ ØÑ%Ø�Ù$2Ø—
‘
˜6Ó" F§J¡J¨yÓ$9ó%Ñ!ˆQ�ð !Ð r7   r   r   Nr˜   r0   r‹   r™   rš   )Ú$_normalize_comparison_evaluator_funcr   r�   r   rž   rŸ   ry   r    r¡   r¢   r£   r   r
   r   r   r   r   Ú_COMPARISON_OUTPUTr¥   r¦   s        @r%   r§   Ú&DynamicComparisonRunEvaluator.__init__É  s%  ø€ ô( "FÀdÓ!KÑˆˆ~ÞÜ&JÈ5Ó&QÑ#ˆU�N÷	!ô 	ˆdŒ�DÔÝ)àÑØ$×5Ñ5Øð 6ð ˆDŒJô ! ¨
Ð4IÓJˆDŒJÜ×&Ò& t×,Ñ,ØÑ Üð3óð ð
 %×5Ñ5Øð 6ð ˆDŒJô !  zÐ3HÓIˆD�Jà#×4Ñ4ÜÜÜ!¤#™¬´Ñ(9Ð:Ü*ð,ñð óð  .ð 5ð 	ˆDŒIô !  zÐ3HÓIˆD�Jr7   c                ó   • [        U S5      $ rÃ   rÄ   rY   s    r%   rÆ   Ú&DynamicComparisonRunEvaluator.is_async	  rÈ   r7   c                óæ   ^ ^^• [        T S5      (       d  [        UUU 4S jSS5      $ [        R                  " 5       nT R	                  T5      nT R                  TTX4S.S9nT R                  XST5      $ )z�Compare runs to score preferences.

Args:
    runs: A list of runs to compare.
    example: An optional example to be used in the evaluation.

r¥   c                 ó(   >• TR                  TT 5      $ r«   )Úacompare_runs)rg   rû   rZ   s   €€€r%   rÍ   Ú<DynamicComparisonRunEvaluator.compare_runs.<locals>.<lambda>  s   ø€ ˜×*Ñ*¨4°Ô9r7   Úcompare_runsr  ©rÒ   ÚtagsrÓ   )rÅ   r&   rÕ   rÖ   Ú	_get_tagsr¥   Ú_format_results©rZ   rû   rg   rM   r  r®   s   ```   r%   r  Ú*DynamicComparisonRunEvaluator.compare_runs  sw   ú€ ô �t˜V×$Ñ$Ü,Þ9ØØóð ô
 Ÿ
š
›ˆØ�~‰~˜dÓ#ˆà—‘ØØØ'4ÑCð ð 
ˆð
 ×#Ñ# F¸4Ó@Ð@r7   c              ƒ  óø   #   • [        U S5      (       d  U R                  X5      $ [        R                  " 5       nU R	                  U5      nU R                  UUX4S.S9I Sh  v•N nU R                  XSU5      $  N7f)an  Evaluate a run asynchronously using the wrapped async function.

This method directly invokes the wrapped async function with the
    provided arguments.

Args:
    runs (Run): The runs to be evaluated.
    example (Optional[Example]): An optional example to be used
        in the evaluation.

Returns:
    ComparisonEvaluationResult: The result of the evaluation.
rŸ   r  rÓ   N)rÅ   r  rÕ   rÖ   r	  rŸ   r
  r  s         r%   r  Ú+DynamicComparisonRunEvaluator.acompare_runs,  s|   é € ô  �t˜W×%Ñ%Ø×$Ñ$ TÓ3Ð3ÜŸ
š
›ˆØ�~‰~˜dÓ#ˆà—z‘zØØØ'4ÑCð "ð 
÷ 
ˆð
 ×#Ñ# F¸4Ó@Ð@ñ
ùs   ‚AA:Á!A8Á"A:c                ó$   • U R                  X5      $ )ay  Make the evaluator callable, allowing it to be used like a function.

This method enables the evaluator instance to be called directly, forwarding the
call to `evaluate_run`.

Args:
    run (Run): The run to be evaluated.
    example (Optional[Example]): An optional example to be used in the evaluation.

Returns:
    ComparisonEvaluationResult: The result of the evaluation.
)r  )rZ   rû   rg   s      r%   rÞ   Ú&DynamicComparisonRunEvaluator.__call__H  s   € ð × Ñ  Ó/Ð/r7   c                ó"   • SU R                    S3$ )rá   z<DynamicComparisonRunEvaluator râ   rã   rY   s    r%   rä   Ú&DynamicComparisonRunEvaluator.__repr__Y  s   € à0°·±°¸AÐ>Ð>r7   c                óà   • / nU  He  nUR                  S[        UR                  5      -   5        [        USS5      (       d  M>  UR                  S[        UR                  5      -   5        Mg     U$ )zExtract tags from runs.zrun:rÏ   Nzexperiment:)Úappendr,   r×   ry   rÏ   )rû   r  r!   s      r%   r	  Ú'DynamicComparisonRunEvaluator._get_tags]  s\   € ð ˆÛˆCØ�K‰K˜¤ S§V¡V£Ñ,Ô-Ü�s˜L¨$×/Ó/Ø—‘˜M¬C°·±Ó,?Ñ?Ö@ñ ð ˆr7   c                óà  • [        U[        5      (       a  UR                  (       d  X!l        U$ [        U[        5      (       a9  [	        X15       VVs0 sH  u  pEUR
                  U_M     snnU R                  US.nO<[        U[        5      (       a  SU;  a  U R                  US'   OSU< 3n[        U5      e [        S0 SU0UED6$ s  snnf ! [         a  n[        SU 35      UeS nAff = f)N)r‰   rC   rM   rC   zXExpected 'dict', 'list' or 'ComparisonEvaluationResult' result object. Received: result=rM   zˆExpected a dictionary with a 'key' and dictionary of scores mappingrun IDs to numeric scores, or ComparisonEvaluationResult object, got r.   )
rT   r‡   rM   ÚlistÚzipr×   r    rI   r²   r   )rZ   r®   rM   rû   r!   rD   Úmsgrµ   s           r%   r
  Ú-DynamicComparisonRunEvaluator._format_resultsh  s	  € ô �fÔ8×9Ñ9Ø×'×'Ø'4Ô$ØˆMÜ˜¤×%Ñ%ä;>¸tÔ;LÔMÑ;L©Z¨S˜3Ÿ6™6 5š=Ñ;LÒMØ—z‘zØ!.ñ‰Fô
 ˜¤×%Ñ%Ø˜FÓ"Ø $§
¡
��u‘øð-Ø%+¡Ið/ð ô ˜S“/Ð!ð		Ü-ñ Ø" MÐ<°VÐ<ñð ùó Nøô" ó 	Üðà�xð!óð ð	ûð	ús   ÁC	Â:C Ã
C-ÃC(Ã(C-)r    rŸ   r¥   r«   )r¥   úfCallable[[Sequence[Run], Optional[Example]], Union[_COMPARISON_OUTPUT, Awaitable[_COMPARISON_OUTPUT]]]rŸ   zUOptional[Callable[[Sequence[Run], Optional[Example]], Awaitable[_COMPARISON_OUTPUT]]]ré   )rû   úSequence[Run]rg   r€   r]   r‡   rê   )rû   r  r]   r„   )r®   z-Union[dict, list, ComparisonEvaluationResult]rM   rç   rû   r  r]   r‡   )r0   r1   r2   r3   r4   r§   r…   rÆ   r  r  rÞ   rä   Ústaticmethodr	  r
  r6   r.   r7   r%   rø   rø   Æ  só   † Ù>ð ð>Jð
ð>Jð
õ>Jð@ ó&ó ð&ð AEðAØ!ðAØ,=ðAà	#õAð6 AEðAØ!ðAØ,=ðAà	#õAð: AEð0Ø!ð0Ø,=ð0à	#õ0ô"?ð óó ðð"à=ð"ð !ð"ð ð	"ð
 
$÷"r7   rø   c                ó   • [        U 5      $ )z.Create a comaprison evaluator from a function.)rø   rí   s    r%   Úcomparison_evaluatorr  �  s   € ô )¨Ó.Ð.r7   c                ó  ^ ^^	^
^• Sm[         R                  " T 5      m
T
R                  R                  5        VVs/ sH#  u  pUR                  UR
                  :w  d  M!  UPM%     nnnT
R                  R                  5        VVs/ sH0  u  pUR                  [         R                  R                  Ld  M.  UPM2     snnm	U(       aA  [        U	U4S jU 5       5      (       d7  [        U Vs/ sH  oDT	;  d  M
  UPM     sn5      S:w  a  ST S3n[        U5      e[        U	U4S jU 5       5      (       a  USS/:X  a  T S 4$ [         R                  " T 5      (       aO        SU
4S	 jjm      SUU 4S
 jjn[        T S5      (       a  [        T S5      OUR                  Ul        UT4$       SU
4S jjmSUU 4S jjn[        T S5      (       a  [        T S5      OUR                  Ul        UT4$ s  snnf s  snnf s  snf )N)r!   rg   r’   ÚoutputsÚreference_outputsÚattachmentsc              3  óB   >#   • U H  oT;   =(       d    UT;   v •  M     g 7fr«   r.   ©r¬   ÚpnameÚargs_with_defaultsÚsupported_argss     €€r%   r¯   Ú,_normalize_evaluator_func.<locals>.<genexpr>°  ó&   øé € ð 
ÙPXÀu�^Ñ#×B uÐ0BÑ'BÔBÒPXùó   ƒé   úUInvalid evaluator function. Must have at least one argument. Supported arguments are ú€. Please see https://docs.smith.langchain.com/evaluation/how_to_guides/evaluation/evaluate_llm_application#use-custom-evaluatorsc              3  óB   >#   • U H  oT;   =(       d    UT;   v •  M     g 7fr«   r.   r%  s     €€r%   r¯   r)  ¾  ó%   øé € ð ÙLTÀ5�Ñ×> 5Ð,>Ñ#>Ô>ÊHùr+  r!   rg   c                óì  >• U UU(       a  UR                   O0 U R                  =(       d    0 U(       a  UR                  =(       d    0 O0 U(       a  UR                  =(       d    0 O0 S.n0 n/ n0 nTR                  R	                  5        He  u  pgXb;   d  M  UR
                  UR                  UR                  4;   a  UR                  X&   5        OX&   X6'   US;   a  [        X&   5      OX&   XV'   Mg     XCU4$ ©N)r!   rg   r’   r!  r#  r"  )r!   rg   ©
r’   r!  r#  Ú
parametersÚitemsÚkindÚPOSITIONAL_OR_KEYWORDÚPOSITIONAL_ONLYr  rö   ©	r!   rg   Úarg_mapÚkwargsÚargsr”   Ú
param_nameÚparamÚsigs	           €r%   Ú_prepare_inputsÚ2_normalize_evaluator_func.<locals>._prepare_inputsÈ  óñ   ø€ ð Ø&Þ07˜gŸnšn¸RØ"Ÿ{™{×0¨bÞ@G 7×#6Ñ#6×#<¸"øÈRÞBI¨¯©×)>¸BøÈrñ�ð �Ø�Ø "�Ø),¯©×)=Ñ)=Ö)?Ñ%�Jà!Õ,Ø Ÿ:™:Ø!×7Ñ7Ø!×1Ñ1ð*ó ð !ŸK™K¨Ñ(;Õ<à18Ñ1D˜FÑ.ð  *Ð-?Ó?ô *¨'Ñ*=Ô>à!(Ñ!4ð &Ó1ñ *@ð  ]Ð2Ð2r7   c              “  óF   >#   • T" X5      u  p#nT" U0 UD6I S h  v•N $  N7fr«   r.   ©r!   rg   r<  r;  r“   r@  r¥   s        €€r%   ÚawrapperÚ+_normalize_evaluator_func.<locals>.awrapperç  s-   øé € ñ %4°CÓ$AÑ!�˜qÙ! 4Ð2¨6Ñ2×2Ð2Ñ2ùó   ƒ!š›!r0   c                óì  >• U UU(       a  UR                   O0 U R                  =(       d    0 U(       a  UR                  =(       d    0 O0 U(       a  UR                  =(       d    0 O0 S.n0 n/ n0 nTR                  R	                  5        He  u  pgXb;   d  M  UR
                  UR                  UR                  4;   a  UR                  X&   5        OX&   X6'   US;   a  [        X&   5      OX&   XV'   Mg     XCU4$ r2  r3  r9  s	           €r%   r@  rA  ö  rB  r7   c                ó*   >• T" X5      u  p#nT" U0 UD6$ r«   r.   rD  s        €€r%   ÚwrapperÚ*_normalize_evaluator_func.<locals>.wrapper  s"   ø€ Ù$3°CÓ$AÑ!�˜qÙ˜TÐ, VÑ,Ð,r7   )r!   r   rg   r€   r]   útuple[list, dict, dict])r!   r   rg   r€   r]   r¤   ©r¡   Ú	signaturer4  r5  r6  ÚVAR_KEYWORDÚdefaultÚ	ParameterÚemptyr³   rò   r²   r¢   rÅ   ry   r0   ©r¥   r&  ÚpÚall_argsÚar  rE  rJ  r@  r'  r?  r(  s   `       @@@@r%   rœ   rœ   —  s9  ü€ ð€Nô ×
Ò
˜DÓ
!€CØ&)§n¡n×&:Ñ&:Ô&<ÔXÑ&<™(˜%ÀÇÁÈ!Ï-É-Ñ@W—Ñ&<€HÑXð Ÿ™×,Ñ,Ô.ôá.‰HˆEØ�9‰9œG×-Ñ-×3Ñ3Ð3÷ 	Ù.òÐö
 Üõ 
ÙPXó
÷ 
ñ 
ô ™HÓD™H�qÐ1CÑ(C—™HÑDÓEÈÓJð1Ø1?Ð0@ð AFðGð 	ô ˜‹oÐô õ ÙLTó÷ ñ à	ØØðó 
ð �TˆzÐä×&Ò& t×,Ñ,ð3Øð3Ø#4ð3à(÷3ð>3Øð3Ø#4ð3à!÷3ð 3ô ˜4 ×,Ñ,ô ˜˜jÔ)à×&Ñ&ð Ôð
 ˜oÐ.Ð.ð3Øð3Ø#4ð3à(÷3÷>-ð -ô ˜4 ×,Ñ,ô ˜˜jÔ)à×%Ñ%ð Ôð
 ˜_Ð-Ð-ùók Yùóùò Es#   »G1ÁG1Â,G7Â4G7Ã)G=Ã5G=c                ó  ^ ^^	^
^• Sm[         R                  " T 5      m
T
R                  R                  5        VVs/ sH#  u  pUR                  UR
                  :w  d  M!  UPM%     nnnT
R                  R                  5        VVs/ sH0  u  pUR                  [         R                  R                  Ld  M.  UPM2     snnm	U(       aA  [        U	U4S jU 5       5      (       d7  [        U Vs/ sH  oDT	;  d  M
  UPM     sn5      S:w  a  ST S3n[        U5      e[        U	U4S jU 5       5      (       a  USS/:X  a  T S 4$ [         R                  " T 5      (       aO        SU
4S	 jjm      SUU 4S
 jjn[        T S5      (       a  [        T S5      OUR                  Ul        UT4$       SU
4S jjm      SUU 4S jjn[        T S5      (       a  [        T S5      OUR                  Ul        UT4$ s  snnf s  snnf s  snf )N©rû   rg   r’   r!  r"  c              3  óB   >#   • U H  oT;   =(       d    UT;   v •  M     g 7fr«   r.   r%  s     €€r%   r¯   Ú7_normalize_comparison_evaluator_func.<locals>.<genexpr>3  r*  r+  r,  r-  r.  c              3  óB   >#   • U H  oT;   =(       d    UT;   v •  M     g 7fr«   r.   r%  s     €€r%   r¯   rZ  A  r0  r+  rû   rg   c                óÚ  >• U UU(       a  UR                   O0 U  Vs/ sH  o"R                  =(       d    0 PM     snU(       a  UR                  =(       d    0 O0 S.n0 n/ n0 nT	R                  R                  5        He  u  pxXs;   d  M  UR                  UR
                  UR                  4;   a  UR                  X7   5        OX7   XG'   US;   a  [        X7   5      OX7   Xg'   Mg     XTU4$ s  snf ©NrX  )rû   rg   ©	r’   r!  r4  r5  r6  r7  r8  r  rö   ©
rû   rg   r!   r:  r;  r<  r”   r=  r>  r?  s
            €r%   r@  Ú=_normalize_comparison_evaluator_func.<locals>._prepare_inputsK  óô   ø€ ð !Ø&Þ07˜gŸnšn¸RÙ=AÓB¹T°c§¡× 1¨rÒ 1¹TÑBÞBI¨¯©×)>¸BøÈrñ�ð �Ø�Ø "�Ø),¯©×)=Ñ)=Ö)?Ñ%�Jà!Õ,Ø Ÿ:™:Ø!×7Ñ7Ø!×1Ñ1ð*ó ð !ŸK™K¨Ñ(;Õ<à18Ñ1D˜FÑ.ð  *Ð-@Ó@ô *¨'Ñ*=Ô>à!(Ñ!4ð &Ó1ñ *@ð  ]Ð2Ð2ùò+  Có   œC(
c              “  óF   >#   • T" X5      u  p#nT" U0 UD6I S h  v•N $  N7fr«   r.   ©rû   rg   r<  r;  r“   r@  r¥   s        €€r%   rE  Ú6_normalize_comparison_evaluator_func.<locals>.awrapperi  s-   øé € ñ %4°DÓ$BÑ!�˜qÙ! 4Ð2¨6Ñ2×2Ð2Ñ2ùrG  r0   c                óÚ  >• U UU(       a  UR                   O0 U  Vs/ sH  o"R                  =(       d    0 PM     snU(       a  UR                  =(       d    0 O0 S.n0 n/ n0 nT	R                  R                  5        He  u  pxXs;   d  M  UR                  UR
                  UR                  4;   a  UR                  X7   5        OX7   XG'   US;   a  [        X7   5      OX7   Xg'   Mg     XTU4$ s  snf r]  r^  r_  s
            €r%   r@  r`  x  ra  rb  c                ó*   >• T" X5      u  p#nT" U0 UD6$ r«   r.   rd  s        €€r%   rJ  Ú5_normalize_comparison_evaluator_func.<locals>.wrapper–  s$   ø€ ñ %4°DÓ$BÑ!�˜qÙ˜TÐ, VÑ,Ð,r7   )rû   r  rg   r€   r]   rL  )rû   r  rg   r€   r]   rþ   rM  rS  s   `       @@@@r%   rý   rý   !  sP  ü€ ð S€NÜ
×
Ò
˜DÓ
!€CØ&)§n¡n×&:Ñ&:Ô&<ÔXÑ&<™(˜%ÀÇÁÈ!Ï-É-Ñ@W—Ñ&<€HÑXð Ÿ™×,Ñ,Ô.ôá.‰HˆEØ�9‰9œG×-Ñ-×3Ñ3Ð3÷ 	Ù.òÐö
 Üõ 
ÙPXó
÷ 
ñ 
ô ™HÓD™H�qÐ1CÑ(C—™HÑDÓEÈÓJð1Ø1?Ð0@ð AFðGð 	ô ˜‹oÐô õ ÙLTó÷ ñ à	ØØðó 
ð �TˆzÐä×&Ò& t×,Ñ,ð3Ø#ð3Ø.?ð3à(÷3ð<3Ø#ð3Ø.?ð3à#÷3ð 3ô ˜4 ×,Ñ,ô ˜˜jÔ)à×&Ñ&ð Ôð
 ˜_Ð,Ð,ð3Ø#ð3Ø.?ð3à(÷3ð<-Ø#ð-Ø.?ð-à#÷-ð -ô ˜4 ×,Ñ,ô ˜˜jÔ)à×%Ñ%ð Ôð
 ˜OÐ+Ð+ùók Yùóùò Es#   »G7ÁG7Â,G=Â4G=Ã)HÃ5Hc                ó|  • [        U [        [        [        45      (       a  SU 0n U $ U (       d  [	        SU  35      e[        U [
        5      (       a,  [        S U  5       5      (       d  [	        SU  S35      eSU 0n U $ [        U [        5      (       a  SU 0n U $ [        U [        5      (       a   U $ [	        SU  35      e)	NrD   zdExpected a non-empty dict, str, bool, int, float, list, EvaluationResult, or EvaluationResults. Got c              3  ó@   #   • U H  n[        U[        5      v •  M     g 7fr«   )rT   rI   )r¬   Úxs     r%   r¯   Ú+_format_evaluator_result.<locals>.<genexpr>¯  s   é € Ð7±¨1”:˜a¤×&Ð&²ùs   ‚z8Expected a list of dicts or EvaluationResults. Received Ú.rb   r+   zZExpected a dict, str, bool, int, float, list, EvaluationResult, or EvaluationResults. Got )	rT   rè   rV   rU   r²   r  r³   r,   rI   )r®   s    r%   r¿   r¿   ¤  sï   € ô �&œ4¤¬Ð,×-Ñ-Ø˜6Ð"ˆð* €Mö) Üð;Ø;A¸(ðDó
ð 	
ô 
�FœD×	!Ñ	!ÜÑ7±Ó7×7Ñ7ÜØJÈ6È(ÐRSÐTóð ð ˜VÐ$ˆð €Mô 
�FœC×	 Ñ	 Ø˜6Ð"ˆð €Mô 
�FœD×	!Ñ	!Øð €Mô	 ð&Ø&, Xð/ó
ð 	
r7   c                ó
  ^ ^^^	• Sm	[         R                  " T 5      mTR                  R                  5        VVs/ sH  u  pUPM	     nnnTR                  R                  5        VVs/ sH0  u  pUR                  [         R
                  R                  Ld  M.  UPM2     snnmU(       aA  [        UU	4S jU 5       5      (       dG  [        U Vs/ sH  oDT;  d  M
  UPM     sn5      S:w  a!  ST	 S3nU(       a	  USU S3-  n[        U5      e[        U	4S jU 5       5      (       a  USS	/:X  a  T $       SU U4S
 jjn[        T S5      (       a  [        T S5      Ul        U$ UR                  Ul        U$ s  snnf s  snnf s  snf )N©rû   Úexamplesr’   r!  r"  c              3  óB   >#   • U H  oT;   =(       d    UT;   v •  M     g 7fr«   r.   r%  s     €€r%   r¯   Ú/_normalize_summary_evaluator.<locals>.<genexpr>Ö  r*  r+  r,  r-  rm  z Received arguments c              3  ó*   >#   • U H	  oT;   v •  M     g 7fr«   r.   )r¬   r&  r(  s     €r%   r¯   rr  ä  s   øé € Ð?±h¨U˜.Ö(²hùr±   rû   rp  c           	     ó  >• U UU Vs/ sH  o"R                   PM     snU  Vs/ sH  o3R                  =(       d    0 PM     snU Vs/ sH  o"R                  =(       d    0 PM     snS.n0 n/ nTR                  R                  5        HM  u  pxXt;   d  M  UR                  UR
                  UR                  4;   a  UR                  XG   5        MG  XG   XW'   MO     T
" U0 UD6n	[        U	[        5      (       a  U	$ [        U	5      $ s  snf s  snf s  snf )Nro  )r’   r!  r4  r5  r6  r7  r8  r  rT   rA   r¿   )rû   rp  rg   r!   r:  r;  r<  r=  r>  r®   r¥   r?  s             €€r%   rJ  Ú-_normalize_summary_evaluator.<locals>.wrapperë  s  ø€ ð Ø$Ù9AÓB¹¨gŸ>œ>¹ÑBÙ9=Ó>¹°#ŸK™K×-¨2Ò-¹Ñ>ÙKSÓ%TÉ8À§o¡o×&;¸Ò&;É8Ñ%TñˆGð ˆFØˆDØ%(§^¡^×%9Ñ%9Ö%;Ñ!�
àÕ(Ø—z‘zØ×3Ñ3Ø×-Ñ-ð&ó ð Ÿ™ GÑ$7Ö8à-4Ñ-@˜Ó*ñ &<ñ ˜4Ð* 6Ñ*ˆFÜ˜&Ô"2×3Ñ3Ø�Ü+¨FÓ3Ð3ùò) CùÚ>ùÚ%Ts   ˆD ¢D
ÁD
r0   )rû   zSequence[schemas.Run]rp  zSequence[schemas.Example]r]   r‚   )r¡   rN  r4  r5  rP  rQ  rR  r³   rò   r²   rÅ   ry   r0   )
r¥   r&  rT  rU  rV  r  rJ  r'  r?  r(  s
   `      @@@r%   Ú_normalize_summary_evaluatorrv  Ì  s¦  û€ ØS€NÜ
×
Ò
˜DÓ
!€CØ&)§n¡n×&:Ñ&:Ô&<Ô=Ñ&<™(˜%“Ñ&<€HÑ=ð Ÿ™×,Ñ,Ô.ôá.‰HˆEØ�9‰9œG×-Ñ-×3Ñ3Ð3÷ 	Ù.òÐö
 Üõ 
ÙPXó
÷ 
ñ 
ô ™HÓD™H�qÐ1CÑ(C—™HÑDÓEÈÓJð1Ø1?Ð0@ÀðCð 	ö ØÐ)¨(¨°1Ð5Ñ5ˆCÜ˜‹oÐô Ô?±hÓ?×?Ñ?À8ØØðPó Dð ˆð	4Ø'ð	4Ø3Lð	4à7÷	4ð 	4ô: *1°°z×)BÑ)BŒG�D˜*Ó%ð 	Ôð ˆð IP×HXÑHXð 	Ôð ˆùów >ùóùò Es   ºE4Á',E:ÂE:ÃF ÃF )r"   z&Callable[[], Coroutine[Any, Any, Any]]r#   r,   r$   r,   r]   r	   )r¥   ræ   )rô   r	   )r¥   r  r]   rø   )r¥   r
   r]   z¢tuple[Union[Callable[[Run, Optional[Example]], _RUNNABLE_OUTPUT], Callable[[Run, Optional[Example]], Awaitable[_RUNNABLE_OUTPUT]]], Optional[Callable[..., dict]]])r¥   r
   r]   zºtuple[Union[Callable[[Sequence[Run], Optional[Example]], _COMPARISON_OUTPUT], Callable[[Sequence[Run], Optional[Example]], Awaitable[_COMPARISON_OUTPUT]]], Optional[Callable[..., dict]]])r®   z;Union[EvaluationResults, dict, str, int, bool, float, list]r]   zUnion[EvaluationResults, dict])r¥   r
   r]   ÚSUMMARY_EVALUATOR_T)Br4   Ú
__future__r   r   r¡   ÚloggingrÕ   Úabcr   Úcollections.abcr   r   r   Ú	functoolsr   Útypingr	   r
   r   r   r   r   Úpydanticr   r   r   r   r   Útyping_extensionsr   r�   r   rm   r   Ú$langsmith.evaluation._key_extractionr   Úlangsmith.schemasr   r   r   r   Ú	getLoggerr0   rW   r&   r(   r9   rA   r`   rd   rI   r¤   r‡   rþ   r‹   rî   ró   rö   rø   r  rœ   rý   r¿   r  rw  rv  r.   r7   r%   Ú<module>rƒ     s  ðÙ Eå "ã Û Û Û Ý ß :Ñ :Ý ÷÷ ÷ TÕ SÝ 'å 'Ý Ý Lß BÓ Bà	×	Ò	˜8Ó	$€ðØ=ðàðð ðð 	ô	ô'ˆyô 'ô6�Y eò 6ô'�yô 'ôT"˜	¨ò "÷(&ñ (&ðV Ð)Ð+<¸dÐBÑCÐ ô ô ð( Ð5°tÐ;Ñ<Ð ôl5˜,ô l5ð^	%ðô	%ð €ô÷Dñ DðN/ðð/ð
 #ô/ðG.Ø
ðG.ðôG.ðT@,Ø
ð@,ðô@,ðFØGðà#ôð8 ØØ	�'—+‘+Ñ	 ¨¯©Ñ 9Ð:ØÐÐ 1Ð1Ñ2ð	4ñð Ø	ˆg�k‰kÑ	˜D §¡Ñ1Ð2ØÐÐ 1Ð1Ñ2ð	4ñð	ñ	Ð õ>r7   