jdZddlmZddlZddlZddlZddlZddlmZddl m Z m Z m Z ddl mZddlmZmZmZmZmZmZddlmZmZmZmZmZdd lmZdd lmZ dd lm!Z!dd l"m#Z#dd l$m%Z%m&Z&m'Z'm(Z(ejRe*Z+ d)dZ,GddeZ-GddedZ.GddeZ/GddedZ0GddZ1ee/e0e2fZ3GddeZ4ee4e2fZ5Gdde1Z6 d*dZ7d Z8d+d!Z9Gd"d#Z: d,d$Z; d-d%Z< d.d&Z= d/d'Z>eee e!jPe e!jNgee/e0ffee?e!jPe?e!jNgee/e0fffZ@d0d(ZAy)1z?This module contains the evaluator classes for evaluating runs.) annotationsN)abstractmethod) Awaitable CoroutineSequence)wraps)AnyCallableLiteralOptionalUnioncast) BaseModel ConfigDictFieldValidationErrormodel_validator) TypedDict run_helpers)schemas)_safe_extract_feedback_keys) SCORE_TYPE VALUE_TYPEExampleRunc tjtd|d|d#t$rtj|cYSwxYw)Nz Cannot call `zC` on an async run evaluator from within a running event loop. Use `z ` instead.)asyncioget_running_loop RuntimeErrorrun)awaitable_factory method_nameasync_method_names h/var/www/html/Fitness-lenito-AI-main/venv/lib/python3.12/site-packages/langsmith/evaluation/evaluator.py_run_async_evaluator_syncr& s` 0  "   }%$$5#6j B  0{{,.//0s(#A Ac(eZdZUdZded< ded<y)Categoryz$A category for categorical feedback.Optional[Union[float, int]]valuestrlabelN__name__ __module__ __qualname____doc____annotations__r%r(r(/s. &&C J&r4r(c@eZdZUdZded< ded< ded< ded<y ) FeedbackConfigzkConfiguration to define a type of feedback. Applied on on the first creation of a `feedback_key`. z0Literal['continuous', 'categorical', 'freeform']typer)minmaxz%Optional[list[Union[Category, dict]]] categoriesNr-r3r4r%r6r68s* ;: $$; $$A55r4r6F)totalceZdZUdZded< dZded< dZded< dZd ed < dZd ed < dZ d ed < e e Z ded< dZ ded< dZded< dZded< dZd ed< edZedddZy)EvaluationResultzEvaluation result.r+keyNrscorerr*zOptional[dict]metadataz Optional[str]comment correction)default_factorydictevaluator_infoz%Optional[Union[FeedbackConfig, dict]]feedback_configOptional[Union[uuid.UUID, str]] source_run_id target_run_idextraforbid)rJafter)modec|jBt|jttfr"t j d|j|S)z:Warn when numeric values are passed via the `value` field.zJNumeric values should be provided in the 'score' field, not 'value'. Got: )r? isinstancer*intfloatloggerwarningselfs r%check_value_non_numericz(EvaluationResult.check_value_non_numericfsE :: *TZZ#u"F NN &  r4)returnr=)r.r/r0r1r2r?r*r@rArBrrDrErFrHrIrJr model_configrrVr3r4r%r=r=Gs H@E:0E:8#Hn#8!G]!2!%J%: 6ND65=AO:A;59M29659M29!E> )H-L'"#r4r=ceZdZUdZded<y)EvaluationResultszqBatch evaluation results. This makes it easy for your evaluator to return multiple metrics at once. zlist[EvaluationResult]resultsNr-r3r4r%rZrZqs $#!r4rZcdeZdZdZe d ddZ d ddZeddZy) RunEvaluatorzEvaluator interface class.Ncy)zEvaluate an example.Nr3)rUr!exampleevaluator_run_ids r% evaluate_runzRunEvaluator.evaluate_runsr4cKtjfd}tjj d|d{S7w)z#Evaluate an example asynchronously.ctjdi5jcdddS#1swYyxYw)Nr3)rhtracing_contextra)current_contextr`r_r!rUsr%_run_with_contextz5RunEvaluator.aevaluate_run.._run_with_contexts3##6o6((g7GH766s4=N)rdget_tracing_contextrrrun_in_executor)rUr!r_r`rgrfs```` @r% aevaluate_runzRunEvaluator.aevaluate_runsH002 I I--/??FWXXXXsAA A AcDt|dd}t|dd}|r|gSgS)aFeedback keys this evaluator emits, when statically known. Used to render per-evaluator experiment progress and to synthesize error feedback when the evaluator raises. Best-effort: empty when the keys can't be determined without running the evaluator. Subclasses that know their keys should override this. The default also resolves evaluators that wrap a single named inner evaluator exposed as ``.evaluator`` (e.g. the LangChain string-evaluator integration). evaluatorNevaluation_name)getattr)rUinnernames r% feedback_keyszRunEvaluator.feedback_keyss0k40u/6v%2%r4NNr!rr_Optional[Example]r`Optional[uuid.UUID]rW*Union[EvaluationResult, EvaluationResults]rW list[str]) r.r/r0r1rrarjpropertyrqr3r4r%r]r]|s$&*04 # ###. # 4 ##&*04 Y  Y# Y. Y 4 Y & &r4r]cHeZdZUdZded< ded< dZded< dZd ed <y) ComparisonEvaluationResultzFeedback scores for the results of comparative evaluations. These are generated by functions that compare two or more runs, returning a ranking or other feedback. r+r>z'dict[Union[uuid.UUID, str], SCORE_TYPE]scoresNrGrHz6Optional[Union[str, dict[Union[uuid.UUID, str], str]]]rA)r.r/r0r1r2rHrAr3r4r%r{r{s8 H@ 33459M296FJG CJr4r{ceZdZdZ d d dZ d ddZ ddZ ddZeddZ eddZ d ddZ d dfd Z d dd Z dd ZxZS)DynamicRunEvaluatoraA dynamic evaluator that wraps a function and transforms it into a `RunEvaluator`. This class is designed to be used with the `@run_evaluator` decorator, allowing functions that take a `Run` and an optional `Example` as arguments, and return an `EvaluationResult` or `EvaluationResults`, to be used as instances of `RunEvaluator`. Attributes: func (Callable): The function that is wrapped by this evaluator. c2t||_t|\}|rt|\}d fd }t||ddlm}|*|j |||_t|dd|_ tj|r8| td|j |||_t|dd|_ y|j tttt t"gt$f|||_t|dd|_ y) zInitialize the `DynamicRunEvaluator` with a given function. Args: func (Callable): A function that takes a `Run` and an optional `Example` as arguments, and returns a dict or `ComparisonEvaluationResult`. cf|S|jd|jd\}}}|S)Nr!r_getinputs_ traced_inputsprepare_inputss r%process_inputsz4DynamicRunEvaluator.__init__..process_inputss>% $2 5!6::i#8% !Q=! r4rrNrr.r~Func was provided as a coroutine function, but afunc was also provided. If providing both, func should be a regular function to avoid ambiguity.rrDrWrD)r_feedback_keys_normalize_evaluator_funcr langsmithrensure_traceableafuncrn_nameinspectiscoroutinefunction TypeErrorrr rr r_RUNNABLE_OUTPUTfuncrUrrrrrs @r%__init__zDynamicRunEvaluator.__init__s#(:$?!:4!@~ &?&F #UN ! d D)  $55n6DJ! 4IJDJ  & &t , 3 %55^6DJ!z3HIDJ#44XsHW$568HHI4P-5DI!z3HIDJr4c<ttrjs|_S stddvr|r|jd<t fddDrtdtdid|iS#t $r}td|d}~wwxYw) NziExpected an EvaluationResult object, or dict with a metric 'key' and optional 'score'; got empty result: r>c3&K|]}|v ywNr3).0kresults r% z@DynamicRunEvaluator._coerce_evaluation_result..sJ,Iq1F?,I)r?r*rAzrExpected an EvaluationResult object, or dict with a metric 'key' and optional 'score' or categorical 'value'; got rHz[Expected an EvaluationResult object, or dict with a metric 'key' and optional 'score'; got r3)rOr=rH ValueErrorrallr)rUrrH allow_no_keyes ` r%_coerce_evaluation_resultz-DynamicRunEvaluator._coerce_evaluation_results f. /'''4$M  FFLXOF"| $ u J,IJJ OOUhX$Q &P&PQ Q 44:8=  sAA>> BBBcd|vrB|j}|dDcgc]}|j||c}|d<tdi|S|jtt||dScc}w)Nr[)rHT)rHrr3)copyrrZrrD)rUr[rHcprs r%_coerce_evaluation_resultsz.DynamicRunEvaluator._coerce_evaluation_results%s  B!++A..q .N+ByM%*r* *-- w }4.  sA)ct|tr|js||_|St|}|j ||Sr)rOr=rH_format_evaluator_resultr)rUrrHs r%_format_resultz"DynamicRunEvaluator._format_result6sB f. /'''4$M)&1..v}EEr4ct|dSzCheck if the evaluator function is asynchronous. Returns: bool: `True` if the evaluator function is asynchronous, `False` otherwise. rhasattrrTs r%is_asynczDynamicRunEvaluator.is_asyncDtW%%r4c|jS)zFeedback keys this evaluator emits, statically inferred from its source. Best-effort: empty when keys can't be determined without running the function (e.g. a dynamically computed key or unavailable source). )rrTs r%rqz!DynamicRunEvaluator.feedback_keysMs"""r4c(tdstfdddS|tj}dji}t ddrt j|d<j||d  }j||S) aEvaluate a run using the wrapped function. This method directly invokes the wrapped function with the provided arguments. Args: run (Run): The run to be evaluated. example (Optional[Example]): An optional example to be used in the evaluation. Returns: Union[EvaluationResult, EvaluationResults]: The result of the evaluation. rc(jSr)rj)r_r!rUsr%z2DynamicRunEvaluator.evaluate_run..is**38r4rarjNrI session_id experimentrun_idr@langsmith_extra) rr&uuiduuid4idrnr+rrr)rUr!r_r`r@rs``` r%raz DynamicRunEvaluator.evaluate_runVs"tV$,8   ##zz| $3SVV#< 3 d +%(%8H\ "  '7XN  ""6+;<rrTs r%__repr__zDynamicRunEvaluator.__repr__s&tzzl!44r4r)rXCallable[[Run, Optional[Example]], Union[_RUNNABLE_OUTPUT, Awaitable[_RUNNABLE_OUTPUT]]]rzIOptional[Callable[[Run, Optional[Example]], Awaitable[_RUNNABLE_OUTPUT]]])F)rzUnion[EvaluationResult, dict]rH uuid.UUIDrboolrWr=)r[zUnion[dict, EvaluationResults]rHrrWrv)rzMUnion[EvaluationResult, EvaluationResults, dict, str, int, bool, float, list]rHrrWrvrWrrwrrrs)r!rr_rtr`ru)r!rr_rtrWrvrWr+)r.r/r0r1rrrrryrrqrarjrr __classcell__)rs@r%r~r~sO, 9J 9J 9J~# -!   < / !  4  " F  F ! F 4 F&&##&*04 != !=#!=. != 4 !=L&*04 = =#=. =D6://!2/ 3/"5r4r~ct|S)zmCreate a run evaluator from a function. Decorator that transforms a function into a `RunEvaluator`. )r~rs r% run_evaluatorrs t $$r4i'c^t|}t|tkDr|dtdz dz}|S)Nz...))reprlen_MAXSIZE)objss r% _maxsize_reprrs1 S A 1v n1  & Hr4ceZdZdZ d d dZed dZ d ddZ d ddZ d ddZ ddZ e dd Z dd Z y)DynamicComparisonRunEvaluatorz4Compare predictions (as traces) from 2 or more runs.Nc t|\}|rt|\}d fd }t||ddlm}|*|j |||_t |dd|_tj|r8| td|j |||_t |dd|_y|j tttttt gt"f|||_t |dd|_y) zInitialize the `DynamicRunEvaluator` with a given function. Args: func (Callable): A function that takes a `Run` and an optional `Example` as arguments, and returns an `EvaluationResult` or `EvaluationResults`. cf|S|jd|jd\}}}|S)Nrunsr_rrs r%rz>DynamicComparisonRunEvaluator.__init__..process_inputss>% $2 6"FJJy$9% !Q=! r4rrNrr.r~rr)$_normalize_comparison_evaluator_funcrrrrrrnrrrrrr rrr r_COMPARISON_OUTPUTrrs @r%rz&DynamicComparisonRunEvaluator.__init__s(("Fd!K~ &J5&Q #UN ! d D)  $55n6DJ! 4IJDJ  & &t , 3 %55^6DJ!z3HIDJ#44!#(9:*,  .5 DI!z3HIDJr4ct|dSrrrTs r%rz&DynamicComparisonRunEvaluator.is_async rr4ctdstfdddStj}j }j ||d}j ||S)zCompare runs to score preferences. Args: runs: A list of runs to compare. example: An optional example to be used in the evaluation. rc(jSr) acompare_runs)r_rrUsr%rz.s**49r4 compare_runsrrtagsr)rr&rr _get_tagsr_format_resultsrUrr_rHrrs``` r%rz*DynamicComparisonRunEvaluator.compare_runssxtV$,9   ~~d#  '4dC  ##FM4@@r4cKt|ds|j||Stj}|j |}|j ||||dd{}|j |||S7w)aEvaluate a run asynchronously using the wrapped async function. This method directly invokes the wrapped async function with the provided arguments. Args: runs (Run): The runs to be evaluated. example (Optional[Example]): An optional example to be used in the evaluation. Returns: ComparisonEvaluationResult: The result of the evaluation. rrrN)rrrrrrrrs r%rz+DynamicComparisonRunEvaluator.acompare_runs,s tW%$$T73 3 ~~d#zz  '4dC"   ##FM4@@  sAA: A8!A:c&|j||S)aMake the evaluator callable, allowing it to be used like a function. This method enables the evaluator instance to be called directly, forwarding the call to `evaluate_run`. Args: run (Run): The run to be evaluated. example (Optional[Example]): An optional example to be used in the evaluation. Returns: ComparisonEvaluationResult: The result of the evaluation. )r)rUrr_s r%rz&DynamicComparisonRunEvaluator.__call__Hs  w//r4c"d|jdS)rz>r4cg}|D]^}|jdt|jzt|dds8|jdt|jz`|S)zExtract tags from runs.zrun:rNz experiment:)appendr+rrnr)rrr!s r%rz'DynamicComparisonRunEvaluator._get_tags]sYC KKSVV, -sL$/ MC,??@ r4ct|tr|js||_|St|tr9t ||Dcic]\}}|j |c}}|j |d}n4t|trd|vr |j |d<nd|}t| tdid|i|Scc}}w#t$r}td||d}~wwxYw)N)r|r>rHr>zXExpected 'dict', 'list' or 'ComparisonEvaluationResult' result object. Received: result=rHzExpected a dictionary with a 'key' and dictionary of scores mappingrun IDs to numeric scores, or ComparisonEvaluationResult object, got r3) rOr{rHlistziprrrDrr)rUrrHrr!r?msgrs r%rz-DynamicComparisonRunEvaluator._format_resultshs f8 9'''4$M  %;>tV;LM;LZS%3665=;LMzz!.F  %F" $ u -%+I/ S/ ! -"M >J >J >J@&&AEA!A,=A #A6AEA!A,=A #A:AE0!0,=0 #0"?"="!" " $ "r4rct|S)z.Create a comaprison evaluator from a function.)rrs r%comparison_evaluatorrs ) ..r4c d tj jjDcgc]!\}}|j|j k7s |#}}} jjDcgc]-\}}|j tjjur|/c}} |r6t fd|Ds2t|Dcgc] }| vs| c}dk7rd d}t|t fd|Dr|ddgk(rdfStjrF d fd  dfd }td r td n |j|_|fS d fd dfd }td r td n |j|_|fScc}}wcc}}wcc}w)N)r!r_routputsreference_outputs attachmentsc32K|]}|vxs|vywrr3rpnameargs_with_defaultssupported_argss r%rz,_normalize_evaluator_func..) PXuE^ # Bu0B'B BPXUInvalid evaluator function. Must have at least one argument. Supported arguments are . Please see https://docs.smith.langchain.com/evaluation/how_to_guides/evaluation/evaluate_llm_application#use-custom-evaluatorsc32K|]}|vxs|vywrr3rs r%rz,_normalize_evaluator_func..(LT5>5,>#>>Hrr!r_c|||r |jni|jxsi|r|jxsini|r|jxsinid}i}g}i}jj D]e\}}||vs |j |j |jfvr|j||n||||<|dvrt||n||||<g|||fSN)r!r_rrrr)r!r_ rrr parametersitemskindPOSITIONAL_OR_KEYWORDPOSITIONAL_ONLYrr r!r_arg_mapkwargsargsr param_nameparamsigs r%_prepare_inputsz2_normalize_evaluator_func.._prepare_inputs&07gnnR"{{0b@G7#6#6#<"RBI)>Br  " ),)=)=)?%J!W, ::!77!11*!KK (;<181DF:. *-??*'**=>!(!4&j1*@V]22r4cJK||\}}}|i|d{S7wrr3r!r_rrrrrs r%awrapperz+_normalize_evaluator_func..awrappers2%4C$A!vq!4262222 #!#r.c|||r |jni|jxsi|r|jxsini|r|jxsinid}i}g}i}jj D]e\}}||vs |j |j |jfvr|j||n||||<|dvrt||n||||<g|||fSr r rs r%rz2_normalize_evaluator_func.._prepare_inputsrr4c.||\}}}|i|Srr3rs r%wrapperz*_normalize_evaluator_func..wrappers&$3C$A!vqT,V,,r4)r!rr_rtrWtuple[list, dict, dict])r!rr_rtrWrr signaturerrr VAR_KEYWORDdefault Parameteremptyrrrrrrnr. rrpall_argsarrr"rrrrs ` @@@@r%rrs"N   D !C&)nn&:&:&<X&<(%!--@W& 3 3#4 3! 34,j)&&   o. . 3 3#4 3( 3> - 4,j)%%   _- -kYEs!F:F:2G G Gc d tj jjDcgc]!\}}|j|j k7s |#}}} jjDcgc]-\}}|j tjjur|/c}} |r6t fd|Ds2t|Dcgc] }| vs| c}dk7rd d}t|t fd|Dr|ddgk(rdfStjrF d fd  dfd }td r td n |j|_|fS d fd  dfd }td r td n |j|_|fScc}}wcc}}wcc}w)Nrr_rrrc32K|]}|vxs|vywrr3rs r%rz7_normalize_comparison_evaluator_func..3rrrrrc32K|]}|vxs|vywrr3rs r%rz7_normalize_comparison_evaluator_func..Ar rrr_c|||r |jni|Dcgc]}|jxsic}|r|jxsinid}i}g}i} jjD]e\}}||vs |j|j |j fvr|j||n||||<|dvrt||n||||<g|||fScc}wNr/)rr_ rrrrrrrrr rr_r!rrrrrrrs r%rz=_normalize_comparison_evaluator_func.._prepare_inputsK!&07gnnR=ABTc 1r 1TBBI)>Br  " ),)=)=)?%J!W, ::!77!11*!KK (;<181DF:. *-@@*'**=>!(!4&j1*@V]22+ CC cJK||\}}}|i|d{S7wrr3rr_rrrrrs r%rz6_normalize_comparison_evaluator_func..awrapperis2%4D'$B!vq!4262222rr.c|||r |jni|Dcgc]}|jxsic}|r|jxsinid}i}g}i} jjD]e\}}||vs |j|j |j fvr|j||n||||<|dvrt||n||||<g|||fScc}wr3r4r5s r%rz=_normalize_comparison_evaluator_func.._prepare_inputsxr6r7c.||\}}}|i|Srr3r9s r%r"z5_normalize_comparison_evaluator_func..wrappers(%4D'$B!vqT,V,,r4)rrr_rtrWr#)rrr_rtrWrr$r*s ` @@@@r%rr!s9SN   D !C&)nn&:&:&<X&<(%!--@W&.s71:a&sz8Expected a list of dicts or EvaluationResults. Received .r[r*zZExpected a dict, str, bool, int, float, list, EvaluationResult, or EvaluationResults. Got ) rOrrQrPrrrr+rD)rs r%rrs&4,-6"* M);;A( D   FD !777J6(RST V$ M FC 6" M FD ! M &&,X /  r4c d tjjjDcgc]\}}| }}}jjDcgc]-\}}|jtj j ur|/c}}|r6t fd|Ds=t|Dcgc] }|vs| c}dk7rd d}|r |d|dz }t|t fd|Dr|dd gk(rS d fd }td rtd |_ |S|j|_ |Scc}}wcc}}wcc}w) Nrexamplesrrrc32K|]}|vxs|vywrr3rs r%rz/_normalize_summary_evaluator..rrrrr?z Received arguments c3&K|]}|v ywrr3)rrrs r%rz/_normalize_summary_evaluator..s?hU.(hrrrBc |||Dcgc]}|jc}|Dcgc]}|jxsic}|Dcgc]}|jxsic}d}i}g} jjD]K\}}||vs |j|j |j fvr|j||D||||<M |i|} t| tr| St| Scc}wcc}wcc}w)NrA) rrrrrrrrrOr=r) rrBr_r!rrrrrrrrs r%r"z-_normalize_summary_evaluator..wrappers$9ABg7>>B9=>#CKK-2->KS%T8goo&;&;8%T GFD%(^^%9%9%;! E(zz33--& GJ$78-4Z-@z*&<4*6*F&"23 +F3 3)C>%TsC+C0 C5 r.)rzSequence[schemas.Run]rBzSequence[schemas.Example]rWrv) rr%rrr'r(r)rrrrrnr.) rrr+r,r-rr"rrrs ` @@@r%_normalize_summary_evaluatorrFsSN   D !C&)nn&:&:&<=&<(%&Es E %2E; EE)r"z&Callable[[], Coroutine[Any, Any, Any]]r#r+r$r+rWr )rr)rr )rrrWr)rr rWztuple[Union[Callable[[Run, Optional[Example]], _RUNNABLE_OUTPUT], Callable[[Run, Optional[Example]], Awaitable[_RUNNABLE_OUTPUT]]], Optional[Callable[..., dict]]])rr rWztuple[Union[Callable[[Sequence[Run], Optional[Example]], _COMPARISON_OUTPUT], Callable[[Sequence[Run], Optional[Example]], Awaitable[_COMPARISON_OUTPUT]]], Optional[Callable[..., dict]]])rz;Union[EvaluationResults, dict, str, int, bool, float, list]rWzUnion[EvaluationResults, dict])rr rWSUMMARY_EVALUATOR_T)Br1 __future__rrrloggingrabcrcollections.abcrrr functoolsrtypingr r r r r rpydanticrrrrrtyping_extensionsrrrrdr$langsmith.evaluation._key_extractionrlangsmith.schemasrrrr getLoggerr.rRr&r(r6r=rZr]rDrr{rr~rrrrrrrrrrGrFr3r4r%rSsE" ::TS''LBB   8 $ =    'y' 6Ye 6'y'T" "(&(&V)+r4