~tjbdZddlmZddlZddlmZddlZddlZddl Z ddl Z ddl Z ddl m Z mZmZmZmZddlmZmZmZmZmZmZmZmZddlZddlmZddlmZmZddlmZ dd lm!Z"dd l#m$Z%dd l&m'Z'dd l(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;mm?Z?m@Z@mAZAmBZBerddlCZDddlEmFZFeDjGZGneZGe jHeIZJeeeKgeeKfeeKeKgeeKffZL dVdWd0ZM dXdYd4ZN dZd[d7ZOGd8d9e/ZPGd:d/ZQ d\d]dEZRd^dGZSd_dIZTd1dJd`dLZUedMZVdadQZWdbdUZXdS)czV2 Evaluation Interface.) annotationsN) AsyncIterable AsyncIterator AwaitableIterableSequence) TYPE_CHECKINGAnyCallableLiteralOptionalTypeVarUnioncast) run_helpers) run_treesschemas)r)utils)_aiter) _warn_once) AEVALUATOR_TDATA_T EVALUATOR_TExperimentResultRow_collect_evaluator_keys_evaluators_include_attachments_ExperimentManagerMixin_ForwardResults_get_target_args_is_langchain_runnable_load_examples_map_load_experiment _load_tqdm_load_traces_for_experiment _resolve_data_resolve_evaluators_resolve_experiment_resolve_num_examples_target_include_attachments _to_pandas_wrap_summary_evaluators)SUMMARY_EVALUATOR_TEvaluationResultEvaluationResults RunEvaluator)RunnableTlogdataNUnion[DATA_T, AsyncIterable[schemas.Example], Iterable[schemas.Example], None] evaluators4Optional[Sequence[Union[EVALUATOR_T, AEVALUATOR_T]]]summary_evaluators'Optional[Sequence[SUMMARY_EVALUATOR_T]]metadataOptional[dict]experiment_prefix Optional[str] descriptionmax_concurrency Optional[int]num_repetitionsintclientOptional[langsmith.Client]blockingbool experiment6Optional[Union[schemas.TracerSession, str, uuid.UUID]]upload_resultserror_handlingLiteral['log', 'ignore']targetVUnion[ATARGET_T, AsyncIterable[dict], Runnable, str, uuid.UUID, schemas.TracerSession]kwargsr returnAsyncExperimentResultscKt|ttjtjfr|dkt | | t |t |d}t|r>dtd| Dd}t|t|ttjfr|n|j }td|dt|f||||| | d|d {VSt|t tfrd }t||rd |d }t||sd }t|| r|rd| d|}t|| st#dtd|dt%|||||||||| | | | | d {VS)aEvaluate an async target system on a given dataset. Args: target (AsyncCallable[[dict], dict] | AsyncIterable[dict] | Runnable | EXPERIMENT_T | Tuple[EXPERIMENT_T, EXPERIMENT_T]): The target system or experiment(s) to evaluate. Can be an async function that takes a `dict` and returns a `dict`, a langchain `Runnable`, an existing experiment ID, or a two-tuple of experiment IDs. data (Union[DATA_T, AsyncIterable[schemas.Example]]): The dataset to evaluate on. Can be a dataset name, a list of examples, an async generator of examples, or an async iterable of examples. evaluators (Optional[Sequence[EVALUATOR_T]]): A list of evaluators to run on each example. summary_evaluators (Optional[Sequence[SUMMARY_EVALUATOR_T]]): A list of summary evaluators to run on the entire dataset. metadata (Optional[dict]): Metadata to attach to the experiment. experiment_prefix (Optional[str]): A prefix to provide for your experiment name. description (Optional[str]): A description of the experiment. max_concurrency (int | None): The maximum number of concurrent evaluations to run. If `None` then no limit is set. If `0` then no concurrency. num_repetitions (int): The number of times to run the evaluation. Each item in the dataset will be run and evaluated this many times. client (Optional[langsmith.Client]): The LangSmith client to use. blocking (bool): Whether to block until the evaluation is complete. experiment (Optional[schemas.TracerSession]): An existing experiment to extend. If provided, `experiment_prefix` is ignored. For advanced usage only. error_handling (str, default="log"): How to handle individual run errors. `'log'` will trace the runs with the error message as part of the experiment, `'ignore'` will not count the run as part of the experiment at all. Returns: An async iterator over the experiment results. Environment: - `LANGSMITH_TEST_CACHE`: If set, API calls will be cached to disk to save time and cost during testing. Recommended to commit the cache files to your repository for faster CI/CD runs. Requires the `'langsmith[vcr]'` package to be installed. Examples: >>> from typing import Sequence >>> from langsmith import Client, aevaluate >>> from langsmith.schemas import Example, Run >>> client = Client() >>> dataset = client.clone_public_dataset( ... "https://smith.langchain.com/public/419dcab2-1d66-4b94-8901-0357ead390df/d" ... ) >>> dataset_name = "Evaluate Examples" Basic usage: >>> def accuracy(run: Run, example: Example): ... # Row-level evaluator for accuracy. ... pred = run.outputs["output"] ... expected = example.outputs["answer"] ... return {"score": expected.lower() == pred.lower()} >>> def precision(runs: Sequence[Run], examples: Sequence[Example]): ... # Experiment-level evaluator for precision. ... # TP / (TP + FP) ... predictions = [run.outputs["output"].lower() for run in runs] ... expected = [example.outputs["answer"].lower() for example in examples] ... # yes and no are the only possible answers ... tp = sum([p == e for p, e in zip(predictions, expected) if p == "yes"]) ... fp = sum([p == "yes" and e == "no" for p, e in zip(predictions, expected)]) ... return {"score": tp / (tp + fp)} >>> import asyncio >>> async def apredict(inputs: dict) -> dict: ... # This can be any async function or just an API call to your app. ... await asyncio.sleep(0.1) ... return {"output": "Yes"} >>> results = asyncio.run( ... aevaluate( ... apredict, ... data=dataset_name, ... evaluators=[accuracy], ... summary_evaluators=[precision], ... experiment_prefix="My Experiment", ... description="Evaluate the accuracy of the model asynchronously.", ... metadata={ ... "my-prompt-version": "abcd-1234", ... }, ... ) ... ) # doctest: +ELLIPSIS View the evaluation results for experiment:... Evaluating over only a subset of the examples using an async generator: >>> async def example_generator(): ... examples = client.list_examples(dataset_name=dataset_name, limit=5) ... for example in examples: ... yield example >>> results = asyncio.run( ... aevaluate( ... apredict, ... data=example_generator(), ... evaluators=[accuracy], ... summary_evaluators=[precision], ... experiment_prefix="My Subset Experiment", ... description="Evaluate a subset of examples asynchronously.", ... ) ... ) # doctest: +ELLIPSIS View the evaluation results for experiment:... Streaming each prediction to more easily + eagerly debug. >>> results = asyncio.run( ... aevaluate( ... apredict, ... data=dataset_name, ... evaluators=[accuracy], ... summary_evaluators=[precision], ... experiment_prefix="My Streaming Experiment", ... description="Streaming predictions for debugging.", ... blocking=False, ... ) ... ) # doctest: +ELLIPSIS View the evaluation results for experiment:... >>> async def aenumerate(iterable): ... async for elem in iterable: ... print(elem) >>> asyncio.run(aenumerate(results)) Running without concurrency: >>> results = asyncio.run( ... aevaluate( ... apredict, ... data=dataset_name, ... evaluators=[accuracy], ... summary_evaluators=[precision], ... experiment_prefix="My Experiment Without Concurrency", ... description="This was run without concurrency.", ... max_concurrency=0, ... ) ... ) # doctest: +ELLIPSIS View the evaluation results for experiment:... Using Async evaluators: >>> async def helpfulness(run: Run, example: Example): ... # Row-level evaluator for helpfulness. ... await asyncio.sleep(5) # Replace with your LLM API call ... return {"score": run.outputs["output"] == "Yes"} >>> results = asyncio.run( ... aevaluate( ... apredict, ... data=dataset_name, ... evaluators=[helpfulness], ... summary_evaluators=[precision], ... experiment_prefix="My Helpful Experiment", ... description="Applying async evaluators example.", ... ) ... ) # doctest: +ELLIPSIS View the evaluation results for experiment:... !!! warning "Behavior changed in `langsmith` 0.2.0" 'max_concurrency' default updated from None (no limit on concurrency) to 0 (no concurrency at all). r1)r@rFrHr;r3zReceived invalid arguments. c3$K|] \}}||V dSN).0kvs [/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/langsmith/evaluation/_arunner.py zaevaluate..s+AAtq!qAAAAAAAz? should not be specified when target is an existing experiment.z,Running evaluation over existing experiment z...)r5r7r9r>rBrDNzRunning a comparison of two existing experiments asynchronously is not currently supported. Please use the `evaluate()` method instead and make sure that your evaluators are defined as synchronous functions.zReceived unsupported arguments zC. These arguments are not supported when creating a new experiment.zDMust specify 'data' when running evaluations over a target function.zeExpected at most one of 'experiment' or 'experiment_prefix', but both were provided. Got: experiment=z, experiment_prefix=z&'upload_results' parameter is in beta.z&Running evaluation over target system ) r3r5r7r9r;r=r>r@rBrDrFrHrI) isinstancestruuidUUIDr TracerSessionrEanyvaluestupleitems ValueErroridloggerdebugaevaluate_existinglistr _aevaluate)rKr3r5r7r9r;r=r>r@rBrDrFrHrIrM invalid_argsmsg target_ids rW aevaluatermLsF&3 7+@ABBE .2z**"00!%&7!8!8JJ    |""$$ % % "DAA|'9'9';';AAAAADDD  S// !(#ty1ABBQFF  RIRRRSSS'   !1+             FT5M * **  N oo #  9f 9 9 9 oo  Too  )  S) S S?P S S oo A ? @ @ @ IfIIIJJJ !1/#++!))          rYF load_nested,Union[str, uuid.UUID, schemas.TracerSession]c  K|ptj}t|tjr|n2t jtjt||d{V}t jtjt|||d{V} t jtjt||d{V fd| D} t| | ||||||| d{VS)aEvaluate existing experiment runs asynchronously. Args: experiment (Union[str, uuid.UUID]): The identifier of the experiment to evaluate. evaluators (Optional[Sequence[EVALUATOR_T]]): Optional sequence of evaluators to use for individual run evaluation. summary_evaluators (Optional[Sequence[SUMMARY_EVALUATOR_T]]): Optional sequence of evaluators to apply over the entire dataset. metadata (Optional[dict]): Optional metadata to include in the evaluation results. max_concurrency (int | None): The maximum number of concurrent evaluations to run. If `None` then no limit is set. If `0` then no concurrency. client (Optional[langsmith.Client]): Optional Langsmith client to use for evaluation. load_nested: Whether to load all child runs for the experiment. Default is to only load the top-level root runs. blocking (bool): Whether to block until evaluation is complete. Returns: An async iterator over the experiment results. Examples: Define your evaluators >>> from typing import Sequence >>> from langsmith.schemas import Example, Run >>> def accuracy(run: Run, example: Example): ... # Row-level evaluator for accuracy. ... pred = run.outputs["output"] ... expected = example.outputs["answer"] ... return {"score": expected.lower() == pred.lower()} >>> def precision(runs: Sequence[Run], examples: Sequence[Example]): ... # Experiment-level evaluator for precision. ... # TP / (TP + FP) ... predictions = [run.outputs["output"].lower() for run in runs] ... expected = [example.outputs["answer"].lower() for example in examples] ... # yes and no are the only possible answers ... tp = sum([p == e for p, e in zip(predictions, expected) if p == "yes"]) ... fp = sum([p == "yes" and e == "no" for p, e in zip(predictions, expected)]) ... return {"score": tp / (tp + fp)} Load the experiment and run the evaluation. >>> import asyncio >>> import uuid >>> from langsmith import Client, aevaluate, aevaluate_existing >>> client = Client() >>> dataset_name = "__doctest_aevaluate_existing_" + uuid.uuid4().hex[:8] >>> dataset = client.create_dataset(dataset_name) >>> example = client.create_example( ... inputs={"question": "What is 2+2?"}, ... outputs={"answer": "4"}, ... dataset_id=dataset.id, ... ) >>> async def apredict(inputs: dict) -> dict: ... await asyncio.sleep(0.001) ... return {"output": "4"} >>> results = asyncio.run( ... aevaluate( ... apredict, data=dataset_name, experiment_prefix="doctest_experiment" ... ) ... ) # doctest: +ELLIPSIS View the evaluation results for experiment:... >>> experiment_id = results.experiment_name >>> # Consume all results to ensure evaluation is complete >>> async def consume_results(): ... result_list = [r async for r in results] ... return len(result_list) > 0 >>> asyncio.run(consume_results()) True >>> import time >>> time.sleep(3) >>> results = asyncio.run( ... aevaluate_existing( ... experiment_id, ... evaluators=[accuracy], ... summary_evaluators=[precision], ... ) ... ) # doctest: +ELLIPSIS View the evaluation results for experiment:... >>> client.delete_dataset(dataset_id=dataset.id) N)rnc*g|]}|jSrS)reference_example_id)rTrundata_maps rW z&aevaluate_existing..s! ? ? ?3HS- . ? ? ?rY)r3r5r7r9r>rBrDrF) rget_cached_clientrZrr^ aitertools aio_to_thread contextvars copy_contextr"r$r!ri) rFr5r7r9r>rBrnrDprojectrunsr3rts @rWrgrgWs~ 4y244F j'"7 8 8 *(**,|||d{V}n0| tt>||d{V}|r|!|d{V}n<|r|"||d{V}|r|!|d{V}tG|}| r|$d{V|cdddS#1swxYwYdS)N __aiter__rr1) rBr9rFr=r@evaluator_keysr|include_attachmentsreuse_attachmentsrHrIz.yaml) ignore_hostsr>)%asyncioiscoroutinefunctionhasattr iscoroutinerr rtrvrrrRunrwrxryrzr'rAr)r_AsyncExperimentManagerrastartls_utils get_cache_dirget_dataset_idpathlibPathwith_optional_cacheapi_url awith_predictions_and_evaluators ATARGET_Tawith_predictionsawith_summary_evaluatorsawith_evaluatorsrOwait)rKr3r5r7r9r;r=r>r@rBrDrFrHrIis_async_targetr| experiment_num_include_attachmentsmanager cache_dirdsid cache_pathresultss rWriris$ #F++ * FK ( ( TW-@AQAQASAS-T-T * !& ) )  -r+--F" K44Xgk-BF(K(KD(6 ""  K"#F++' 334, 3"3'.z:: 3a7),CCaG%%    fhh      G&t,,I++--------\),,$~~~=   %j?O P P P  U  ' H HF++Z!I!!!( 9 9F++_!:!!" U ' @ @AS T TTTTTTT  ' 8 8!9!!" U ' @ @AS T TTTTTTT(11  !,,.. /s3C-J--J14J1ceZdZdZ dTdUfd# ZdVd'ZdWd)ZdXd+ZdYd-ZdZd/Z d[d0Z dVd1Z d\d]d7Z d\d^d8Z dd9d_d:Zd`d=Zdad?ZdbdAZ dcdddCZ d\dedEZdfdJZdgdKZdhdLZdidMZdjdOZdkdSZxZS)lraManage the execution of experiments asynchronously. Supports lazily running predictions and evaluations in parallel to facilitate result streaming and early debugging. Args: data (DATA_T): The data used for the experiment. Can be a dataset name or ID OR a generator of examples. runs (Optional[Iterable[schemas.Run]]): The runs associated with the experiment predictions. experiment (Optional[schemas.TracerSession]): The tracer session associated with the experiment. experiment_prefix (Optional[str]): The prefix for the experiment name. description (Optional[str]): The description for the experiment. metadata (Optional[dict]): Additional metadata for the experiment. client (Optional[langsmith.Client]): The Langsmith client used for the experiment. evaluation_results (Optional[Iterable[EvaluationResults]]): The evaluation sresults for the experiment. summary_results (Optional[Iterable[EvaluationResults]]): The aggregate results for the experiment. num_repetitions (Optional[int], default=1): The number of repetitions for the experiment. include_attachments (Optional[bool], default=False): Whether to include attachments. This is used for when we pull the examples for the experiment. reuse_attachments (Optional[bool], default=False): Whether to reuse attachments from examples. This is True if we need to reuse attachments across multiple target/evaluator functions. upload_results (Optional[bool], default=True): Whether to upload results to Langsmith. attachment_raw_data_dict (Optional[dict]): A dictionary to store raw data for attachments. Only used if we reuse attachments across multiple target/evaluator functions. error_handling (str, default="log"): How to handle individual run errors. `'log'` will trace the runs with the error message as part of the experiment, `'ignore'` will not count the run as part of the experiment at all. Nr1FTr2rF+Optional[Union[schemas.TracerSession, str]]r9r:r|BOptional[Union[Iterable[schemas.Run], AsyncIterable[schemas.Run]]]rBrCevaluation_results*Optional[AsyncIterable[EvaluationResults]]summary_resultsr=r<r@rA num_examplesr?rOptional[list[str]]rrErrHattachment_raw_data_dictrIrJr3r}cdt||||||_| |_d|_|t j|nd|_||_||_ | |_ | | nt||j |_ | |_| |_||_||_||_dS)N)rFr9rBr=)rB)super__init___data_evaluator_keys _examplesrwensure_async_iterator_runs_evaluation_results_summary_results_num_repetitionsr(rB _num_examples_include_attachments_reuse_attachments_upload_results_attachment_raw_data_dict_error_handling)selfr3rFr9r|rBrrr=r@rrrrrHrrI __class__s rWrz _AsyncExperimentManager.__init__Qs( !#     -CG6:6FJ ,T 2 2 2D $6 / /' L&tDK@@@  %8!"3-)A&-rYexampleschemas.ExamplerNc t|dr|js|Si}|jD]w\}}|jft |j|z|jvrH|dt j|jt |j|z|dd||<r|||<xtj |j|j |j |j |j |j|j|j||j|j S)aReset attachment readers for an example. This is only in the case that an attachment is going to be used by more than 1 callable (target + evaluators). In that case we keep a single copy of the attachment data in self._attachment_raw_data_dict, and create readers from that data. This makes it so that we don't have to keep copies of the same data in memory, instead we can just create readers from the same data. attachmentsN presigned_url mime_typerreaderr rd created_at dataset_idinputsoutputsr9 modified_at source_run_idr _host_url _tenant_id)rrrbrr[rdioBytesIOrExamplerrrrr9rrrr)rrnew_attachmentsname attachments rW_reset_example_attachmentsz2_AsyncExperimentManager._reset_example_attachmentss&w .. g6I N=? ' 3 9 9 ; ; 3 3 D*.: OOd*d.LLL&0%@ j6s7:7MN",K!8 ))%%)3%%z))>O%+!/'')    rYAsyncIterator[schemas.Example]c,Kjtjjj_jr?j8tjj\}_d|2d{V_j dkrJdj2d{Vtfdtj D_tjtj jdtj\_}|S)NrBrcKi|3d{V }|jpiD]5\}}t|j|z|d6Z6S)Nr)rrbr[rdread)rTervalues rW z9_AsyncExperimentManager.aget_examples..s222222222() (;'B'B'D'D22$eII$eHo&:&:&<&<22222sAr1c"Kg|3d{V}| 6SrRrS)rTrs rWruz9_AsyncExperimentManager.aget_examples..s. M M M M M M M M7 M M M MscFg|]}tfdDS)c:g|]}|SrS)r)rTrrs rWruzD_AsyncExperimentManager.aget_examples...s7$+!% ? ? H HrY)async_iter_from_list)rT_ examples_listrs rWruz9_AsyncExperimentManager.aget_examples..s[ -/<rYlock)r_aresolve_datarrBrrrrwateerasync_chain_from_iterablerangerrLock)r examples_copy examples_iterrs` @rW aget_examplesz%_AsyncExperimentManager.aget_examplessc > !+ {$($=DN & 4+I+Q0:0O0O- t~22#0222222222. $q(( M Mdn M M M M M M M M M !:"'t'O%+!/'')    rYr>rKrr5*Sequence[Union[EVALUATOR_T, AEVALUATOR_T]]c Kttdstjd_t  d fd fd}t j||d }t j|d tj \}}} d |2d |2d|2S)zRun predictions and evaluations in a single pipeline. This allows evaluators to process results as soon as they're available from the target function, rather than waiting for all predictions to complete first. _evaluation_feedback_executor max_workersrrc Kt|jjjt jd{V}|d|d}}||dgidjd{V}|S)Nrrsrrsrrfeedback_executor) _aforwardrexperiment_namerrBr)r_arun_evaluatorsr)rpredrsrr5rrKtraceable_targets rWprocess_examplezQ_AsyncExperimentManager.awith_predictions_and_evaluators..process_example4s" ..w77$ +F33$D ?DKSG00&+4b/ #'"D1FMrYcKd{V23d{V}|WV6d{VdS)zCreate a single task per example. That task is to run the target function and all the evaluators sequentially. N)r_aend)rrrs rWprocess_exampleszR_AsyncExperimentManager.awith_predictions_and_evaluators..process_examplesKs (,'9'9';';!;!;!;!;!;!; / / / / / / /g%og......."<**,,         s4MbP?_eager_consumption_timeoutrc4K|3d{V}|dWV6dSNrrSrTrs rWrXzK_AsyncExperimentManager.awith_predictions_and_evaluators..b< 6 6 6 6 6 6 6VI  6 6 6 6 6 6 6 6c4K|3d{V}|dWV6dSNrsrSrs rWrXzK_AsyncExperimentManager.awith_predictions_and_evaluators..c;7777777&&-77777777r c4K|3d{V}|dWV6dSNrrSrs rWrXzK_AsyncExperimentManager.awith_predictions_and_evaluators..d=TTTTTTTv'; <TTTTTTTTr r|r)rr) r&rcfThreadPoolExecutorr_ensure_async_traceablerwaiter_with_concurrencyrrrr) rrKr5r>rexperiment_resultsr1r2r3rrs ``` @@rWrz8_AsyncExperimentManager.awith_predictions_and_evaluators!s>)44 t<== V131FST1U1U1UD .26::         .      (>     ',     _%7PPP Bzz 6 62 6 6 677B777TTQSTTT   rYcK|||t|}tj|dt j\}}|d|2d|2S)N)r>rrrc4K|3d{V}|dWV6dSrrSrTrs rWrXz<_AsyncExperimentManager.awith_predictions..ts; 2 2 2 2 2 2 2tT)_ 2 2 2 2 2 2 2 2r c4K|3d{V}|dWV6dSr"rSr2s rWrXz<_AsyncExperimentManager.awith_predictions..us;3333333$u+33333333r )r|) _apredictr)rwrrrr)rrKr>_experiment_resultsr-r.s rWrz)_AsyncExperimentManager.awith_predictionsgs #nn + ;F C C-   !4aglnnMMMBzz 2 2r 2 2 233333   rYrcKt|}|||}tj|dt j\}}}|d|2d|2d|2S)Nrrrc4K|3d{V}|dWV6dSrrSrs rWrXz;_AsyncExperimentManager.awith_evaluators..rr c4K|3d{V}|dWV6dSr"rSrs rWrXz;_AsyncExperimentManager.awith_evaluators..r#r c4K|3d{V}|dWV6dSr%rSrs rWrXz;_AsyncExperimentManager.awith_evaluators..r&r r')r&_ascorerwrrrr)rr5r>r,r-r.r/s rWrz(_AsyncExperimentManager.awith_evaluatorsxs )44 !\\*o\VV_%7PPP Bzz 6 62 6 6 677B777TTQSTTT   rYr7Sequence[SUMMARY_EVALUATOR_T]cKt|}||}||d{V||S)N)r|r)r+_aapply_summary_evaluatorsrrr)rr7wrapped_evaluatorsaggregate_feedback_gens rWrz0_AsyncExperimentManager.awith_summary_evaluatorssx66HII!%!@!@AS!T!Tzz$$&& & & & & & &!!2   rY"AsyncIterator[ExperimentResultRow]cKtj||d{V|23d{V\}}}t |||WV!6dS)Nr )rw async_ziprrrr)rrsrrs rW aget_resultsz$_AsyncExperimentManager.aget_resultss6@6J NN  D$6$6$8$8888888$:V:V:X:X7 7        2#w 2&#5     7 7 7 sA6dict[str, list[dict]]cNK|jdgiSdd|j2d{ViS)Nrc:Kg|3d{V }|dD]}|6S)NrrS)rTrress rWruz?_AsyncExperimentManager.aget_summary_scores..sk!"9-s)rrs rWaget_summary_scoresz+_AsyncExperimentManager.aget_summary_scoressc  (r? " %)%:  rYAsyncIterator[_ForwardResults]cKt|fd}tj||d23d{V}|WV 6d{VdS)Nc Kd{V23d{V}t|jjjjWVG6dSrR)rrrrrrBr)rfnrrs rW predict_allz6_AsyncExperimentManager._apredict..predict_alls'+'9'9';';!;!;!;!;!;!;       g227;;(NK'("rrNrrMs` ` @rWr4z!_AsyncExperimentManager._apredicts%V , ,       '= [[]]u          &LLLLL jjllsASequence[RunEvaluator]cKtjd5fd}tj||d23d{V}|WV 6 ddddS#1swxYwYdS)Nrr c|K23d{V}|WV#6dS)Nr )rCr)current_resultsr5rrs rW score_allz2_AsyncExperimentManager._ascore..score_alls-1->->-@-@///"OGX0.A-@-@s;rr)r(r)rwr+)rr5r>rSrrs`` @rWr:z_AsyncExperimentManager._ascores  "q 1 1 1 5F       !+ A!!!       f !!                  s'AAAA"%A"rRrrcf.ThreadPoolExecutorc  Ktj}i|dpidji}tjdii|d|jsdndjd5|d |d |d }d  fd }g}|D]&} ||| d{V'|D]} | |d |  t |cdddS#1swxYwYdS)Nr9rFr5localT project_namer9enabledrBrsrr evaluatorr/c lKtj | d{V} j|} jr5 j| j |S#t$r |j }tfd|D} j|} jr5 j| j |cYdS#t$r'}td|Yd}~nd}~wwxYwtdt!|d j dt!d YddSdwwxYw) N)rsrevaluator_run_id)rs project_id _executorc Tg|]$}t|tddi%S)errorT)keyrcommentextra)r-repr)rTrarr\s rWruz[_AsyncExperimentManager._arun_evaluators.._run_single_evaluator.. sP%%%%( !1(+2B,0GG+2D/ !"!"!"%%%rY)rzError parsing feedback keys: zError running evaluator z on run : Texc_info)r\uuid4 aevaluate_runrrB_select_eval_resultsr_log_evaluation_feedback_get_experimentrd Exception feedback_keysr.rerfr`rd) rZevaluator_responseselected_resultsrnerror_responsee2rr\rrrsrs @@rW_run_single_evaluatorzG_AsyncExperimentManager._arun_evaluators.._run_single_evaluators|#':<< 3/8/F/F $ > >w G G)90G00******& (,{'G'G*(($+ <<. #'+';';'='='@&7 = ,+ !!!(1(? ):%%%%%,9%%% * * *,0;+K+K*,,( / K@@ .$'+/+?+?+A+A+D*; A 0//////$ %IR%I%IJJJLL44 ??44 #44*.q''44!%!;!s>BB## F3/A;D0*F30 E!:EF.E!!AF..F3rr rS)rZr/) rhget_tracing_contextrtracing_contextrrBappendextendr) rr5rRrcurrent_contextr9 eval_resultsrs all_resultsrZrrrss ` ` @@rWrz(_AsyncExperimentManager._arun_evaluatorss 022 z*0b T12    ! ,$*.*>H77D+   O O "%(C%i0G*+?@L5 5 5 5 5 5 5 5 5 nK' K K "")>)>y)I)I#I#I#I#I#I#IJJJJ% ; ;% +226:::&#/WO O O O O O O O O O O O O O O O O O sBC$$C(+C(c PKgg}}tj|d{V}tj||23d{V\}}||||56g}|jr|jnd}tj } i| dpi|j |d} tj dii| d| |jsdnd|j d5|D]} | ||} |j | | j} || |jrl| D]i}|dh }|d d}tjt)j|j jfi|d||d d{Vj#t.$r9}t0d t5| d |dYd}~d}~wwxYw dddn #1swxYwYd|iWVdS)Nr9)rF experiment_idr5rVTrW)fn_name target_run_id)excludeevaluator_info)run_idr] source_infoz Error running summary evaluator rerfrrS)rwrrrBrrwrrlrdrtrurrvrBrj__name__rx model_dumppoprxryrzcreate_feedbackrmrer`rd)rr7r|examplesasync_examplesrsraggregate_feedbackr]ryr9rZsummary_eval_resultflattened_resultsrfeedbackrrs rWr=z2_AsyncExperimentManager._aapply_summary_evaluators6sRh#9@R@R@T@T:T:T:T:T:T:TUU","6 NN  n# #  % % % % % % %,#w KK    OOG $ $ $ $ #  262FPT))++..D 022 z*0b #2!+    ! ,$*.*>H77D+   ! ! 0   *3)D(*C*C'(, (H(H+ ) 2)I))%'--.?@@@+ &7  F'-'8'8/AR'8'S'SH-5\\:JD-Q-QN",": + 8 : : $ ;###+#(,+5,: ####!LLQ4 ??QQaQQ!%!) ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! D,-------sCBH B5GH H /G?:H?HHHHcKg}|d{V23d{V}|jr||j)6|rt|nd}|r|ndSrR)rrrwmax isoformat)rrrmax_modified_ats rW_get_dataset_versionz,_AsyncExperimentManager._get_dataset_versionns #'#5#5#7#7777777 8 8 8 8 8 8 8'" 8""7#6777 8 /:C#k***t.=G((***4GsAcKt}|d{V23d{V}|jru|jdr[t |jdt r;|jdD],}t |t r||-|d6t |S)N dataset_splitbase)setrr9getrZrhr[add)rsplitsrsplits rW_get_dataset_splitsz+_AsyncExperimentManager._get_dataset_splitsys#'#5#5#7#7777777 # # # # # # #'  #$((99 #w/@$GG # %-o>**E!%--* 5)))* 6""""8F||sCNonec<K|jsdS|j}|td|}|d{V|d<|d{V|d<|j|ji|j |dS)NzExperiment not started yet.dataset_versiondataset_splits)r9) rrrc_get_experiment_metadatarrrBupdate_projectrdr9)rrFproject_metadatas rWrz_AsyncExperimentManager._aends#  F%  :;; ;88::484M4M4O4O.O.O.O.O.O.O*+373K3K3M3M-M-M-M-M-M-M)* "" M%" #     rYargsr rMcB|jf}|j|j|j|j|j|j|j|j|j |j |j |j |j d }t|t|t|dz}i||}|j|i|S)N) rFr9r|rBrrrrrrrHrrI)rrrrrBrrrrrrrrrrhlenr)rrrM default_argsdefault_kwargs full_args full_kwargss rWrz_AsyncExperimentManager._copys } *Jk"&":#4 ."2#'#r?rKrr5rrNr)r>r?rKrrNr)r5rr>r?rNr)r7r;rNrrNr@)rNrD)NF)r>r?rrErKrrNrJ)r5rOr>r?rNr@)r5rOrRrrrTrNr)r7r;rNrrNr<)rNrrNr)rr rMr rNr)r __module__ __qualname____doc__rrrrrrrrrrrrrCrIr4r:rr=rrrr __classcell__)rs@rWrr(sq&&XCG#'SW-1IMFJ%) &*.2$)"'#3738%,.,.,.,.,.,.,.\* * * * XB::::        *    J*. D D D D D T*.      **.                 "*.$) @*.&ZZZZx6.6.6.6.p H H H H     $99999999rYrceZdZd dZed!dZed"dZed#d Zd!d Zd#d Z d$dZ d%dZ d&dZ d'd(dZ d!dZd)dZd!dZd*dZdS)+rOexperiment_managerrc||_g|_tj|_tj||j|_d|_d|_ dS)Nr) _manager_resultsr Condition _condition create_task _process_data_task_processed_count_comparison_url)rrs rWrzAsyncExperimentResults.__init__sZ+ 35 !+--(););DM)J)JKK !.2rYrNr[c|jjSrR)rrrHs rWrz&AsyncExperimentResults.experiment_names },,rY uuid.UUIDc>|jjS)zThe ID of the experiment.)rrlrdrHs rWr}z$AsyncExperimentResults.experiment_ids},,..11rYr<c>|jjS)z.The URL of the experiment in the LangSmith UI.)rrlurlrHs rWrzAsyncExperimentResults.urls},,..22rYcDK|jd{VS)z:Get the ID of the dataset associated with this experiment.N)rrrHs rWrz%AsyncExperimentResults.get_dataset_ids,]11333333333rYc>K|j}|jsu|jrn|jd{V}|jdd}|dd}|d|d|j|_|jS)z7Get the URL to the comparison view for this experiment.N?rz /projects/p/z /datasets/z/compare?selectedSessions=)rrlrrrrrd)rrFr project_urlbase_urls rWget_comparison_urlz)AsyncExperimentResults.get_comparison_urls]2244 #   #};;========J$...s33A6K"((88;H44z44$.M44  ##rYr@c|SrRrSrHs rWrz AsyncExperimentResults.__aiter__s rYrcK|j4d{V |jt|jkr6|j|j}|xjdz c_|cdddd{VS|jr$|j}||t|jd{V#1d{VswxYwYdS)NTr1) rrrrrdone exceptionrr)rrexcs rW __anext__z AsyncExperimentResults.__anext__ss? - - - - - - - - -(3t}+=+===!]4+@AF))Q.))! - - - - - - - - - - - - - - Z__&&-*..00C! ,,o**,,,,,,,,, - - - - - - - - - - - - - - - - -sAC%AC C C rrcKt}||23d{V}|j4d{V|j||jdddd{Vn#1d{VswxYwYl6|d{V}|j4d{V||_|jdddd{VdS#1d{VswxYwYdSrR) r#rCrrrwnotifyrIr notify_all)rrtqdmitemsummary_scoress rWrz$AsyncExperimentResults._process_datas;||$w335566 ) ) ) ) ) ) )$ ) ) ) ) ) ) ) ) $$T***&&((( ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) )7 '::<<<<<<<<? ) ) ) ) ) ) ) )$2D ! O & & ( ( ( ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) )s)B4B B B !C66 DDrNstartr?end DataFramec0t|j||S)N)rr)r*r)rrrs rW to_pandasz AsyncExperimentResults.to_pandass$-u#>>>>rYcddl}|jrD|jdr*|dd}|S|S)Nrpandas)importlib.utilrutil find_specr _repr_html___repr__)r importlibdfs rWrz"AsyncExperimentResults._repr_html_s` = #Y^55h?? #1%%B>>## #==?? "rYrAc*t|jSrR)rrrHs rW__len__zAsyncExperimentResults.__len__s4=!!!rYcd|jdS)Nz)rrHs rWrzAsyncExperimentResults.__repr__ sA$*>AAAArYc$K|jd{VdSrR)rrHs rWrzAsyncExperimentResults.wait s&jrY)rrr)rNrrr)rNr)rrrNr)rN)rr?rr?rNr)rNrAr)rrrrpropertyrr}rrrrrrrrrrrrSrYrWrOrOs_ 3 3 3 3---X-222X2333X34444 $ $ $ $ - - - - ) ) ) )>B????? ####""""BBBBrYrM,rh.SupportsLangsmithExtra[[dict], Awaitable]rrrr[dictlangsmith.Clientrrct Kd d fd }dfd }tj||i|djpji| } |d kr j| d <n|d kr|| d <nt d|tjd5 t|} fd| D} || d| id{Vn7#t$r*} t d| ddYd} ~ nd} ~ wwxYwtttj cdddS#1swxYwYdS)Nrrun_trees.RunTreerNrc |dSrRrS)rrss rW_get_runz_aforward.._get_runs rY rt.RunTreec j|_dSrR)rdrr)rrs rW_set_reference_example_idz,_aforward.._set_reference_example_ids!(rYexample_version)on_endrXr9rBr2rrignore _on_successz2Unrecognized error_handling value: error_handling=T)rYc0g|]}t|SrS)r)rTargnrs rWruz_aforward..5s#AAAtGGT**AAArYlangsmith_extrazError running target function: r1)rg stacklevel)rsr)rrrNr)rrrNr)rtLangSmithExtrarrrrdrcrvrrmrer`rrrr)rMrrr9rBrrIrrr arg_namesrrrss ` @rWrrsD&*C,,,,,,'$   3 Iw7ITTVV  O29*.// 8 # #)B &&P~PPQQQ D ) ) )    (,,IAAAAyAAAD"d dict: # do work, like chain.invoke(inputs) return {...} await aevaluate(predict, ...)zTarget must be a callable async function. Received a non-callable object. Example usage: async def predict(inputs: dict) -> dict: # do work, like chain.invoke(inputs) return {...} await aevaluate(predict, ...) AsyncTarget)rprocess_inputs) rrr callablercrtis_traceable_functionainvoke traceabler)rKs rWr*r*Es  &v . .7Mf7U7U F   0 0  '' !& ) ) $^F r|2     rY)rrct|trtj|Stjt |||S)z*Return the examples for the given dataset.r)rZrrwrr%)r3rBrs rWrrfsO$ &&6/555  +d6?RSSS  rYTiterableIterable[AsyncIterable[T]]AsyncIterator[T]c4K|D]}|23d{V}|WV 6dS)zChain multiple async iterables.NrS)r  sub_iterablers rWrrws]! &       $JJJJJ',srlist[schemas.Example]AsyncIterable[schemas.Example]cK|D]}|WVdS)z0Convert a list of examples to an async iterable.NrS)rrs rWrrs. rY) NNNNNNrr1NTNTr2) r3r4r5r6r7r8r9r:r;r<r=r<r>r?r@rArBrCrDrErFrGrHrErIrJrKrLrMr rNrO)NNNrNFT)r5r6r7r8r9r:r>r?rBrCrnrErDrErFrorNrO) NNNNNNr1NTNTr2)r3r}r5r6r7r8r9r:r;r<r=r<r>r?r@rArBrCrDrErFrGrHrErIrJrKr~rNrO)Fr2)rMrrrrr[r9rrBrrrErIrJrNr)rrrNr)rKrrNr)r3r}rBrrrErNr)r r rNr )rrrNr)Yr __future__rrconcurrent.futuresfuturesr(ryrloggingrr\collections.abcrrrrrtypingr r r r r rrr langsmithrrtrrrrrlangsmith._internalrrw#langsmith._internal._beta_decoratorrlangsmith.evaluation._runnerrrrrrrrrrr r!r"r#r$r%r&r'r(r)r*r+langsmith.evaluation.evaluatorr,r-r.r/rpdlangchain_core.runnablesr0r getLoggerrrerrrmrgrirrOrrr*rr rrrSrYrWr s""""""  WWWWWWWWWWWWWW                    ''''''((((((((%%%%%%''''''444444::::::.111111 III  8 $ $  dVYt_ $%xt io0M'NN   GKBF#'+!%%&)-IM/4'H H H H H \HLBF#%&)-~~~~~JHLBF#'+!%%))-IM/4MMMMM`F 9F 9F 9F 9F 95F 9F 9F 9R\\\\\\\\J!&/4. . . . . b>>>>J!&       GCLLrY