ó
    üÞ jbÚ  ã                  óÂ  • S r SSKJr  SSKrSSKJr  SSKrSSKrSSK	r	SSK
r
SSKrSSKJrJrJrJrJr  SSKJrJrJrJrJrJrJrJr  SSKrSSKJr  SSKJrJr  SSKJr   SS	KJ!r"  SS
K#J$r%  SSK&J'r'  SSK(J)r)J*r*J+r+J,r,J-r-J.r.J/r/J0r0J1r1J2r2J3r3J4r4J5r5J6r6J7r7J8r8J9r9J:r:J;r;J<r<J=r=  SSK>J?r?J@r@JArAJBrB  \(       a  SSKCrDSSKEJFrF  \DRŽ                  rGO\rG\	R�                  " \I5      rJ\\\K/\\K   4   \\K\K/\\K   4   4   rL             S                               S S jjrM       S!                 S"S jjrN            S#                             S$S jjrO " S S\/5      rP " S S5      rQ  S%               S&S jjrRS'S jrS    S(S jrTSS.       S)S jjrU\" S5      rV    S*S jrW    S+S jrXg),zV2 Evaluation Interface.é    )ÚannotationsN)ÚAsyncIterableÚAsyncIteratorÚ	AwaitableÚIterableÚSequence)ÚTYPE_CHECKINGÚAnyÚCallableÚLiteralÚOptionalÚTypeVarÚUnionÚcast)Úrun_helpers)Ú	run_treesÚschemas)r   )Úutils)Ú_aiter)Ú
_warn_once)ÚAEVALUATOR_TÚDATA_TÚEVALUATOR_TÚExperimentResultRowÚ_collect_evaluator_keysÚ_evaluators_include_attachmentsÚ_ExperimentManagerMixinÚ_ForwardResultsÚ_get_target_argsÚ_is_langchain_runnableÚ_load_examples_mapÚ_load_experimentÚ
_load_tqdmÚ_load_traces_for_experimentÚ_resolve_dataÚ_resolve_evaluatorsÚ_resolve_experimentÚ_resolve_num_examplesÚ_target_include_attachmentsÚ
_to_pandasÚ_wrap_summary_evaluators)ÚSUMMARY_EVALUATOR_TÚEvaluationResultÚEvaluationResultsÚRunEvaluator)ÚRunnableÚAsyncExperimentResultsc             ‹  ó´  #   • [        U [        [        R                  [        R
                  45      (       aß  US:„  [        U5      U(       + [        U5      [        U5      S.n[        UR                  5       5      (       a/  S[        S UR                  5        5       5       S3n[        U5      e[        U [        [        R                  45      (       a  U OU R                  n[        R                  SU S35        [        U 4UUUUU	U
S.UD6I S	h  v•N $ [        U [         [        45      (       a  S
n[        U5      eU(       a  SU S3n[        U5      eU(       d  Sn[        U5      eU(       a  U(       a  SU SU 3n[        U5      eU(       d  [#        S5        [        R                  SU  S35        [%        U UUUUUUUUU	U
UUUS9I S	h  v•N $  NÂ N7f)ab  Evaluate an async target system on a given dataset.

Args:
    target (AsyncCallable[[dict], dict] | AsyncIterable[dict] | Runnable | EXPERIMENT_T | Tuple[EXPERIMENT_T, EXPERIMENT_T]):
        The target system or experiment(s) to evaluate.

        Can be an async function that takes a `dict` and returns a `dict`, a
        langchain `Runnable`, an existing experiment ID, or a two-tuple of experiment IDs.
    data (Union[DATA_T, AsyncIterable[schemas.Example]]): The dataset to evaluate on.

        Can be a dataset name, a list of examples, an async generator of examples, or an async iterable of examples.
    evaluators (Optional[Sequence[EVALUATOR_T]]): A list of evaluators to run
        on each example.
    summary_evaluators (Optional[Sequence[SUMMARY_EVALUATOR_T]]): A list of summary
        evaluators to run on the entire dataset.
    metadata (Optional[dict]): Metadata to attach to the experiment.
    experiment_prefix (Optional[str]): A prefix to provide for your experiment name.
    description (Optional[str]): A description of the experiment.
    max_concurrency (int | None): The maximum number of concurrent
        evaluations to run.

        If `None` then no limit is set. If `0` then no concurrency.
    num_repetitions (int): The number of times to run the evaluation.
        Each item in the dataset will be run and evaluated this many times.
    client (Optional[langsmith.Client]): The LangSmith client to use.
    blocking (bool): Whether to block until the evaluation is complete.
    experiment (Optional[schemas.TracerSession]): An existing experiment to
        extend.

        If provided, `experiment_prefix` is ignored. For advanced usage only.
    error_handling (str, default="log"): How to handle individual run errors.

        `'log'` will trace the runs with the error message as part of the
        experiment, `'ignore'` will not count the run as part of the experiment at
        all.

Returns:
    An async iterator over the experiment results.

Environment:
    - `LANGSMITH_TEST_CACHE`: If set, API calls will be cached to disk to save time and
        cost during testing.

        Recommended to commit the cache files to your repository for faster CI/CD runs.

        Requires the `'langsmith[vcr]'` package to be installed.

Examples:
    >>> from typing import Sequence
    >>> from langsmith import Client, aevaluate
    >>> from langsmith.schemas import Example, Run
    >>> client = Client()
    >>> dataset = client.clone_public_dataset(
    ...     "https://smith.langchain.com/public/419dcab2-1d66-4b94-8901-0357ead390df/d"
    ... )
    >>> dataset_name = "Evaluate Examples"

    Basic usage:

    >>> def accuracy(run: Run, example: Example):
    ...     # Row-level evaluator for accuracy.
    ...     pred = run.outputs["output"]
    ...     expected = example.outputs["answer"]
    ...     return {"score": expected.lower() == pred.lower()}

    >>> def precision(runs: Sequence[Run], examples: Sequence[Example]):
    ...     # Experiment-level evaluator for precision.
    ...     # TP / (TP + FP)
    ...     predictions = [run.outputs["output"].lower() for run in runs]
    ...     expected = [example.outputs["answer"].lower() for example in examples]
    ...     # yes and no are the only possible answers
    ...     tp = sum([p == e for p, e in zip(predictions, expected) if p == "yes"])
    ...     fp = sum([p == "yes" and e == "no" for p, e in zip(predictions, expected)])
    ...     return {"score": tp / (tp + fp)}

    >>> import asyncio
    >>> async def apredict(inputs: dict) -> dict:
    ...     # This can be any async function or just an API call to your app.
    ...     await asyncio.sleep(0.1)
    ...     return {"output": "Yes"}
    >>> results = asyncio.run(
    ...     aevaluate(
    ...         apredict,
    ...         data=dataset_name,
    ...         evaluators=[accuracy],
    ...         summary_evaluators=[precision],
    ...         experiment_prefix="My Experiment",
    ...         description="Evaluate the accuracy of the model asynchronously.",
    ...         metadata={
    ...             "my-prompt-version": "abcd-1234",
    ...         },
    ...     )
    ... )  # doctest: +ELLIPSIS
    View the evaluation results for experiment:...

    Evaluating over only a subset of the examples using an async generator:

    >>> async def example_generator():
    ...     examples = client.list_examples(dataset_name=dataset_name, limit=5)
    ...     for example in examples:
    ...         yield example
    >>> results = asyncio.run(
    ...     aevaluate(
    ...         apredict,
    ...         data=example_generator(),
    ...         evaluators=[accuracy],
    ...         summary_evaluators=[precision],
    ...         experiment_prefix="My Subset Experiment",
    ...         description="Evaluate a subset of examples asynchronously.",
    ...     )
    ... )  # doctest: +ELLIPSIS
    View the evaluation results for experiment:...

    Streaming each prediction to more easily + eagerly debug.

    >>> results = asyncio.run(
    ...     aevaluate(
    ...         apredict,
    ...         data=dataset_name,
    ...         evaluators=[accuracy],
    ...         summary_evaluators=[precision],
    ...         experiment_prefix="My Streaming Experiment",
    ...         description="Streaming predictions for debugging.",
    ...         blocking=False,
    ...     )
    ... )  # doctest: +ELLIPSIS
    View the evaluation results for experiment:...

    >>> async def aenumerate(iterable):
    ...     async for elem in iterable:
    ...         print(elem)
    >>> asyncio.run(aenumerate(results))

    Running without concurrency:

    >>> results = asyncio.run(
    ...     aevaluate(
    ...         apredict,
    ...         data=dataset_name,
    ...         evaluators=[accuracy],
    ...         summary_evaluators=[precision],
    ...         experiment_prefix="My Experiment Without Concurrency",
    ...         description="This was run without concurrency.",
    ...         max_concurrency=0,
    ...     )
    ... )  # doctest: +ELLIPSIS
    View the evaluation results for experiment:...

    Using Async evaluators:

    >>> async def helpfulness(run: Run, example: Example):
    ...     # Row-level evaluator for helpfulness.
    ...     await asyncio.sleep(5)  # Replace with your LLM API call
    ...     return {"score": run.outputs["output"] == "Yes"}

    >>> results = asyncio.run(
    ...     aevaluate(
    ...         apredict,
    ...         data=dataset_name,
    ...         evaluators=[helpfulness],
    ...         summary_evaluators=[precision],
    ...         experiment_prefix="My Helpful Experiment",
    ...         description="Applying async evaluators example.",
    ...     )
    ... )  # doctest: +ELLIPSIS
    View the evaluation results for experiment:...


!!! warning "Behavior changed in `langsmith` 0.2.0"

    'max_concurrency' default updated from None (no limit on concurrency)
    to 0 (no concurrency at all).
é   )Únum_repetitionsÚ
experimentÚupload_resultsÚexperiment_prefixÚdatazReceived invalid arguments. c              3  ó:   #   • U H  u  pU(       d  M  Uv •  M     g 7f©N© )Ú.0ÚkÚvs      ÚW/var/www/html/gaurav/venv/lib/python3.13/site-packages/langsmith/evaluation/_arunner.pyÚ	<genexpr>Úaevaluate.<locals>.<genexpr>  s   é € ÐAÑ';™t˜q¼qŸ™Ò';ùs   ‚’	z? should not be specified when target is an existing experiment.z,Running evaluation over existing experiment z...)Ú
evaluatorsÚsummary_evaluatorsÚmetadataÚmax_concurrencyÚclientÚblockingNzÏRunning a comparison of two existing experiments asynchronously is not currently supported. Please use the `evaluate()` method instead and make sure that your evaluators are defined as synchronous functions.zReceived unsupported arguments zC. These arguments are not supported when creating a new experiment.zDMust specify 'data' when running evaluations over a target function.zeExpected at most one of 'experiment' or 'experiment_prefix', but both were provided. Got: experiment=z, experiment_prefix=z&'upload_results' parameter is in beta.z&Running evaluation over target system )r8   rB   rC   rD   r7   ÚdescriptionrE   r4   rF   rG   r5   r6   Úerror_handling)Ú
isinstanceÚstrÚuuidÚUUIDr   ÚTracerSessionÚboolÚanyÚvaluesÚtupleÚitemsÚ
ValueErrorÚidÚloggerÚdebugÚaevaluate_existingÚlistr   Ú
_aevaluate)Útargetr8   rB   rC   rD   r7   rH   rE   r4   rF   rG   r5   r6   rI   ÚkwargsÚinvalid_argsÚmsgÚ	target_ids                     r?   Ú	aevaluater`   L   s  é € ôF �&œ3¤§	¡	¬7×+@Ñ+@ÐA×BÑBà.°Ñ2Ü˜zÓ*Ø"0Ô0Ü!%Ð&7Ó!8Ü˜“Jñ
ˆô ˆ|×"Ñ"Ó$×%Ñ%à.ÜÑA |×'9Ñ'9Ô';ÓAÓAÐBð CCðDð ô
 ˜S“/Ð!Ü(¨´#´t·y±yÐ1A×BÑB‘FÈÏ	É	ˆ	Ü�‰ÐCÀIÀ;ÈcÐRÔSÜ'Øð	
à!Ø1ØØ+ØØñ	
ð ñ	
÷ 	
ð 		
ô 
�FœT¤5˜M×	*Ñ	*ðNð 	ô
 ˜‹oÐÞ	à-¨f¨Xð 68ð 9ð 	ô ˜‹oÐÞØTˆÜ˜‹oÐÞ	Ö)ðà)˜lÐ*>Ð?PÐ>QðSð 	ô
 ˜‹oÐæÜÐ?Ô@Ü�‰Ð=¸f¸XÀSÐIÔJÜØØØ!Ø1ØØ/Ø#Ø+Ø+ØØØ!Ø)Ø)ñ
÷ 
ð 	
ñK	
ñJ
ùs%   ‚DGÄGÄB=GÇGÇGÇGFc             ƒ  óX  #   • U=(       d    [         R                  " 5       n[        U [        R                  5      (       a  U O6[
        R                  " [        R                  " 5       [        X5      I Sh  v•N n[
        R                  " [        R                  " 5       [        UUUS9I Sh  v•N n	[
        R                  " [        R                  " 5       [        XX5      I Sh  v•N n
U	 Vs/ sH  oºUR                     PM     nn[        U	UUUUUUUUS9	I Sh  v•N $  N¨ Ns N>s  snf  N7f)a•  Evaluate existing experiment runs asynchronously.

Args:
    experiment (Union[str, uuid.UUID]): The identifier of the experiment to evaluate.
    evaluators (Optional[Sequence[EVALUATOR_T]]): Optional sequence of evaluators to use for individual run evaluation.
    summary_evaluators (Optional[Sequence[SUMMARY_EVALUATOR_T]]): Optional sequence of evaluators
        to apply over the entire dataset.
    metadata (Optional[dict]): Optional metadata to include in the evaluation results.
    max_concurrency (int | None): The maximum number of concurrent
        evaluations to run.

        If `None` then no limit is set. If `0` then no concurrency.
    client (Optional[langsmith.Client]): Optional Langsmith client to use for evaluation.
    load_nested: Whether to load all child runs for the experiment.

        Default is to only load the top-level root runs.
    blocking (bool): Whether to block until evaluation is complete.

Returns:
    An async iterator over the experiment results.

Examples:
    Define your evaluators

    >>> from typing import Sequence
    >>> from langsmith.schemas import Example, Run
    >>> def accuracy(run: Run, example: Example):
    ...     # Row-level evaluator for accuracy.
    ...     pred = run.outputs["output"]
    ...     expected = example.outputs["answer"]
    ...     return {"score": expected.lower() == pred.lower()}
    >>> def precision(runs: Sequence[Run], examples: Sequence[Example]):
    ...     # Experiment-level evaluator for precision.
    ...     # TP / (TP + FP)
    ...     predictions = [run.outputs["output"].lower() for run in runs]
    ...     expected = [example.outputs["answer"].lower() for example in examples]
    ...     # yes and no are the only possible answers
    ...     tp = sum([p == e for p, e in zip(predictions, expected) if p == "yes"])
    ...     fp = sum([p == "yes" and e == "no" for p, e in zip(predictions, expected)])
    ...     return {"score": tp / (tp + fp)}

    Load the experiment and run the evaluation.

    >>> import asyncio
    >>> import uuid
    >>> from langsmith import Client, aevaluate, aevaluate_existing
    >>> client = Client()
    >>> dataset_name = "__doctest_aevaluate_existing_" + uuid.uuid4().hex[:8]
    >>> dataset = client.create_dataset(dataset_name)
    >>> example = client.create_example(
    ...     inputs={"question": "What is 2+2?"},
    ...     outputs={"answer": "4"},
    ...     dataset_id=dataset.id,
    ... )
    >>> async def apredict(inputs: dict) -> dict:
    ...     await asyncio.sleep(0.001)
    ...     return {"output": "4"}
    >>> results = asyncio.run(
    ...     aevaluate(
    ...         apredict, data=dataset_name, experiment_prefix="doctest_experiment"
    ...     )
    ... )  # doctest: +ELLIPSIS
    View the evaluation results for experiment:...
    >>> experiment_id = results.experiment_name
    >>> # Consume all results to ensure evaluation is complete
    >>> async def consume_results():
    ...     result_list = [r async for r in results]
    ...     return len(result_list) > 0
    >>> asyncio.run(consume_results())
    True
    >>> import time
    >>> time.sleep(3)
    >>> results = asyncio.run(
    ...     aevaluate_existing(
    ...         experiment_id,
    ...         evaluators=[accuracy],
    ...         summary_evaluators=[precision],
    ...     )
    ... )  # doctest: +ELLIPSIS
    View the evaluation results for experiment:...
    >>> client.delete_dataset(dataset_id=dataset.id)


N)Úload_nested)r8   rB   rC   rD   rE   rF   rG   r5   )r   Úget_cached_clientrJ   r   rN   Ú
aitertoolsÚaio_to_threadÚcontextvarsÚcopy_contextr"   r$   r!   Úreference_example_idrZ   )r5   rB   rC   rD   rE   rF   rb   rG   ÚprojectÚrunsÚdata_mapÚrunr8   s                r?   rX   rX   W  s  é € ð~ ×4”y×2Ò2Ó4€Fô �j¤'×"7Ñ"7×8Ñ8ñ 	ô ×*Ò*Ü×(Ò(Ó*Ô,<¸jó÷ ð	 ô ×)Ò)Ü× Ò Ó"Ü#ØØØñ÷ €Dô  ×-Ò-Ü× Ò Ó"Ô$6¸ó÷ €Hñ ;?Ó?¹$°3�S×-Ñ-Ô.¹$€DÐ?ÜØØØØ-ØØ'ØØØñ
÷ 
ð 
ñ!ñ
ñùò @ñ
ùsT   ‚A2D*Á4DÁ56D*Â+DÂ,6D*Ã"D!Ã#D*Ã+D#ÄD*ÄD(ÄD*ÄD*Ä!D*Ä#D*c             ƒ  ó´  #   • [         R                  " U 5      =(       dI    [        U S5      =(       a$    [         R                  " U R	                  5       5      =(       d    [        U 5      nU	=(       d    [        R                  " 5       n	U(       a  S O [        [        [        R                     U 5      n[        R                  " [        R                  " 5       [         UUU	5      I S h  v•N u  nn[#        [%        U 5      5      ['        U5      -   n[)        UU	UU=(       d    UUU[+        U5      UUS:„  UU-  S:„  UUS9R-                  5       I S h  v•N n[.        R0                  " S 5      nUb5  UR3                  5       I S h  v•N n[4        R6                  " U5      U S3-  nOS n[.        R8                  " UU	R:                  /S9   U(       au  U(       a'  UR=                  [        [>        U 5      X'S9I S h  v•N nO&URA                  [        [>        U 5      US9I S h  v•N nU(       a  URC                  U5      I S h  v•N nO>U(       a  URE                  X'S9I S h  v•N nU(       a  URC                  U5      I S h  v•N n[G        U5      nU
(       a  URI                  5       I S h  v•N   UsS S S 5        $  GNÍ GNh GN: NÈ N£ N… Nh NJ N"! , (       d  f       g = f7f)NÚ	__aiter__r   r3   )rF   rD   r5   rH   r4   Úevaluator_keysrj   Úinclude_attachmentsÚreuse_attachmentsr6   rI   z.yaml)Úignore_hosts©rE   )%ÚasyncioÚiscoroutinefunctionÚhasattrÚiscoroutinern   r    Úrtrc   r   r   r   ÚRunrd   re   rf   rg   r'   Úintr)   r   Ú_AsyncExperimentManagerr   ÚastartÚls_utilsÚget_cache_dirÚget_dataset_idÚpathlibÚPathÚwith_optional_cacheÚapi_urlÚ awith_predictions_and_evaluatorsÚ	ATARGET_TÚawith_predictionsÚawith_summary_evaluatorsÚawith_evaluatorsr1   Úwait)r[   r8   rB   rC   rD   r7   rH   rE   r4   rF   rG   r5   r6   rI   Úis_async_targetrj   Úexperiment_Únum_include_attachmentsÚmanagerÚ	cache_dirÚdsidÚ
cache_pathÚresultss                          r?   rZ   rZ   Ø  s�  é € ô$ 	×#Ò# FÓ+÷ 	*Ü�F˜KÓ(×T¬W×-@Ò-@À×AQÑAQÓASÓ-T÷	*ä! &Ó)ð ð
 ×-”r×+Ò+Ó-€FÞ"‰4¬¬X´g·k±kÑ-BÀFÓ(K€DÜ(×6Ò6Ü× Ò Ó"ÜØØØó÷ Ñ€K�ô "Ü# FÓ+óä'¨
Ó3ñ4Ðô ,ØØØØ×3Ð"3ØØ'Ü.¨zÓ:ØØ3°aÑ7Ø)Ð,CÑCÀaÑGØ%Ø%ñ÷ �fƒh÷€Gô ×&Ò& tÓ,€IØÑØ×+Ñ+Ó-×-ˆÜ—\’\ )Ó,°$°°u¨~Ñ=‰
àˆ
Ü	×	%Ò	% jÀÇÁÐ?OÓ	PÞÞà '× HÑ HÜœ FÓ+¨Zð !Ið !÷ ‘ð !(× 9Ñ 9Üœ FÓ+¸_ð !:ð !÷ �ö "Ø '× @Ñ @ÐASÓ T×T�øæØ '× 8Ñ 8Øð !9ð !÷ �ö "Ø '× @Ñ @ÐASÓ T×T�Ü(¨Ó1ˆÞØ—,‘,“.× Ð Ø÷/ 
QÑ	Pò=òò  .ñññ Uññ Uñ !÷- 
QÕ	Püs¼   ‚C#KÃ%J2Ã&A'KÅJ5Å0KÅ>J8Å?AKÇ0KÇ2J;Ç3&KÈJ=ÈKÈ9J?È:KÉKÉKÉ8KÉ9)KÊ"KÊ#KÊ(KÊ5KÊ8KÊ;KÊ=KÊ?KËKËKËKË
KËKc                  óÞ  ^ • \ rS rSrSr               S                               SU 4S jjjrSS jrSS jrS S jrS!S jr	S"S	 jr
S#S
 jrSS jr S$       S%S jjr S$     S&S jjrSS.     S'S jjr    S(S jrS)S jrS*S jr  S+       S,S jjr S$     S-S jjr        S.S jr    S/S jrS0S jrS1S jrS2S jrS3S jrSrU =r$ )4r{   i(  a   Manage the execution of experiments asynchronously.

Supports lazily running predictions and evaluations in parallel to facilitate
result streaming and early debugging.

Args:
    data (DATA_T): The data used for the experiment. Can be a dataset name or ID OR
        a generator of examples.
    runs (Optional[Iterable[schemas.Run]]): The runs associated with the experiment
        predictions.
    experiment (Optional[schemas.TracerSession]): The tracer session
        associated with the experiment.
    experiment_prefix (Optional[str]): The prefix for the experiment name.
    description (Optional[str]): The description for the experiment.
    metadata (Optional[dict]): Additional metadata for the experiment.
    client (Optional[langsmith.Client]): The Langsmith client used for
         the experiment.
    evaluation_results (Optional[Iterable[EvaluationResults]]): The evaluation
        sresults for the experiment.
    summary_results (Optional[Iterable[EvaluationResults]]): The aggregate results
        for the experiment.
    num_repetitions (Optional[int], default=1): The number of repetitions for
        the experiment.
    include_attachments (Optional[bool], default=False): Whether to include
        attachments. This is used for when we pull the examples for the experiment.
    reuse_attachments (Optional[bool], default=False): Whether to reuse attachments
        from examples. This is True if we need to reuse attachments across multiple
        target/evaluator functions.
    upload_results (Optional[bool], default=True): Whether to upload results
        to Langsmith.
    attachment_raw_data_dict (Optional[dict]): A dictionary to store raw data
        for attachments. Only used if we reuse attachments across multiple
        target/evaluator functions.
    error_handling (str, default="log"): How to handle individual run errors.

        `'log'` will trace the runs with the error message as part of the
        experiment, `'ignore'` will not count the run as part of the experiment at
        all.
Nc               ó(  >• [         TU ]  UUUUS9  Xl        X°l        S U l        Ub  [
        R                  " U5      OS U l        X`l        Xpl	        X�l
        U
b  U
O[        XR                  S9U l        XÀl        XÐl        Xàl        Xðl        UU l        g )N)r5   rD   rF   rH   )rF   )ÚsuperÚ__init__Ú_dataÚ_evaluator_keysÚ	_examplesrd   Úensure_async_iteratorÚ_runsÚ_evaluation_resultsÚ_summary_resultsÚ_num_repetitionsr(   rF   Ú_num_examplesÚ_include_attachmentsÚ_reuse_attachmentsÚ_upload_resultsÚ_attachment_raw_data_dictÚ_error_handling)Úselfr8   r5   rD   rj   rF   Úevaluation_resultsÚsummary_resultsrH   r4   Únum_examplesro   rp   rq   r6   Úattachment_raw_data_dictrI   Ú	__class__s                    €r?   r•   Ú _AsyncExperimentManager.__init__Q  sª   ø€ ô( 	‰ÑØ!ØØØ#ð	 	ñ 	
ð Œ
Ø-ÔØCGˆŒà6:Ñ6FŒJ×,Ò,¨TÔ2ÈDð 	Œ
ð $6Ô Ø /ÔØ /Ôð Ñ'ñ ä& t·K±KÑ@ð 	Ôð
 %8Ô!Ø"3ÔØ-ÔØ)AÔ&Ø-ˆÕó    c                ó”  • [        US5      (       a  UR                  (       d  U$ 0 nUR                  R                  5        Hƒ  u  p4U R                  bm  [	        UR
                  5      U-   U R                  ;   aG  US   [        R                  " U R                  [	        UR
                  5      U-      5      US   S.X#'   M  XBU'   M…     [        R                  " UR
                  UR                  UR                  UR                  UR                  UR                  UR                  UR                   UUR"                  UR$                  S9$ )aŸ  Reset attachment readers for an example.

This is only in the case that an attachment is going to be used by more
than 1 callable (target + evaluators). In that case we keep a single copy
of the attachment data in self._attachment_raw_data_dict, and create
readers from that data. This makes it so that we don't have to keep
copies of the same data in memory, instead we can just create readers
from the same data.
ÚattachmentsÚpresigned_urlÚ	mime_type©r®   Úreaderr¯   ©rU   Ú
created_atÚ
dataset_idÚinputsÚoutputsrD   Úmodified_atÚsource_run_idr­   Ú	_host_urlÚ
_tenant_id)rv   r­   rS   r¢   rK   rU   ÚioÚBytesIOr   ÚExampler³   r´   rµ   r¶   rD   r·   r¸   r¹   rº   )r¤   ÚexampleÚnew_attachmentsÚnameÚ
attachments        r?   Ú_reset_example_attachmentsÚ2_AsyncExperimentManager._reset_example_attachments  s  € ô �w ×.Ñ.°g×6I×6IØˆNà=?ˆØ '× 3Ñ 3× 9Ñ 9Ö ;ÑˆDà×.Ñ.Ñ:Ü˜Ÿ
™
“O dÑ*¨d×.LÑ.LÓLð &0°Ñ%@Ü ŸjšjØ×6Ñ6´s¸7¿:¹:³ÈÑ7MÑNóð ",¨KÑ!8ñ)�Ó%ð )3 Ó%ñ !<ô  �ŠØ�z‰zØ×)Ñ)Ø×)Ñ)Ø—>‘>Ø—O‘OØ×%Ñ%Ø×+Ñ+Ø!×/Ñ/Ø'Ø×'Ñ'Ø×)Ñ)ñ
ð 	
r«   c              ƒ  óð  #   • U R                   Gc  [        U R                  U R                  U R                  S9U l         U R
                  (       a�  U R                  c�  [        R                  " U R                   5      u  ol         U VVVs0 s S h  v•N nUR                  =(       d    0 R                  5        H/  u  p4[        UR                  5      U-   US   R                  5       _M1     Ma  U R                  S:”  a  U R                    Vs/ s S h  v•N oUPM  [        R                  " [        R&                  " U R                   5      S[(        R*                  " 5       S9u  U l         nU$  NÙ
 Os  snnnf snnnU l        N“ Nn
 Os  snf nn[        [!        U R                  5       VVs/ sH3  n[#        U Vs/ sH  nU R%                  U5      PM     Os  snf sn5      PM5     Os  snnf snn5      U l         NÝ7f)N©rF   rp   r±   r3   é   ©Úlock)r˜   Ú_aresolve_datar–   rF   rŸ   r    r¢   rd   Úateer­   rS   rK   rU   Úreadr�   Úasync_chain_from_iterableÚrangeÚasync_iter_from_listrÂ   r™   rt   ÚLock)	r¤   Úexamples_copyÚerÀ   Úvaluer¾   Úexamples_listÚ_Úexamples_iters	            r?   Úaget_examplesÚ%_AsyncExperimentManager.aget_examples«  s›  é € Ø�>‰>Ò!Ü+Ø—
‘
Ø—{‘{Ø$(×$=Ñ$=ñˆDŒNð
 ×&×&¨4×+IÑ+IÑ+QÜ0:·²ÀÇÁÓ0OÑ-�œ~ñ $1÷2õ 2àØ()¯©×(;¸×'BÑ'BÖ'D™˜ô ˜Ÿ™“I Ñ$ e¨H¡o×&:Ñ&:Ó&<Ò<á'Dñ %ð ×$Ñ$ qÓ(Ø>B¿nºn× MÓ M°7¢ô )3¯ªÜ×,Ò,¨T¯^©^Ó<¸aÄgÇlÂlÃnñ)
Ñ%ˆŒ˜ð Ðó-2ù÷ 2ð 2�Õ.ó !NùÒ M�Ð MÜ!:ô "' t×'<Ñ'<Ô!=ôñ ">˜Aô -ñ 0=óá/< Gð !%× ?Ñ ?ÀÖ HÚ/<ùôöò ">ùöó
"�•ùsŒ   ‚BG6Â
E.ÂE,
ÂE*ÂE,
ÂAE.Ã-G6ÄFÄFÄF 
ÄFÄFÄAG6Å*E,
Å,E.Å-G6Æ FÆFÆ%G6Æ(G"
Æ6GÇG"
Ç!G6c              ƒ  óP  #   • U R                   b  [        U R                   SS 5      (       dW  [        R                  " U R	                  5       I S h  v•N 5      I S h  v•N nUc  [        S5      e[        UR                  5      $ [        U R                   R                  5      $  NR NH7f)NÚreference_dataset_idz!No examples found in the dataset.)	Ú_experimentÚgetattrrd   Úpy_anextrÖ   rT   rK   r´   rÙ   )r¤   r¾   s     r?   r   Ú&_AsyncExperimentManager.get_dataset_idÌ  sŽ   é € Ø×ÑÑ#¬7Ø×ÑÐ4°d÷,
ñ ,
ô '×/Ò/°d×6HÑ6HÓ6J×0JÓK×KˆGØ‰Ü Ð!DÓEÐEÜ�w×)Ñ)Ó*Ð*Ü�4×#Ñ#×8Ñ8Ó9Ð9ñ	 1KÑKùs%   ‚AB&ÁB"ÁB&ÁB$ÁAB&Â$B&c               ó  #   • U R                   c  [        S5      e[        R                  " [        R                  " U R                   5      S[
        R                  " 5       S9u  U l         nU  S h  v•N nU7v •  M   N
 g 7f)NzRuns not loaded yet.rÆ   rÇ   )rš   rT   rd   rÊ   r™   rt   rÏ   )r¤   rj   rl   s      r?   Ú	aget_runsÚ!_AsyncExperimentManager.aget_runsÖ  sd   é € Ø�:‰:ÑÜÐ3Ó4Ð4Ü%Ÿ?š?Ü×,Ò,¨T¯Z©ZÓ8¸!Ä'Ç,Â,Ã.ñ
ÑˆŒ
�Dñ ÷ 	�#Ø�Iñ	™ùs*   ‚A*B Á,A>Á0A<Á1A>Á4B Á<A>Á>B c               óN  #   • U R                   c*  U R                  5       I S h  v•N   S h  v•N nS/ 07v •  M  [        R                  " [        R                  " U R                   5      S[
        R                  " 5       S9u  U l         nU  S h  v•N nU7v •  M   Nw Np
 g  N
 g 7f)Nr‘   rÆ   rÇ   )r›   rÖ   rd   rÊ   r™   rt   rÏ   )r¤   rÔ   r¥   Úresults       r?   Úaget_evaluation_resultsÚ/_AsyncExperimentManager.aget_evaluation_resultsß  s‘   é € Ø×#Ñ#Ñ+Ø!%×!3Ñ!3Ó!5×5Ð5÷ &�aØ  "�oÕ%ä;E¿?º?Ü×0Ò0°×1IÑ1IÓJØÜ—\’\“^ñ<Ñ8ˆDÔ$Ð&8ñ
 !3÷ �fØ•ñ 6ñ &Ñ5ñÑ 2ùsT   ‚!B%£B¤B%¨B¬B­B°AB%ÂB#ÂB!ÂB#Â	B%ÂBÂB%Â!B#Â#B%c              ƒ  ó¾  #   •  [         R                  " U R                  5       I S h  v•N 5      I S h  v•N nU(       d  [	        S5      eU R
                  (       a  U R                  U5      OS nU R                  X!5        U R                  U R                  S'   U R                  U R                  5       I S h  v•N US9$  N– NŒ! [         a    [	        S5      ef = f N$7f)Nz\No examples found in the dataset. Please ensure the data provided to aevaluate is not empty.z[No examples found in the dataset.Please ensure the data provided to aevaluate is not empty.r4   )r5   )rd   rÜ   rÖ   ÚStopAsyncIterationrT   r¡   Ú_get_projectÚ_print_experiment_startr�   Ú	_metadataÚ_copy)r¤   Úfirst_exampleri   s      r?   r|   Ú_AsyncExperimentManager.astartì  sÙ   é € ð	Ü",×"5Ò"5¸D×<NÑ<NÓ<P×6PÓ"Q×QˆMö ÜðMóð ð 7;×6J×6J�$×#Ñ# MÔ2ÐPTˆØ×$Ñ$ WÔ<Ø,0×,AÑ,Aˆ�‰Ð(Ñ)Ø�z‰zØ×$Ñ$Ó&×&Øð ð 
ð 	
ñ 7QÑQøÜ!ó 	ÜðMóð ð	úñ 'ùsH   ‚C„#C §B>¨C ³C ´C ¸A>CÂ6CÂ7CÂ>C Ã C ÃCÃCc                ób  • 0 nUR                   =(       d    0 R                  5        H…  u  p4U R                  bo  [        UR                  5      U-   U R                  ;   aI  [
        R                  " U R                  [        UR                  5      U-      5      nUS   UUS   S.X#'   M�  XBU'   M‡     [        R                  " UR                  UR                  UR                  UR                  UR                  UR                  UR                  UR                  UUR                   UR"                  S9$ )Nr®   r¯   r°   r²   )r­   rS   r¢   rK   rU   r»   r¼   r   r½   r³   r´   rµ   r¶   rD   r·   r¸   r¹   rº   )r¤   r¾   r¿   rÀ   rÁ   r±   s         r?   Ú_get_example_with_readersÚ1_AsyncExperimentManager._get_example_with_readers  s  € Ø=?ˆØ!(×!4Ñ!4×!:¸× AÑ AÖ CÑˆDà×.Ñ.Ñ:Ü˜Ÿ
™
“O dÑ*¨d×.LÑ.LÓLäŸšØ×2Ñ2´3°w·z±z³?ÀTÑ3IÑJó�ð &0°Ñ%@Ø$Ø!+¨KÑ!8ñ)�Ó%ð )3 Ó%ñ !Dô  �ŠØ�z‰zØ×)Ñ)Ø×)Ñ)Ø—>‘>Ø—O‘OØ×%Ñ%Ø×+Ñ+Ø!×/Ñ/Ø'Ø×'Ñ'Ø×)Ñ)ñ
ð 	
r«   c             ƒ  ó’  ^ ^^^	^
#   • [        T5      m[        T S5      (       d  [        R                  " SS9T l        [        T5      m
SUU UU
4S jjm	U	U 4S jn[        R                  " UU" 5       SS9n[        R                  " US[        R                  " 5       S	9u  pgnT R                  S
 U 5       S U 5       S U 5       S9$ 7f)zÓRun predictions and evaluations in a single pipeline.

This allows evaluators to process results as soon as they're available from
the target function, rather than waiting for all predictions to complete first.
Ú_evaluation_feedback_executoré   ©Úmax_workersc           
   “  ó.  >#   • [        TTR                  U 5      TR                  TR                  TR                  [        T5      TR                  5      I S h  v•N nUS   US   p TR                  TUU S/ 0S.TR                  S9I S h  v•N nU$  N8 N7f)Nr¾   rl   r‘   ©rl   r¾   r¥   ©Úfeedback_executor)	Ú	_aforwardrî   Úexperiment_nameré   rF   r)   r£   Ú_arun_evaluatorsrñ   )r¾   Úpredrl   râ   rB   r¤   r[   Útraceable_targets       €€€€r?   Úprocess_exampleÚQ_AsyncExperimentManager.awith_predictions_and_evaluators.<locals>.process_example4  sª   øé € ä"Ø Ø×.Ñ.¨wÓ7Ø×$Ñ$Ø—‘Ø—‘Ü+¨FÓ3Ø×$Ñ$ó÷ ˆDð   	™?¨D°©K�SØ×0Ñ0ØàØ&Ø+4°b¨/ñð
 #'×"DÑ"Dð 1ð ÷ ˆFð ˆMñ'ñùs$   ƒABÁBÁ1BÂ
BÂBÂBc                ó¦   >#   • TR                  5       I Sh  v•N   Sh  v•N n T" U 5      7v •  M   N N
 TR                  5       I Sh  v•N    g7f)zpCreate a single task per example.

That task is to run the target function and all the evaluators
sequentially.
N)rÖ   Ú_aend)r¾   rþ   r¤   s    €€r?   Úprocess_examplesÚR_AsyncExperimentManager.awith_predictions_and_evaluators.<locals>.process_examplesK  sJ   øé € ð (,×'9Ñ'9Ó';×!;Ð!;÷ /�gÙ% gÓ.Õ.ñ "<ñ /Ð!;ð —*‘*“,×Òùs4   ƒA—2˜Aœ6 4¡6¤A´6¶AÁ
AÁAçü©ñÒMbP?©Ú_eager_consumption_timeouté   rÇ   c               ó8   #   • U  S h  v•N oS   7v •  M   N
 g 7f©Nr¾   r;   ©r<   râ   s     r?   r@   ÚK_AsyncExperimentManager.awith_predictions_and_evaluators.<locals>.<genexpr>b  ó   é € ×6Ð6¨�I×Ó6ùó   ‚„ˆ‰Œ
–˜c               ó8   #   • U  S h  v•N oS   7v •  M   N
 g 7f©Nrl   r;   r
  s     r?   r@   r  c  ó   é € ×7Ð7¨&˜—-Ó7ùr  c               ó8   #   • U  S h  v•N oS   7v •  M   N
 g 7f©Nr¥   r;   r
  s     r?   r@   r  d  ó   é € ×TÐTÀvÐ';× <ÓTùr  ©rj   r¥   )r¾   úschemas.Example)r&   rv   ÚcfÚThreadPoolExecutorrñ   Ú_ensure_async_traceablerd   Úaiter_with_concurrencyrÊ   rt   rÏ   rê   )r¤   r[   rB   rE   r  Úexperiment_resultsÚr1Úr2Úr3rþ   rý   s   ```      @@r?   r„   Ú8_AsyncExperimentManager.awith_predictions_and_evaluators!  s»   üé € ô )¨Ó4ˆ
ä�tÐ<×=Ñ=Ü13×1FÒ1FÐSTÑ1UˆDÔ.ä2°6Ó:Ð÷	ò 	ö.		ô (×>Ò>ØÙÓØ',ñ
Ðô  —_’_Ð%7¸ÄÇÂÃÑP‰
ˆ�à�z‰zÙ6±2Ó6Ù7±BÓ7ÙTÑQSÓTð ð 
ð 	
ùs   ‡C Cc             ƒ  óÐ   #   • U R                  UU[        U5      S9n[        R                  " US[        R
                  " 5       S9u  pEU R                  S U 5       S U 5       S9$ 7f)N)rE   rp   rÆ   rÇ   c               ó8   #   • U  S h  v•N oS   7v •  M   N
 g 7fr	  r;   ©r<   rü   s     r?   r@   Ú<_AsyncExperimentManager.awith_predictions.<locals>.<genexpr>t  s   é € ×2Ð2 t�)�_Ó2ùr  c               ó8   #   • U  S h  v•N oS   7v •  M   N
 g 7fr  r;   r!  s     r?   r@   r"  u  s   é € ×3Ð3¨�u—+Ó3ùr  )rj   )Ú	_apredictr)   rd   rÊ   rt   rÏ   rê   )r¤   r[   rE   Ú_experiment_resultsr  r  s         r?   r†   Ú)_AsyncExperimentManager.awith_predictionsg  sl   é € ð #Ÿn™nØØ+Ü ;¸FÓ Cð -ð 
Ðô
 —’Ð!4°a¼g¿lºl»nÑM‰ˆØ�z‰zÙ2©rÓ2Ù3±Ó3ð ð 
ð 	
ùs   ‚A$A&rs   c             ƒ  óâ   #   • [        U5      nU R                  XS9n[        R                  " US[        R
                  " 5       S9u  pEnU R                  S U 5       S U 5       S U 5       S9$ 7f)Nrs   r  rÇ   c               ó8   #   • U  S h  v•N oS   7v •  M   N
 g 7fr	  r;   r
  s     r?   r@   Ú;_AsyncExperimentManager.awith_evaluators.<locals>.<genexpr>‚  r  r  c               ó8   #   • U  S h  v•N oS   7v •  M   N
 g 7fr  r;   r
  s     r?   r@   r)  ƒ  r  r  c               ó8   #   • U  S h  v•N oS   7v •  M   N
 g 7fr  r;   r
  s     r?   r@   r)  „  r  r  r  )r&   Ú_ascorerd   rÊ   rt   rÏ   rê   )r¤   rB   rE   r  r  r  r  s          r?   rˆ   Ú(_AsyncExperimentManager.awith_evaluatorsx  sn   é € ô )¨Ó4ˆ
Ø!Ÿ\™\¨*˜\ÐVÐÜ—_’_Ð%7¸ÄÇÂÃÑP‰
ˆ�Ø�z‰zÙ6±2Ó6Ù7±BÓ7ÙTÑQSÓTð ð 
ð 	
ùs   ‚A-A/c              ƒ  ó°   #   • [        U5      nU R                  U5      nU R                  U R                  5       I S h  v•N U R	                  5       US9$  N7f)N)rj   r¦   )r+   Ú_aapply_summary_evaluatorsrê   rÖ   rß   )r¤   rC   Úwrapped_evaluatorsÚaggregate_feedback_gens       r?   r‡   Ú0_AsyncExperimentManager.awith_summary_evaluators‡  sZ   é € ô 6Ð6HÓIÐØ!%×!@Ñ!@ÐASÓ!TÐØ�z‰zØ×$Ñ$Ó&×&Ø—‘Ó!Ø2ð ð 
ð 	
Ù&ùs   ‚;A½A¾Ac               óâ   #   • [         R                  " U R                  5       U R                  5       I S h  v•N U R	                  5       5        S h  v•N u  pn[        UUUS97v •  M   N3 N
 g 7f)Nrö   )rd   Ú	async_ziprß   rÖ   rã   r   )r¤   rl   r¾   r¥   s       r?   Úaget_resultsÚ$_AsyncExperimentManager.aget_results“  sj   é € Ü6@×6JÒ6JØ�N‰NÓ D×$6Ñ$6Ó$8×8¸$×:VÑ:VÓ:Xô7
÷ 	Ñ2�#Ð 2ô &ØØØ#5ñõ ñ 9ñ	ñ 7
ùs3   ‚3A/µA)
¶A/ÁA-ÁA+ÁA-ÁA/Á+A-Á-A/c              ƒ  ó    #   • U R                   c  S/ 0$ SU R                    VVs/ s S h  v•N nUS    H  nUPM     M   N
 Os  snnf snn0$ 7f)Nr‘   )rœ   )r¤   r‘   Úress      r?   Úaget_summary_scoresÚ+_AsyncExperimentManager.aget_summary_scores�  s`   é € Ø× Ñ Ñ(Ø˜r�?Ð"àà%)×%:Ò%:÷ô à!Ø" 9Ô-�Có á-ñ óùö ð
ð 	
ùs.   ‚"A¤A¦A
ª?«A
®A¿A
ÁAÁAc              óÄ   ^ ^^#   • [        U5      mUUU 4S jn[        R                  " X$" 5       SS9  S h  v•N nU7v •  M   N
 T R                  5       I S h  v•N    g 7f)Nc            
    óô   >#   • TR                  5       I S h  v•N   S h  v•N n [        TTR                  U 5      TR                  TR                  TR
                  TTR                  5      7v •  MV   NZ NS
 g 7fr:   )rÖ   rù   rî   rú   ré   rF   r£   )r¾   Úfnrp   r¤   s    €€€r?   Úpredict_allÚ6_AsyncExperimentManager._apredict.<locals>.predict_all³  sm   øé € Ø'+×'9Ñ'9Ó';×!;Ð!;÷ 
�gäØØ×2Ñ2°7Ó;Ø×(Ñ(Ø—N‘NØ—K‘KØ'Ø×(Ñ(óõ ñ "<ñ 
Ñ!;ùs0   ƒA8—A2˜A8œA6 A4¡A6¤AA8Á4A6Á6A8r  r  )r  rd   r  r  )r¤   r[   rE   rp   r>  râ   r=  s   `  `  @r?   r$  Ú!_AsyncExperimentManager._apredictª  sY   úé € ô % VÓ,ˆ÷	ô '×=Ò=Ø˜[›]Àuò
÷ 	�&ð �Lñ	ð 
ð
 �j‰j‹l×Òùs1   ….A ³A·A¸A»A ÁAÁA ÁAÁA c               óØ   ^ ^^#   • [         R                  " SS9 mUUU 4S jn[        R                  " X#" 5       SS9  S h  v•N nU7v •  M   N
 S S S 5        g ! , (       d  f       g = f7f)Nrò   ró   c                ór   >#   • TR                  5         S h  v•N n TR                  TU TS97v •  M   N
 g 7f)Nr÷   )r5  rû   )Úcurrent_resultsrB   rø   r¤   s    €€€r?   Ú	score_allÚ2_AsyncExperimentManager._ascore.<locals>.score_allÎ  sA   øé € Ø-1×->Ñ->Ô-@÷ ˜/à×/Ñ/Ø" OÐGXð 0ð õ ñÑ-@ùs   ƒ7”5˜3™5œ7³5µ7r  r  )r  r  rd   r  )r¤   rB   rE   rD  râ   rø   s   ``   @r?   r,  Ú_AsyncExperimentManager._ascoreÇ  sZ   úé € ô
 ×"Ò"¨qÒ1Ð5F÷ô !+× AÒ AØ £Èò!÷ �fð •ñð !÷ 2×1Ö1üs?   …A*š#A½AÁAÁAÁAÁAÁAÁ	A*Á
A'Á#A*c           
   ƒ  ó  ^ ^^^#   • [         R                  " 5       n0 US   =(       d    0 EST R                  0En[         R                  " S0 0 UESUT R                  (       d  SOST R
                  S.ED6   US   mUS   mUS	   nSUUUU 4S
 jjn/ nU H"  n	UR                  U" U	5      I S h  v•N 5        M$     U H  n
U
c  M  US   R                  U
5        M     [        TTUS9sS S S 5        $  NB! , (       d  f       g = f7f)NrD   r5   rB   ÚlocalT©Úproject_namerD   ÚenabledrF   rl   r¾   r¥   c              “  ó   >#   • [         R                  " 5       n U R                  TTR                  T	5      US9I S h  v•N nTR                  R                  U5      nTR                  (       a4  TR                  R                  UTTR                  5       R                  T
S9  U$  Nf! [         Ga  n U R                  n[        U Vs/ sH  n[        UU[        U5      SS0S9PM     Os  snf snS9nTR                  R                  U5      nTR                  (       a4  TR                  R                  UTTR                  5       R                  T
S9  Us S nA$ ! [         a"  n[        R!                  SU 35         S nAOS nAff = f[        R#                  S[        U 5       S	TR                   S
[        U5       3SS9   S nAg S nAff = f7f)N)rl   r¾   Úevaluator_run_id)rl   Ú
project_idÚ	_executorÚerrorT)Úkeyr¸   ÚcommentÚextra)r‘   zError parsing feedback keys: zError running evaluator z on run ú: ©Úexc_info)rL   Úuuid4Úaevaluate_runrî   rF   Ú_select_eval_resultsr¡   Ú_log_evaluation_feedbackÚ_get_experimentrU   Ú	ExceptionÚfeedback_keysr.   r-   ÚreprrV   rW   rP  )Ú	evaluatorrM  Úevaluator_responseÚselected_resultsrÑ   r]  rQ  Úerror_responseÚe2r¾   rø   rl   r¤   s            €€€€r?   Ú_run_single_evaluatorÚG_AsyncExperimentManager._arun_evaluators.<locals>._run_single_evaluatorò  s×  øé € Ü#'§:¢:£<Ð ð3Ø/8×/FÑ/FØØ $× >Ñ >¸wÓ GØ)9ð 0Gð 0÷ *Ð&ð
 (,§{¡{×'GÑ'GØ*ó(Ð$ð ×+×+ØŸ™×<Ñ<Ø.Ø #Ø'+×';Ñ';Ó'=×'@Ñ'@Ø&7ð	 =ñ ð ,Ð+ñ!*øô" !ô !ðØ(1×(?Ñ(?˜ä):ñ ,9ó%ñ ,9 Cô !1Ø(+Ø2BÜ,0°«GØ+2°D¨/ô	!"ò ,9ùô%ñ
*˜ð ,0¯;©;×+KÑ+KØ*ó,Ð(ð  ×/×/Ø ŸK™K×@Ñ@Ø .Ø$'Ø+/×+?Ñ+?Ó+A×+DÑ+DØ*;ð	 Añ ð  0Õ/øÜ$ó ÜŸ™Ð'DÀRÀDÐ%IÔJÜûðúô —L‘LØ2´4¸	³?Ð2Cð D Ø #§¡˜x r¬$¨q«'¨ð4à!%ð !ö ûð;!üsv   ƒGš#B& ½B$¾A%B& Â#GÂ$B& Â&GÂ2EÃ C(
Ã'A,EÅGÅGÅ
FÅ#F Å;GÆ FÆ<GÇGÇGÇGr‘   rö   r;   )r_  r/   )	ÚrhÚget_tracing_contextrú   Útracing_contextr¡   rF   ÚappendÚextendr   )r¤   rB   rC  rø   Úcurrent_contextrD   Úeval_resultsrd  Úall_resultsr_  râ   r¾   rl   s   `  `       @@r?   rû   Ú(_AsyncExperimentManager._arun_evaluatorsÚ  s,  ûé € ô ×0Ò0Ó2ˆð
Ø˜zÑ*×0¨bð
à˜T×1Ñ1Ð2ð
ˆô ×Òñ 
ðØ!ðà ,Ø$Ø*.×*>×*>™7ÀDØŸ+™+òó
ð " %Ñ(ˆCØ% iÑ0ˆGØ*Ð+?Ñ@ˆL÷5ò 5ðn ˆKÛ'�	Ø×"Ñ"Ñ)>¸yÓ)I×#IÖJñ (ó &�ØÓ%Ø  Ñ+×2Ñ2°6Ö:ñ &ô 'ØØØ#/ñ÷W
ñ 
ñL $J÷M
õ 
üs6   †A0DÁ68C3Â.C1Â/C3Ã"C3Ã'
DÃ1C3Ã3
DÃ=Dc               ó¾  #   • / / p2[         R                  " U R                  5       I S h  v•N 5      n[         R                  " U R	                  5       U5        S h  v•N u  pVUR                  U5        UR                  U5        M/   N\ N,
 / nU R                  (       a  U R                  5       R                  OS n[        R                  " 5       n	0 U	S   =(       d    0 EU R                  US.En
[        R                  " S0 0 U	ESU
U R                  (       d  SOSU R                  S.ED6   U GH	  n U" X#5      nU R                  R                  UUR                  S9nUR!                  U5        U R                  (       a{  U Hs  nUR#                  S1S	9nUR%                  S
S 5      n[         R&                  " [(        R*                  " 5       U R                  R,                  40 UDS UUS.D6I S h  v•N    Mu     MÎ  MÐ  ! [.         a/  n[0        R3                  S[5        U5       SU 3SS9   S nAGM  S nAff = f   S S S 5        O! , (       d  f       O= fSU07v •  g 7f)NrD   )r5   Úexperiment_idrB   rH  TrI  )Úfn_nameÚtarget_run_id)ÚexcludeÚevaluator_info)Úrun_idrN  Úsource_infoz Error running summary evaluator rT  rU  r‘   r;   )rd   r™   rÖ   r4  rß   ri  r¡   r[  rU   rf  rg  rú   rh  rF   rY  Ú__name__rj  Ú
model_dumpÚpopre   rf   rg   Úcreate_feedbackr\  rV   rP  r^  )r¤   rC   rj   ÚexamplesÚasync_examplesrl   r¾   Úaggregate_feedbackrN  rk  rD   r_  Úsummary_eval_resultÚflattened_resultsrâ   Úfeedbackrt  rÑ   s                     r?   r/  Ú2_AsyncExperimentManager._aapply_summary_evaluators6  sZ  é € ð ˜RˆhÜ#×9Ò9À×@RÑ@RÓ@T×:TÓUˆÜ",×"6Ò"6Ø�N‰NÓ˜nô#
÷ 	%‘,�#ð �K‰K˜ÔØ�O‰O˜GÖ$ñ ;Uñ	%ð #
ð
  ÐØ26×2F×2F�T×)Ñ)Ó+×.Ò.ÈDˆ
Ü×0Ò0Ó2ˆð
Ø˜zÑ*×0¨bð
ð #×2Ñ2Ø!+ñð
ˆô ×Òñ 
ðØ!ðà ,Ø$Ø*.×*>×*>™7ÀDØŸ+™+òó
ô 0�	ðÙ*3°DÓ*CÐ'Ø(,¯©×(HÑ(HØ+Ø )× 2Ñ 2ð )Ið )Ð%ð '×-Ñ-Ð.?Ô@Ø×+×+Û&7˜FØ'-×'8Ñ'8À/ÐARÐ'8Ð'S˜HØ-5¯\©\Ð:JÈDÓ-Q˜NÜ",×":Ò":Ü +× 8Ò 8Ó :Ø $§¡× ;Ñ ;ñ#ð #+ð#ð (,Ø+5Ø,:ó#÷ ó ó '8ñ ,øô !ó Ü—L‘LØ:¼4À	»?Ð:KÈ2ÈaÈSÐQØ!%ð !÷ ð ûðúñ) 0÷
÷ 
ö 
úðD Ð,Ð-Ô-ùsƒ   ‚'I©Bª-IÁB
ÁBÁB
Á(IÂB
Â
B IÄ*IÄ2B=G=Ç/G2
Ç0	G=Ç9IÇ=
H6È#H1È*IÈ1H6È6IÈ;	IÉ
IÉIc              ƒ  ó  #   • / nU R                  5       I S h  v•N   S h  v•N nUR                  (       d  M  UR                  UR                  5        M9   N= N6
 U(       a  [        U5      OS nU(       a  UR	                  5       $ S $ 7fr:   )rÖ   r·   ri  ÚmaxÚ	isoformat)r¤   r·   r¾   Úmax_modified_ats       r?   Ú_get_dataset_versionÚ,_AsyncExperimentManager._get_dataset_versionn  sx   é € ØˆØ#'×#5Ñ#5Ó#7×7Ð7÷ 	8�'Ø×"×"Ñ"ð ×"Ñ" 7×#6Ñ#6Ö7ñ	 8ñ 	8Ð7ö /:œ#˜kÔ*¸tˆÞ.=ˆ×(Ñ(Ó*ÐGÀ4ÐGùs4   ‚B˜A™B�A¡A¢A¥B¹BÁAÁ.Bc              ƒ  óÌ  #   • [        5       nU R                  5       I S h  v•N   S h  v•N nUR                  (       a‚  UR                  R                  S5      (       ab  [	        UR                  S   [
        5      (       a@  UR                  S    H+  n[	        U[        5      (       d  M  UR                  U5        M-     Mœ  UR                  S5        M¯   N³ N¬
 [        U5      $ 7f)NÚdataset_splitÚbase)ÚsetrÖ   rD   ÚgetrJ   rY   rK   Úadd)r¤   Úsplitsr¾   Úsplits       r?   Ú_get_dataset_splitsÚ+_AsyncExperimentManager._get_dataset_splitsy  s²   é € Ü“ˆØ#'×#5Ñ#5Ó#7×7Ð7÷ 
	#�'à× × Ø×$Ñ$×(Ñ(¨×9Ñ9Ü˜w×/Ñ/°Ñ@Ä$×GÑGà$×-Ñ-¨oÔ>�EÜ! %¬×-Ó-ØŸ
™
 5Ö)ó ?ð —
‘
˜6Ö"ñ 8ñ 
	#Ð7ô �F‹|Ðùs6   ‚C$ C¡C$¥C©CªC­A9C$Â*+C$ÃCÃC$c              ƒ  ó\  #   • U R                   (       d  g U R                  nUc  [        S5      eU R                  5       nU R	                  5       I S h  v•N US'   U R                  5       I S h  v•N US'   U R                  R                  UR                  0 UR                  EUES9  g  NU N<7f)NzExperiment not started yet.Údataset_versionÚdataset_splits)rD   )
r¡   rÚ   rT   Ú_get_experiment_metadatar†  r�  rF   Úupdate_projectrU   rD   )r¤   r5   Úproject_metadatas      r?   r  Ú_AsyncExperimentManager._aend‰  s¯   é € Ø×#×#ØØ×%Ñ%ˆ
ØÑÜÐ:Ó;Ð;à×8Ñ8Ó:ÐØ48×4MÑ4MÓ4O×.OÐÐ*Ñ+Ø37×3KÑ3KÓ3M×-MÐÐ)Ñ*Ø�‰×"Ñ"Ø�M‰MðØ×%Ñ%ðà"ðð 	#ò 	
ñ /PÙ-Mùs$   ‚AB,ÁB(ÁB,Á-B*Á.;B,Â*B,c                ó¶  • U R                   4nU R                  U R                  U R                  U R                  U R
                  U R                  U R                  U R                  U R                  U R                  U R                  U R                  U R                  S.n[        U5      [        U[        U5      S  5      -   n0 UEUEnU R                   " U0 UD6$ )N)r5   rD   rj   rF   r¥   r¦   r§   ro   rp   rq   r6   r¨   rI   )r–   rÚ   ré   rš   rF   r›   rœ   rž   r—   rŸ   r    r¡   r¢   r£   rY   Úlenr©   )r¤   Úargsr\   Údefault_argsÚdefault_kwargsÚ	full_argsÚfull_kwargss          r?   rê   Ú_AsyncExperimentManager._copy›  sÆ   € ØŸ
™
�}ˆà×*Ñ*ØŸ™Ø—J‘JØ—k‘kØ"&×":Ñ":Ø#×4Ñ4Ø ×.Ñ.Ø"×2Ñ2Ø#'×#<Ñ#<Ø!%×!8Ñ!8Ø"×2Ñ2Ø(,×(FÑ(FØ"×2Ñ2ñ
ˆô ˜“J¤ l´3°t³9°;Ð&?Ó!@Ñ@ˆ	Ø2˜Ð2¨6Ð2ˆØ�~Š~˜yÐ8¨KÑ8Ð8r«   )r¢   r–   r£   rñ   r›   r—   r˜   rŸ   rž   r�   r    rš   rœ   r¡   )NNNNNNNr3   NNFFTNÚlog) r5   z+Optional[Union[schemas.TracerSession, str]]rD   úOptional[dict]rj   zBOptional[Union[Iterable[schemas.Run], AsyncIterable[schemas.Run]]]rF   úOptional[langsmith.Client]r¥   ú*Optional[AsyncIterable[EvaluationResults]]r¦   r¤  rH   úOptional[str]r4   rz   r§   úOptional[int]ro   úOptional[list[str]]rp   rO   rq   rO   r6   rO   r¨   r¢  rI   úLiteral['log', 'ignore']r8   ú-Union[DATA_T, AsyncIterable[schemas.Example]])r¾   r  Úreturnr  )rª  úAsyncIterator[schemas.Example]©rª  rK   )rª  zAsyncIterator[schemas.Run])rª  ú AsyncIterator[EvaluationResults])rª  r{   r:   )rE   r¦  r[   r…   rB   ú*Sequence[Union[EVALUATOR_T, AEVALUATOR_T]]rª  r{   )rE   r¦  r[   r…   rª  r{   )rB   r®  rE   r¦  rª  r{   )rC   úSequence[SUMMARY_EVALUATOR_T]rª  r{   ©rª  ú"AsyncIterator[ExperimentResultRow])rª  zdict[str, list[dict]])NF)rE   r¦  rp   rO   r[   r…   rª  zAsyncIterator[_ForwardResults])rB   úSequence[RunEvaluator]rE   r¦  rª  r±  )rB   r²  rC  r   rø   zcf.ThreadPoolExecutorrª  r   )rC   r¯  rª  r­  ©rª  r¥  )rª  r§  ©rª  ÚNone)r›  r
   r\   r
   rª  r{   )rw  Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r•   rÂ   rÖ   r   rß   rã   r|   rî   r„   r†   rˆ   r‡   r5  r9  r$  r,  rû   r/  r†  r�  r  rê   Ú__static_attributes__Ú__classcell__)r©   s   @r?   r{   r{   (  sk  ø† ñ&ðX CGØ#'ØSWØ-1ØIMØFJØ%)Ø Ø&*Ø.2Ø$)Ø"'Ø#Ø37Ø38ð%,.ð @ð	,.ð
 !ð,.ð Qð,.ð +ð,.ð Gð,.ð Dð,.ð #ð,.ð ð,.ð $ð,.ð ,ð,.ð "ð,.ð  ð,.ð  ð!,.ð" #1ð#,.ð$ 1ð%,.à;÷,.ð ,.ô\*
ôXôB:ôôô
ô*
ðJ *.ðD
ð
 'ðD
àðD
ð ?ðD
ð 
!õD
ðT *.ð	
ð 'ð	
àð
ð
 
!õ
ð* *.ñ	
à>ð
ð 'ð	
ð
 
!õ
ð

à9ð

ð 
!ô

ôô	
ð" *.Ø$)ðð 'ð	ð
 "ðàðð 
(õð@ *.ðà*ðð 'ðð 
,õ	ð&Zà*ðZð -ðZð 1ð	Zð
 
ôZðx6.Ø"?ð6.à	)ô6.ôp	Hôô 
÷$9ò 9r«   r{   c                  óÒ   • \ rS rSr  SS jr\SS j5       r\SS j5       r\SS j5       rSS jr	SS jr
SS jrSS	 jrSS
 jr S     SS jjrSS jrSS jrSS jrSS jrSrg)r1   i±  c                óÖ   • Xl         / U l        [        R                  " 5       U l        [        R
                  " U R                  U R                   5      5      U l        SU l        S U l	        g )Nr   )
Ú_managerÚ_resultsrt   Ú	ConditionÚ
_conditionÚcreate_taskÚ_process_dataÚ_taskÚ_processed_countÚ_comparison_url)r¤   Úexperiment_managers     r?   r•   ÚAsyncExperimentResults.__init__²  sQ   € ð +ŒØ35ˆŒÜ!×+Ò+Ó-ˆŒÜ×(Ò(¨×);Ñ);¸D¿M¹MÓ)JÓKˆŒ
Ø !ˆÔØ.2ˆÕr«   c                ó.   • U R                   R                  $ r:   )r¾  rú   ©r¤   s    r?   rú   Ú&AsyncExperimentResults.experiment_name½  s   € à�}‰}×,Ñ,Ð,r«   c                óJ   • U R                   R                  5       R                  $ )zThe ID of the experiment.)r¾  r[  rU   rÊ  s    r?   rp  Ú$AsyncExperimentResults.experiment_idÁ  s   € ð �}‰}×,Ñ,Ó.×1Ñ1Ð1r«   c                óJ   • U R                   R                  5       R                  $ )z.The URL of the experiment in the LangSmith UI.)r¾  r[  ÚurlrÊ  s    r?   rÏ  ÚAsyncExperimentResults.urlÆ  s   € ð �}‰}×,Ñ,Ó.×2Ñ2Ð2r«   c              ƒ  óR   #   • U R                   R                  5       I Sh  v•N $  N7f)z:Get the ID of the dataset associated with this experiment.N)r¾  r   rÊ  s    r?   r   Ú%AsyncExperimentResults.get_dataset_idË  s   é € à—]‘]×1Ñ1Ó3×3Ð3Ñ3ùs   ‚' %¡'c              ƒ  óx  #   • U R                   R                  5       nU R                  (       d~  UR                  (       am  U R                   R	                  5       I Sh  v•N nUR                  R                  S5      S   nUR                  S5      S   nU SU SUR                   3U l        U R                  $  N[7f)z7Get the URL to the comparison view for this experiment.NÚ?r   z/projects/p/z
/datasets/z/compare?selectedSessions=)r¾  r[  rÆ  rÏ  r   r�  rU   )r¤   r5   r´   Úproject_urlÚbase_urls        r?   Úget_comparison_urlÚ)AsyncExperimentResults.get_comparison_urlÏ  s¢   é € à—]‘]×2Ñ2Ó4ˆ
Ø×#×#¨
¯¯Ø#Ÿ}™}×;Ñ;Ó=×=ˆJØ$Ÿ.™.×.Ñ.¨sÓ3°AÑ6ˆKØ"×(Ñ(¨Ó8¸Ñ;ˆHà�*˜J z lð 3$Ø$.§M¡M ?ð4ð Ô ð ×#Ñ#Ð#ñ >ùs   ‚AB:ÁB8ÁAB:c                ó   • U $ r:   r;   rÊ  s    r?   rn   Ú AsyncExperimentResults.__aiter__Ü  s   € Øˆr«   c              ƒ  ó  #   • U R                    IS h  v•N    U R                  [        U R                  5      :  aA  U R                  U R                     nU =R                  S-  sl        UsS S S 5      IS h  v•N   $ U R                  R                  5       (       a%  U R                  R                  5       nUb  Ue[        eU R                   R                  5       I S h  v•N   MÌ   NÑ No N
! , IS h  v•N  (       d  f       g = f7f)Nr3   )	rÁ  rÅ  rš  r¿  rÄ  ÚdoneÚ	exceptionræ   r‰   )r¤   râ   Úexcs      r?   Ú	__anext__Ú AsyncExperimentResults.__anext__ß  s¼   é € Ø—?—?“?ØØ×(Ñ(¬3¨t¯}©}Ó+=Ó=Ø!Ÿ]™]¨4×+@Ñ+@ÑA�FØ×)Ò)¨QÑ.Õ)Ø!÷ #—?‘?ð —Z‘Z—_‘_×&Ñ&ØŸ*™*×.Ñ.Ó0�CØ‘Ø!˜	Ü,Ð,Ø—o‘o×*Ñ*Ó,×,Ð,ñ ó #ñ -÷ #—?–?üsd   ‚D“C%”D—AC+Á+DÁ7C'Á8DÁ=A!C+ÃC)ÃC+Ã%DÃ'DÃ)C+Ã+DÃ1C4Ã2DÃ>Dc              ƒ  óN  #   • [        5       nU" UR                  5       5        S h  v•N nU R                   IS h  v•N   U R                  R	                  U5        U R                  R                  5         S S S 5      IS h  v•N   Me   N` NM N
! , IS h  v•N  (       d  f       Mƒ  = f
 UR                  5       I S h  v•N  nU R                   IS h  v•N    X@l        U R                  R                  5         S S S 5      IS h  v•N    g ! , IS h  v•N  (       d  f       g = f7fr:   )	r#   r5  rÁ  r¿  ri  Únotifyr9  rœ   Ú
notify_all)r¤   r�   ÚtqdmÚitemÚsummary_scoress        r?   rÃ  Ú$AsyncExperimentResults._process_dataí  s°   é € Ü‹|ˆÙ˜w×3Ñ3Ó5Ô6÷ 	)�$Ø——“Ø—‘×$Ñ$ TÔ*Ø—‘×&Ñ&Ô(÷ '—‘ñ	)ß&——“úð 7ð  '×:Ñ:Ó<×<Ð<ˆØ—?—?”?Ø$2Ô!Ø�O‰O×&Ñ&Ô(÷ #—?—?—?—?�?üs­   ‚!D%£B)§B¨B)«D%¼B
½D%Á 6BÁ6D%ÂBÂD%ÂB)Â
D%ÂD%ÂB&	ÂBÂB&	Â!D%Â=C Â>D%ÃCÃD%Ã!DÃ9D%ÄDÄD%ÄD"ÄDÄD"ÄD%Nc                ó*   • [        U R                  XS9$ )N)ÚstartÚend)r*   r¿  )r¤   ré  rê  s      r?   Ú	to_pandasÚ AsyncExperimentResults.to_pandasø  s   € ô ˜$Ÿ-™-¨uÑ>Ð>r«   c                óÐ   • SS K nU R                  (       aB  UR                  R                  S5      (       a"  U R	                  SS5      nUR                  5       $ U R                  5       $ )Nr   Úpandasé   )Úimportlib.utilr¿  ÚutilÚ	find_specrë  Ú_repr_html_Ú__repr__)r¤   Ú	importlibÚdfs      r?   ró  Ú"AsyncExperimentResults._repr_html_ý  sI   € Ûà�=�=˜YŸ^™^×5Ñ5°h×?Ñ?Ø—‘  1Ó%ˆBØ—>‘>Ó#Ð#à—=‘=“?Ð"r«   c                ó,   • [        U R                  5      $ r:   )rš  r¿  rÊ  s    r?   Ú__len__ÚAsyncExperimentResults.__len__  s   € Ü�4—=‘=Ó!Ð!r«   c                ó"   • SU R                    S3$ )Nz<AsyncExperimentResults Ú>)rú   rÊ  s    r?   rô  ÚAsyncExperimentResults.__repr__	  s   € Ø)¨$×*>Ñ*>Ð)?¸qÐAÐAr«   c              ƒ  ó8   #   • U R                   I S h  v•N   g  N7fr:   )rÄ  rÊ  s    r?   r‰   ÚAsyncExperimentResults.wait  s   é € Ø�j‰j×Óùs   ‚’“)rÆ  rÁ  r¾  rÅ  r¿  rœ   rÄ  )rÇ  r{   r¬  )rª  z	uuid.UUIDr³  r°  )rª  r   )r�   r{   rª  rµ  )r   N)ré  r¦  rê  r¦  rª  Ú	DataFrame)rª  rz   r´  )rw  r¶  r·  r¸  r•   Úpropertyrú   rp  rÏ  r   r×  rn   rß  rÃ  rë  ró  rù  rô  r‰   rº  r;   r«   r?   r1   r1   ±  sœ   † ð	3à3ô	3ð ó-ó ð-ð ó2ó ð2ð ó3ó ð3ô4ô$ôô-ô	)ð >Bð?Ø"ð?Ø-:ð?à	õ?ô
#ô"ôB÷r«   c              ƒ  óž  ^^#   • S mSU4S jjnSU4S jjn[         R                  " UU0 UESTR                  =(       d    TR                  R	                  5       0EUS9n	US:X  a  TR
                  U	S'   OUS:X  a  X‰S'   O[        S	U< 35      e[         R                  " S
S9    [        U 5      n
U
 Vs/ sH  n[        TU5      PM     nnU " USU	06I S h  v•N   [        [        [        R                   T5      TS9sS S S 5        $ s  snf  N4! [         a"  n[        R                  SU 3S
SS9   S nANYS nAff = f! , (       d  f       g = f7f)Nc                ó
   >• U mg r:   r;   )Úrrl   s    €r?   Ú_get_runÚ_aforward.<locals>._get_run  s   ø€ à‰r«   c                ó(   >• TR                   U l        g r:   )rU   rh   )r  r¾   s    €r?   Ú_set_reference_example_idÚ,_aforward.<locals>._set_reference_example_id  s   ø€ Ø!(§¡ˆÕr«   Úexample_version)Úon_endrJ  rD   rF   r¡  rh   ÚignoreÚ_on_successz2Unrecognized error_handling value: error_handling=T)rK  Úlangsmith_extrazError running target function: r3   )rV  Ú
stacklevel)rl   r¾   )r  zrun_trees.RunTreerª  rµ  )r  z
rt.RunTreerª  rµ  )rf  ÚLangSmithExtrar·   r³   r„  rU   rT   rh  r   rÛ   r\  rV   rP  r   r   r   ry   )r=  r¾   rú   rD   rF   rp   rI   r  r  r  Ú	arg_namesÚargnr›  rÑ   rl   s    `            @r?   rù   rù     sQ  ùé € ð &*€C÷÷,ô ×'Ò'ØØ$ð
Øð
à × 3Ñ 3× I°w×7IÑ7I×TÑTÓVñ
ð ñ€Oð ˜ÓØ29·*±*ˆÐ.Ò/Ø	˜8Ó	#Ø)B˜Ò&äÐN¸~Ñ>OÐPÓQÐQä	×	Ò	 DÓ	)ð	Ü(¨Ó,ˆIÙ7@ÓA±y¨t”G˜G TÖ*±yˆDÐAÙ�dÐ<¨OÑ<×<Ð<ô
 Ü”W—[‘[ #Ó&Øñ
÷ 
*Ñ	)ùò BÙ<øÜó 	Ü�L‰LØ1°!°Ð5ÀÐQRð ö ûð	ú÷ 
*Õ	)üsf   „BEÂ#D<Â%DÂ4DÃ	DÃDÃDÃ!D<Ã<
EÄDÄ
D9ÄD4Ä/D<Ä4D9Ä9D<Ä<
E
ÅEc                ó   • SU ;   a  U S   $ U $ )Nrµ   r;   )rµ   s    r?   Ú_default_process_inputsr  A  s   € Ø'¨6Ó1ˆ6�(ÑÐ=°vÐ=r«   c                óT  • [         R                  " U 5      (       d6  [        U 5      (       d&  [        U 5      (       a  [	        S5      e[	        S5      e[
        R                  " U 5      (       a  U $ [        U 5      (       a  U R                  n [
        R                  " S[        S9" U 5      $ )NzÔTarget must be an async function. For sync functions, use evaluate. Example usage:

async def predict(inputs: dict) -> dict:
    # do work, like chain.invoke(inputs)
    return {...}
await aevaluate(predict, ...)zÚTarget must be a callable async function. Received a non-callable object. Example usage:

async def predict(inputs: dict) -> dict:
    # do work, like chain.invoke(inputs)
    return {...}
await aevaluate(predict, ...)ÚAsyncTarget)rÀ   Úprocess_inputs)
rt   ru   r    ÚcallablerT   rf  Úis_traceable_functionÚainvokeÚ	traceabler  )r[   s    r?   r  r  E  sž   € ô ×&Ò& v×.Ñ.Ô7MÈf×7UÑ7UÜ�F×ÑÜð0óð ô ð0óð ô 
×Ò ×'Ñ'Øˆä! &×)Ñ)Ø—^‘^ˆFÜ�|Š|ØÜ2ò
ð óð 	r«   )rp   c               ó”   • [        U [        5      (       a  [        R                  " U 5      $ [        R                  " [	        XUS95      $ )z*Return the examples for the given dataset.rÅ   )rJ   r   rd   r™   r%   )r8   rF   rp   s      r?   rÉ   rÉ   f  s@   € ô �$œ×&Ñ&Ü×/Ò/°Ó5Ð5Ü×+Ò+Ü�dÐ?RÑSóð r«   ÚTc               óH   #   • U  H  nU  Sh  v•N nU7v •  M     g N
 M  7f)zChain multiple async iterables.Nr;   )ÚiterableÚsub_iterablerå  s      r?   rÌ   rÌ   w  s'   é € ó !ˆÙ&÷ 	�$Ø�Jò !ñ	š,ùs   ‚"ŠŽ�’"�Ÿ"c               ó(   #   • U  H  nU7v •  M
     g7f)z0Convert a list of examples to an async iterable.Nr;   )r{  r¾   s     r?   rÎ   rÎ   €  s   é € ó ˆØ�ò ùs   ‚)NNNNNNr   r3   NTNTr¡  ) r8   zNUnion[DATA_T, AsyncIterable[schemas.Example], Iterable[schemas.Example], None]rB   ú4Optional[Sequence[Union[EVALUATOR_T, AEVALUATOR_T]]]rC   ú'Optional[Sequence[SUMMARY_EVALUATOR_T]]rD   r¢  r7   r¥  rH   r¥  rE   r¦  r4   rz   rF   r£  rG   rO   r5   ú6Optional[Union[schemas.TracerSession, str, uuid.UUID]]r6   rO   rI   r¨  r[   zVUnion[ATARGET_T, AsyncIterable[dict], Runnable, str, uuid.UUID, schemas.TracerSession]r\   r
   rª  r1   )NNNr   NFT)rB   r"  rC   r#  rD   r¢  rE   r¦  rF   r£  rb   rO   rG   rO   r5   z,Union[str, uuid.UUID, schemas.TracerSession]rª  r1   )NNNNNNr3   NTNTr¡  )r8   r©  rB   r"  rC   r#  rD   r¢  r7   r¥  rH   r¥  rE   r¦  r4   rz   rF   r£  rG   rO   r5   r$  r6   rO   rI   r¨  r[   zFUnion[ATARGET_T, AsyncIterable[dict], Iterable[schemas.Run], Runnable]rª  r1   )Fr¡  )r=  ú,rh.SupportsLangsmithExtra[[dict], Awaitable]r¾   r  rú   rK   rD   ÚdictrF   úlangsmith.Clientrp   rO   rI   r¨  rª  r   )rµ   r&  rª  r&  )r[   r…   rª  r%  )r8   r©  rF   r'  rp   rO   rª  r«  )r  zIterable[AsyncIterable[T]]rª  zAsyncIterator[T])r{  zlist[schemas.Example]rª  zAsyncIterable[schemas.Example])Yr¹  Ú
__future__r   rt   Úconcurrent.futuresÚfuturesr  rf   r»   Úloggingr€   rL   Úcollections.abcr   r   r   r   r   Útypingr	   r
   r   r   r   r   r   r   Ú	langsmithr   rf  r   r   rx   r   r}   Úlangsmith._internalr   rd   Ú#langsmith._internal._beta_decoratorr   Úlangsmith.evaluation._runnerr   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   Úlangsmith.evaluation.evaluatorr,   r-   r.   r/   rî  ÚpdÚlangchain_core.runnablesr0   r   Ú	getLoggerrw  rV   r&  r…   r`   rX   rZ   r{   r1   rù   r  r  rÉ   r  rÌ   rÎ   r;   r«   r?   Ú<module>r6     s  ðÙ å "ã Ý Û Û 	Û Û Û ß WÕ W÷	÷ 	ó 	ó Ý 'ß (Ý %Ý 'Ý 4Ý :÷÷ ÷ ÷ ÷ õ ÷.ó ö ÛÝ1à—‘�Ià€Ià	×	Ò	˜8Ó	$€àØˆdˆV�Y˜t‘_Ð$Ñ% x°°t°¸iÈ¹oÐ0MÑ'NÐNñ€	ð 	ØGKØBFØ#Ø'+Ø!%Ø%&ØØ)-ØØIMØØ/4ð'H
ð
ðH
ð EðH
ð @ðH
ð ðH
ð %ðH
ð ðH
ð #ðH
ð ðH
ð 'ðH
ð  ð!H
ð" Gð#H
ð$ ð%H
ð& -ð'H
ððH
ð( ð)H
ð* õ+H
ð\ HLØBFØ#Ø%&Ø)-ØØð~ð Eð~ð @ð	~ð
 ð~ð #ð~ð 'ð~ð ð~ð ð~Ø<ð~ð õ~ðJ HLØBFØ#Ø'+Ø!%Ø%)ØØ)-ØØIMØØ/4ðMð 8ðMð Eð	Mð
 @ðMð ðMð %ðMð ðMð #ðMð ðMð 'ðMð ðMð GðMð ðMð -ðMØRðMð  õ!Mô`F
9Ð5ô F
9÷R\ñ \ðJ !&Ø/4ð.
Ø4ð.
àð.
ð ð.
ð ð	.
ð
 ð.
ð ð.
ð -ð.
ð õ.
ôb>ðØðà1ôðJ !&ñ	Ø
7ðð ðð ð	ð
 $õñ ˆCƒL€ðØ(ðàôðØ#ðà#õr«   