Ë
    ³Œj"\ ã            	      ó.  — d Z ddlmZ ddlZddlmZ ddlZddlZddl	Z	ddl
Z
ddlZddlZddlZddlZddlZddlZddlmZmZmZmZmZmZmZ ddlmZ ddlmZmZmZmZm Z m!Z!m"Z"m#Z# ddl$m%Z%m&Z& ddl'Z'ddl'm(Z) dd	l'm*Z+ dd
l'm,Z- ddl'm.Z. ddl'm/Z0 ddl1m2Z2 ddl3m4Z4m5Z5 ddl6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z? erddl@ZAddlBmCZC eAjˆ                  ZDneZD ejŠ                  eF«      ZGe"eeHgeHf   eeHeHgeHf   f   ZIe"eJej–                  ee.j˜                     e.jš                  f   ZNe"e<ee.jž                  e e.j˜                     ge"e:e;f   f   ede"eHe;e:f   f   f   ZPe"ee.jž                  e e.j˜                     gee"e:e;f      f   f   ZQe"eJej–                  e.j¤                  f   ZSe&	 	 	 	 	 	 	 	 	 	 	 	 dG	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dHd„«       ZTe&	 	 	 	 	 	 	 	 	 	 	 	 dG	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dId„«       ZT	 	 	 	 	 	 	 	 	 	 	 	 	 dJ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dKd„ZT	 	 	 	 	 	 	 dL	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dMd„ZU G d„ de%«      ZV G d„ d«      ZWeee.jž                     e e.j˜                     ge"e"e8eHf   ee"e8eHf      f   f   ZX	 	 	 	 	 	 	 dN	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dOd„ZY G d„ d«      ZZ	 	 	 	 	 	 dPd„Z[	 	 	 	 dQd „Z\dRd!„Z]	 	 	 	 	 	 	 	 	 	 	 	 dS	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dTd"„Z^dUd#„Z_	 	 	 	 	 	 dVd$„Z`	 dW	 	 	 	 	 	 	 dXd%„Za	 	 	 	 	 	 dYd&„Zb e!d'«      ZcdZd(„Zd e!d)d*¬+«      Ze G d,„ d*«      Zf G d-„ d.ef«      Zg	 	 	 	 d[d/„Zh	 	 	 	 d\d0„Zi G d1„ d2e%«      Zj	 	 d]	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d^d3„ZkdUd4„Zl	 	 	 	 d_d5„Zm	 	 	 	 	 	 d`d6„Zndd7œ	 	 	 	 	 	 	 dad8„Zodbd9„Zp	 	 	 	 dcd:„Zqddd;„Zrded<„Zsdfd=„Ztdgd>„Zudhd?„Zv	 	 	 	 	 	 	 	 did@„ZwdjdA„Zx	 	 dk	 	 	 	 	 dldB„Zy	 	 dk	 	 	 	 	 dldC„Zz ejö                  d¬D«      dmdE„«       Z|dndF„Z}y)ozV2 Evaluation Interface.é    )ÚannotationsN)ÚAsyncIterableÚ	AwaitableÚ	GeneratorÚIterableÚIteratorÚSequenceÚSized)Úcopy_context)ÚTYPE_CHECKINGÚAnyÚCallableÚLiteralÚOptionalÚTypeVarÚUnionÚcast)Ú	TypedDictÚoverload)Úenv)Úrun_helpers)Ú	run_trees)Úschemas)Úutils)Ú_v2_migration_utils)Ú
_warn_onceÚsuppress_deprecation_warning)	ÚSUMMARY_EVALUATOR_TÚComparisonEvaluationResultÚDynamicComparisonRunEvaluatorÚEvaluationResultÚEvaluationResultsÚRunEvaluatorÚ_normalize_summary_evaluatorÚcomparison_evaluatorÚrun_evaluator©ÚRunnable.é   ÚExperimentResultsc                ó   — y ©N© ©ÚtargetÚdataÚ
evaluatorsÚsummary_evaluatorsÚmetadataÚexperiment_prefixÚdescriptionÚmax_concurrencyÚnum_repetitionsÚclientÚblockingÚ
experimentÚupload_resultsÚkwargss                 úf/var/www/html/Fitness-lenito-AI-main/venv/lib/python3.12/site-packages/langsmith/evaluation/_runner.pyÚevaluater>   _   s   € ð" ó    ÚComparativeExperimentResultsc                ó   — y r,   r-   r.   s                 r=   r>   r>   s   s   € ð" $'r?   c               ó.  — t        | t        t        j                  t        j
                  f«      rå|dkD  t        |«      | t        |«      t        |«      dœ}t        |j                  «       «      r/dt        d„ |j                  «       D «       «      › d�}t        |«      ‚t        | t        t        j                  f«      r| n| j                  }t        j                  d|› d�«       t        | ft!        t"        t$        t&              |«      ||||	|
dœ|¤ŽS t        | t(        t        f«      �r%|dkD  t        |«      | t        |«      t        |«      d	œ}t+        | «      d
k7  st-        d„ | D «       «      sd| ›�}t        |«      ‚t        |j                  «       «      r/dt        d„ |j                  «       D «       «      › d�}t        |«      ‚|�||d<   | D �cg c]0  }t        |t        t        j                  f«      r|n|j                  ‘Œ2 }}t        j                  d|› d�«       t/        | ft!        t$        t0           |xs d«      |||	|dœ|¤ŽS |rd|› d�}t        |«      ‚|sd}t        |«      ‚t3        | «      r"t5        j6                  | «      rd}t        |«      ‚|r|rd|› d|› �}t        |«      ‚|st9        d«       t        j                  d| › d�«       t;        | |t!        t"        t$        t&              |«      |||||||	|
|||¬«      S c c}w )a  Evaluate a target system on a given dataset.

    Args:
        target (TARGET_T | Runnable | EXPERIMENT_T | Tuple[EXPERIMENT_T, EXPERIMENT_T]):
            The target system or experiment(s) to evaluate.

            Can be a function that takes a dict and returns a `dict`, a langchain `Runnable`, an
            existing experiment ID, or a two-tuple of experiment IDs.
        data (DATA_T): The dataset to evaluate on.

            Can be a dataset name, a list of examples, or a generator of examples.
        evaluators (Sequence[EVALUATOR_T] | Sequence[COMPARATIVE_EVALUATOR_T] | None):
            A list of evaluators to run on each example. The evaluator signature
            depends on the target type.
        summary_evaluators (Sequence[SUMMARY_EVALUATOR_T] | None): A list of summary
            evaluators to run on the entire dataset.

            Should not be specified if comparing two existing experiments.
        metadata (dict | None): Metadata to attach to the experiment.
        experiment_prefix (str | None): A prefix to provide for your experiment name.
        description (str | None): A free-form text description for the experiment.
        max_concurrency (int | None): The maximum number of concurrent
            evaluations to run.

            If `None` then no limit is set. If `0` then no concurrency.
        client (langsmith.Client | None): The LangSmith client to use.
        blocking (bool): Whether to block until the evaluation is complete.
        num_repetitions (int): The number of times to run the evaluation.
            Each item in the dataset will be run and evaluated this many times.
        experiment (schemas.TracerSession | None): An existing experiment to
            extend.

            If provided, `experiment_prefix` is ignored.

            For advanced usage only. Should not be specified if target is an existing
            experiment or two-tuple fo experiments.
        error_handling (str, default="log"): How to handle individual run errors.

            `'log'` will trace the runs with the error message as part of the
            experiment, `'ignore'` will not count the run as part of the experiment at
            all.

    Returns:
        ExperimentResults: If target is a function, `Runnable`, or existing experiment.
        ComparativeExperimentResults: If target is a two-tuple of existing experiments.

    Examples:
        Prepare the dataset:

        >>> from typing import Sequence
        >>> from langsmith import Client
        >>> from langsmith.evaluation import evaluate
        >>> from langsmith.schemas import Example, Run
        >>> client = Client()
        >>> dataset = client.clone_public_dataset(
        ...     "https://smith.langchain.com/public/419dcab2-1d66-4b94-8901-0357ead390df/d"
        ... )
        >>> dataset_name = "Evaluate Examples"

        Basic usage:

        >>> def accuracy(run: Run, example: Example):
        ...     # Row-level evaluator for accuracy.
        ...     pred = run.outputs["output"]
        ...     expected = example.outputs["answer"]
        ...     return {"score": expected.lower() == pred.lower()}
        >>> def precision(runs: Sequence[Run], examples: Sequence[Example]):
        ...     # Experiment-level evaluator for precision.
        ...     # TP / (TP + FP)
        ...     predictions = [run.outputs["output"].lower() for run in runs]
        ...     expected = [example.outputs["answer"].lower() for example in examples]
        ...     # yes and no are the only possible answers
        ...     tp = sum([p == e for p, e in zip(predictions, expected) if p == "yes"])
        ...     fp = sum([p == "yes" and e == "no" for p, e in zip(predictions, expected)])
        ...     return {"score": tp / (tp + fp)}
        >>> def predict(inputs: dict) -> dict:
        ...     # This can be any function or just an API call to your app.
        ...     return {"output": "Yes"}
        >>> results = evaluate(
        ...     predict,
        ...     data=dataset_name,
        ...     evaluators=[accuracy],
        ...     summary_evaluators=[precision],
        ...     experiment_prefix="My Experiment",
        ...     description="Evaluating the accuracy of a simple prediction model.",
        ...     metadata={
        ...         "my-prompt-version": "abcd-1234",
        ...     },
        ... )  # doctest: +ELLIPSIS
        View the evaluation results for experiment:...

        Evaluating over only a subset of the examples

        >>> experiment_name = results.experiment_name
        >>> examples = client.list_examples(dataset_name=dataset_name, limit=5)
        >>> results = evaluate(
        ...     predict,
        ...     data=examples,
        ...     evaluators=[accuracy],
        ...     summary_evaluators=[precision],
        ...     experiment_prefix="My Experiment",
        ...     description="Just testing a subset synchronously.",
        ... )  # doctest: +ELLIPSIS
        View the evaluation results for experiment:...

        Streaming each prediction to more easily + eagerly debug.

        >>> results = evaluate(
        ...     predict,
        ...     data=dataset_name,
        ...     evaluators=[accuracy],
        ...     summary_evaluators=[precision],
        ...     description="I don't even have to block!",
        ...     blocking=False,
        ... )  # doctest: +ELLIPSIS
        View the evaluation results for experiment:...
        >>> for i, result in enumerate(results):  # doctest: +ELLIPSIS
        ...     pass



        Evaluating a LangChain object:

        >>> from langchain_core.runnables import chain as as_runnable
        >>> @as_runnable
        ... def nested_predict(inputs):
        ...     return {"output": "Yes"}
        >>> @as_runnable
        ... def lc_predict(inputs):
        ...     return nested_predict.invoke(inputs)
        >>> results = evaluate(
        ...     lc_predict.invoke,
        ...     data=dataset_name,
        ...     evaluators=[accuracy],
        ...     description="This time we're evaluating a LangChain object.",
        ...     summary_evaluators=[precision],
        ... )  # doctest: +ELLIPSIS
        View the evaluation results for experiment:...

    !!! warning "Behavior changed in `langsmith` 0.2.0"

        'max_concurrency' default updated from None (no limit on concurrency)
        to 0 (no concurrency at all).
    r)   )r7   r:   r;   r4   r0   zReceived invalid arguments. c              3  ó,   K  — | ]  \  }}|sŒ	|–— Œ y ­wr,   r-   ©Ú.0ÚkÚvs      r=   Ú	<genexpr>zevaluate.<locals>.<genexpr>6  ó   è ø€ ÐAÑ';™t˜q !ºqœÑ';ùó   ‚
�z? should not be specified when target is an existing experiment.z,Running evaluation over existing experiment z...)r1   r2   r3   r6   r8   r9   )r7   r:   r;   r2   r0   é   c              3  óz   K  — | ]3  }t        |t        t        j                  t        j
                  f«      –— Œ5 y ­wr,   )Ú
isinstanceÚstrÚuuidÚUUIDr   ÚTracerSession)rE   Úts     r=   rH   zevaluate.<locals>.<genexpr>N  s,   è ø€ ð '
ÙLRÀqŒJ�qœ3¤§	¡	¬7×+@Ñ+@ÐA×BÉFùs   ‚9;z­Received invalid target. If a tuple is specified it must have length 2 and each element should by the ID or schemas.TracerSession of an existing experiment. Received target=c              3  ó,   K  — | ]  \  }}|sŒ	|–— Œ y ­wr,   r-   rD   s      r=   rH   zevaluate.<locals>.<genexpr>Z  rI   rJ   zA should not be specified when target is two existing experiments.r6   z6Running pairwise evaluation over existing experiments r-   )r1   r4   r5   r8   r3   zReceived unsupported arguments zC. These arguments are not supported when creating a new experiment.zDMust specify 'data' when running evaluations over a target function.zåAsync functions are not supported by `evaluate`. Please use `aevaluate` instead:

from langsmith import aevaluate

await aevaluate(
    async_target_function,
    data=data,
    evaluators=evaluators,
    # ... other parameters
)zeExpected at most one of 'experiment' or 'experiment_prefix', but both were provided. Got: experiment=z, experiment_prefix=z&'upload_results' parameter is in beta.z&Running evaluation over target system )r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   Úerror_handling)rM   rN   rO   rP   r   rQ   ÚboolÚanyÚvaluesÚtupleÚitemsÚ
ValueErrorÚidÚloggerÚdebugÚevaluate_existingr   r   r	   ÚEVALUATOR_TÚlistÚlenÚallÚevaluate_comparativeÚCOMPARATIVE_EVALUATOR_TÚcallableÚrhÚis_asyncr   Ú	_evaluate)r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   rT   r<   Úinvalid_argsÚmsgÚ	target_idrR   Ú
target_idss                       r=   r>   r>   ‡   sy  € ôH �&œ3¤§	¡	¬7×+@Ñ+@ÐAÔBà.°Ñ2Ü˜zÓ*Ø"0Ð0Ü!%Ð&7Ó!8Ü˜“Jñ
ˆô ˆ|×"Ñ"Ó$Ô%à.ÜÑA |×'9Ñ'9Ô';ÓAÓAÐBð CCðDð ô
 ˜S“/Ð!Ü(¨´#´t·y±yÐ1AÔB‘FÈÏ	É	ˆ	Ü�‰ÐCÀIÀ;ÈcÐRÔSÜ Øð	
äœH¤X¬kÑ%:Ñ;¸ZÓHØ1ØØ+ØØñ	
ð ñ	
ð 		
ô 
�FœT¤5˜MÕ	*à.°Ñ2Ü˜zÓ*Ø"0Ð0Ü"&Ð'9Ó":Ü˜“Jñ
ˆô ˆv‹;˜!Ò¤3ñ '
ÙLRó'
ô $
ð9à17°	ð;ð ô
 ˜S“/Ð!Ü�×$Ñ$Ó&Ô'à.ÜÑA |×'9Ñ'9Ô';ÓAÓAÐBð CEðFð ô
 ˜S“/Ð!ØÐ&Ø(7ˆFÐ$Ñ%ÙNTÓUÉfÈœ: a¬#¬t¯y©yÐ)9Ô:‘aÀÇÁÑDÈfˆ
ÐUÜ�‰ØDÀZÀLÐPSÐTô	
ô $Øð
äœHÔ%<Ñ=¸zÒ?OÈRÓPØ/Ø#ØØñ
ð ñ
ð 	
ñ 
à-¨f¨Xð 68ð 9ð 	ô ˜‹oÐÙØTˆÜ˜‹oÐÜ	�&Ô	œbŸk™k¨&Ô1ðð 	ô ˜‹oÐÙ	Ñ)ðà)˜lÐ*>Ð?PÐ>QðSð 	ô
 ˜‹oÐáÜÐ?Ô@Ü�‰Ð=¸f¸XÀSÐIÔJÜØØÜœH¤X¬kÑ%:Ñ;¸ZÓHØ1ØØ/Ø#Ø+Ø+ØØØ!Ø)Ø)ô
ð 	
ùò] Vs   Ç5LFc               ó  — |xs t        j                  d¬«      }t        | |«      }t        |||¬«      }	t	        ||«      }
|	D �cg c])  }|
t        t        j                  |j                  «         ‘Œ+ }}t        |	||||||||¬«	      S c c}w )aÿ  Evaluate existing experiment runs.

    Args:
        experiment (Union[str, uuid.UUID]): The identifier of the experiment to evaluate.
        evaluators (Optional[Sequence[EVALUATOR_T]]): Optional sequence of evaluators to use for individual run evaluation.
        summary_evaluators (Optional[Sequence[SUMMARY_EVALUATOR_T]]): Optional sequence of evaluators
            to apply over the entire dataset.
        metadata (Optional[dict]): Optional metadata to include in the evaluation results.
        max_concurrency (int | None): The maximum number of concurrent
            evaluations to run.

            If `None` then no limit is set. If `0` then no concurrency.
        client (Optional[langsmith.Client]): Optional Langsmith client to use for evaluation.
        load_nested: Whether to load all child runs for the experiment.

            Default is to only load the top-level root runs.
        blocking (bool): Whether to block until evaluation is complete.

    Returns:
        The evaluation results.

    Environment:
        - `LANGSMITH_TEST_CACHE`: If set, API calls will be cached to disk to save time and
            cost during testing.

            Recommended to commit the cache files to your repository for faster CI/CD runs.

            Requires the `'langsmith[vcr]'` package to be installed.

    Examples:
        Define your evaluators

        >>> from typing import Sequence
        >>> from langsmith.schemas import Example, Run
        >>> def accuracy(run: Run, example: Example):
        ...     # Row-level evaluator for accuracy.
        ...     pred = run.outputs["output"]
        ...     expected = example.outputs["answer"]
        ...     return {"score": expected.lower() == pred.lower()}
        >>> def precision(runs: Sequence[Run], examples: Sequence[Example]):
        ...     # Experiment-level evaluator for precision.
        ...     # TP / (TP + FP)
        ...     predictions = [run.outputs["output"].lower() for run in runs]
        ...     expected = [example.outputs["answer"].lower() for example in examples]
        ...     # yes and no are the only possible answers
        ...     tp = sum([p == e for p, e in zip(predictions, expected) if p == "yes"])
        ...     fp = sum([p == "yes" and e == "no" for p, e in zip(predictions, expected)])
        ...     return {"score": tp / (tp + fp)}

        Load the experiment and run the evaluation.

        >>> import uuid
        >>> from langsmith import Client
        >>> from langsmith.evaluation import evaluate, evaluate_existing
        >>> client = Client()
        >>> dataset_name = "__doctest_evaluate_existing_" + uuid.uuid4().hex[:8]
        >>> dataset = client.create_dataset(dataset_name)
        >>> example = client.create_example(
        ...     inputs={"question": "What is 2+2?"},
        ...     outputs={"answer": "4"},
        ...     dataset_id=dataset.id,
        ... )
        >>> def predict(inputs: dict) -> dict:
        ...     return {"output": "4"}
        >>> # First run inference on the dataset
        ... results = evaluate(
        ...     predict, data=dataset_name, experiment_prefix="doctest_experiment"
        ... )  # doctest: +ELLIPSIS
        View the evaluation results for experiment:...
        >>> experiment_id = results.experiment_name
        >>> # Wait for the experiment to be fully processed and check if we have results
        >>> len(results) > 0
        True
        >>> import time
        >>> time.sleep(5)  # Wait longer for runs to be indexed
        >>> results = evaluate_existing(
        ...     experiment_id,
        ...     evaluators=[accuracy],
        ...     summary_evaluators=[precision],
        ... )  # doctest: +ELLIPSIS
        View the evaluation results for experiment:...
        >>> client.delete_dataset(dataset_id=dataset.id)
    )i N  i‘_ )Ú
timeout_ms©Úload_nested)r0   r1   r2   r3   r6   r8   r9   r:   )
ÚrtÚget_cached_clientÚ_load_experimentÚ_load_traces_for_experimentÚ_load_examples_mapr   rO   rP   Úreference_example_idrh   )r:   r1   r2   r3   r6   r8   rp   r9   ÚprojectÚrunsÚdata_mapÚrunr0   s                r=   r^   r^      s™   € ð| ÒH”r×+Ñ+Ð7GÔH€FÜ˜z¨6Ó2€GÜ& w°ÀKÔP€DÜ! &¨'Ó2€HÙKOÓPÉ4ÀCˆH”Tœ$Ÿ)™) S×%=Ñ%=Ó>Ó?È4€DÐPÜØØØØ-ØØ'ØØØô
ð 
ùò Qs   Á.B	c                  ó,   — e Zd ZU ded<   ded<   ded<   y)ÚExperimentResultRowúschemas.Runrz   úschemas.ExampleÚexampler"   Úevaluation_resultsN©Ú__name__Ú
__module__Ú__qualname__Ú__annotations__r-   r?   r=   r|   r|     s   … Ø	ÓØÓØ)Ô)r?   r|   c                  ó°   — e Zd ZdZddd„Zedd„«       Zedd„«       Zedd„«       Zdd„Z	edd„«       Z
dd„Zdd	„Zdd
„Z	 d	 	 	 	 	 dd„Zdd„Zdd„Zdd„Zy)r*   aÁ  Represents the results of an evaluate() call.

    This class provides an iterator interface to iterate over the experiment results
    as they become available. It also provides methods to access the experiment name,
    the number of results, and to wait for the results to be processed.

    Methods:
        experiment_name() -> str: Returns the name of the experiment.
        wait() -> None: Waits for the experiment data to be processed.
    c                óD  — || _         g | _        t        j                  «       | _        t        j                  «       | _        d | _        |s@t        j                  | j                  ¬«      | _        | j                  j                  «        y d | _        | j                  «        y )N©r/   )Ú_managerÚ_resultsÚqueueÚQueueÚ_queueÚ	threadingÚEventÚ_processing_completeÚ_processing_errorÚThreadÚ_process_dataÚ_threadÚstart)ÚselfÚexperiment_managerr9   s      r=   Ú__init__zExperimentResults.__init__"  sw   € Ø*ˆŒØ35ˆŒÜ8=¿¹»ˆŒÜ$-§O¡OÓ$5ˆÔ!Ø:>ˆÔÙÜ7@×7GÑ7GØ×)Ñ)ô8ˆDŒLð �L‰L×ÑÕ àˆDŒLØ×ÑÕ r?   c                ó.   — | j                   j                  S r,   )r‰   Úexperiment_name©r–   s    r=   rš   z!ExperimentResults.experiment_name1  s   € à�}‰}×,Ñ,Ð,r?   c                óJ   — | j                   j                  «       j                  S )zThe ID of the experiment.)r‰   Ú_get_experimentr[   r›   s    r=   Úexperiment_idzExperimentResults.experiment_id5  s   € ð �}‰}×,Ñ,Ó.×1Ñ1Ð1r?   c                ó  — | j                   j                  «       }|j                  rZ|j                  j                  d«      d   }|j                  d«      d   }|› d| j                   j                  › d|j
                  › �S y)z.The URL of the experiment in the LangSmith UI.Ú?r   ú/projects/p/ú
/datasets/ú/compare?selectedSessions=N)r‰   r�   ÚurlÚsplitÚ
dataset_idr[   )r–   r:   Úproject_urlÚbase_urls       r=   r¤   zExperimentResults.url:  s   € ð —]‘]×2Ñ2Ó4ˆ
Ø�>Š>Ø$Ÿ.™.×.Ñ.¨sÓ3°AÑ6ˆKØ"×(Ñ(¨Ó8¸Ñ;ˆHà�*˜J t§}¡}×'?Ñ'?Ð&@ð A$Ø$.§M¡M ?ð4ðð r?   c                ó.   — | j                   j                  S )z:Get the ID of the dataset associated with this experiment.)r‰   r¦   r›   s    r=   Úget_dataset_idz ExperimentResults.get_dataset_idG  s   € à�}‰}×'Ñ'Ð'r?   c                ó   — | j                   S )z3The URL to the comparison view for this experiment.)r¤   r›   s    r=   Úcomparison_urlz ExperimentResults.comparison_urlK  s   € ð �x‰xˆr?   c              #  ót  K  — d}| j                   j                  «       r2| j                  j                  «       r|t	        | j
                  «      k  rœ	 |t	        | j
                  «      k  r| j
                  |   –— |dz  }n| j                  j                  dd¬«       | j                   j                  «       sŒh| j                  j                  «       sŒƒ|t	        | j
                  «      k  rŒœ| j                  �| j                  ‚y # t        j                  $ r | j                  �| j                  ‚Y �Œ/w xY w­w)Nr   r)   Tgš™™™™™¹?)ÚblockÚtimeout)
r�   Úis_setr�   Úemptyra   rŠ   Úgetr‹   ÚEmptyr‘   )r–   Úixs     r=   Ú__iter__zExperimentResults.__iter__P  s  è ø€ Øˆà×)Ñ)×0Ñ0Ô2Ø—;‘;×$Ñ$Ô&Ø”C˜Ÿ™Ó&Ò&ð	Øœ˜DŸM™MÓ*Ò*ØŸ-™-¨Ñ+Ò+Ø˜!‘G‘Bà—K‘K—O‘O¨$¸�OÔ<ð ×)Ñ)×0Ñ0Õ2Ø—;‘;×$Ñ$Õ&Ø”C˜Ÿ™Ó&Ó&ð ×!Ñ!Ð-Ø×(Ñ(Ð(ð .øô	 —;‘;ò Ø×)Ñ)Ð5Ø×0Ñ0Ð0Úðüs=   ‚AD8ÁAD ÂD8Â9D8ÃD8Ã-D8Ä+D5Ä1D8Ä4D5Ä5D8c                óÂ  — t        «       }	 | j                  j                  «       } ||«      D ]8  }| j                  j	                  |«       | j
                  j                  |«       Œ: | j                  j                  «       }|| _        | j                  j                  «        y # t        $ r}|| _
        Y d }~Œ0d }~ww xY w# | j                  j                  «        w xY wr,   )Ú
_load_tqdmr‰   Úget_resultsr�   ÚputrŠ   ÚappendÚget_summary_scoresÚ_summary_resultsÚBaseExceptionr‘   r�   Úset)r–   ÚtqdmÚresultsÚitemÚsummary_scoresÚes         r=   r“   zExperimentResults._process_datad  s²   € Ü‹|ˆð	,Ø—m‘m×/Ñ/Ó1ˆGÙ˜Wž�Ø—‘—‘ Ô%Ø—‘×$Ñ$ TÕ*ð &ð "Ÿ]™]×=Ñ=Ó?ˆNØ$2ˆDÔ!ð ×%Ñ%×)Ñ)Õ+øô ò 	'Ø%&ˆD×"Ñ"ûð	'ûð ×%Ñ%×)Ñ)Õ+ús*   ŒA>B% Â%	B?Â.B:Â5C Â:B?Â?C ÃCc                ó,   — t        | j                  «      S r,   )ra   rŠ   r›   s    r=   Ú__len__zExperimentResults.__len__s  s   € Ü�4—=‘=Ó!Ð!r?   Nc                ó2   — t        | j                  ||¬«      S )N©r•   Úend)Ú
_to_pandasrŠ   )r–   r•   rÈ   s      r=   Ú	to_pandaszExperimentResults.to_pandasv  s   € ô ˜$Ÿ-™-¨u¸#Ô>Ð>r?   c                ó¸   — dd l }| j                  r;|j                  j                  d«      r | j	                  «       }|j                  «       S | j                  «       S )Nr   Úpandas)Úimportlib.utilrŠ   ÚutilÚ	find_specrÊ   Ú_repr_html_Ú__repr__)r–   Ú	importlibÚdfs      r=   rÐ   zExperimentResults._repr_html_{  sB   € Ûà�=Š=˜YŸ^™^×5Ñ5°hÔ?Ø—‘Ó!ˆBØ—>‘>Ó#Ð#à—=‘=“?Ð"r?   c                ó"   — d| j                   › d�S )Nz<ExperimentResults Ú>)rš   r›   s    r=   rÑ   zExperimentResults.__repr__„  s   € Ø$ T×%9Ñ%9Ð$:¸!Ð<Ð<r?   c                ó€   — | j                   r| j                   j                  «        | j                  �| j                  ‚y)z£Wait for the evaluation runner to complete.

        This method blocks the current thread until the evaluation runner has
        finished its execution.
        N)r”   Újoinr‘   r›   s    r=   ÚwaitzExperimentResults.wait‡  s8   € ð �<Š<Ø�L‰L×ÑÔØ×!Ñ!Ð-Ø×(Ñ(Ð(ð .r?   )T)r—   Ú_ExperimentManagerr9   rU   ©ÚreturnrN   )rÛ   ú	uuid.UUID©rÛ   úOptional[str])rÛ   zIterator[ExperimentResultRow]©rÛ   ÚNone)rÛ   Úint©r   N)r•   úOptional[int]rÈ   rã   rÛ   Ú	DataFrame)r‚   rƒ   r„   Ú__doc__r˜   Úpropertyrš   rž   r¤   rª   r¬   rµ   r“   rÅ   rÊ   rÐ   rÑ   rØ   r-   r?   r=   r*   r*     s¢   „ ñ	ô!ð ò-ó ð-ð ò2ó ð2ð ò
ó ð
ó(ð òó ðó)ó(,ó"ð >Bð?Ø"ð?Ø-:ð?à	ó?ó
#ó=ô	)r?   c	          
     ó 	  ‡‡‡.— t        | «      dk  rt        d«      ‚|st        d«      ‚|dk  rt        d«      ‚‰xs t        j                  «       Š| D �	cg c]  }	t	        |	‰«      ‘Œ }
}	|
D �cg c]  }t        |j                  «      ‘Œ }}t        t        |«      «      dk(  st        d«      ‚|
D �cg c]  }|j                  ‘Œ }}|€f|
D �cg c]  }|j                  €Œ|j                  ‘Œ }}d	j                  |«      d
z   t        t        j                  «       j                  dd «      z   }n0|d
z   t        t        j                  «       j                  dd «      z   }t        j                  «       }‰j                  |||||¬«      Š.t        t         t"        j$                  t"        j$                  f   t!        |
«      «      }t'        |‰.«      }t)        |«       |
D �cg c]  }t+        |‰|¬«      ‘Œ }}d}|D ]%  }|D �ch c]  }|j,                  ’Œ }}|€|}Œ!||z  }Œ' |�t/        |«      ng }|D �cg c]  }|€Œ|‘Œ	 }}d}i }t1        dt        |«      |«      D ][  }||||z    }‰j3                  |
d   j                  |
d   j4                  j7                  d«      |¬«      D ]  } | || j                  <   Œ Œ] t9        j:                  t.        «      }!|D ]N  }|D ]G  }|j,                  |v sŒ|!t        t        j<                  |j,                  «         j?                  |«       ŒI ŒP |xs g D �"cg c]  }"tA        |"«      ‘Œ }#}"i }$	 	 	 	 	 	 	 	 	 	 dˆˆ.ˆfd„}%tC        «       }&tE        jF                  |xs d¬«      5 }'g }( |&|!jI                  «       «      D ]h  \  })}d|i|$|)<   |#D ]W  }*|dkD  r*|'jK                  |%|||)   |*|'«      }+|(j?                  |+«       Œ2 |%|||)   |*|'«      \  },}-|-|$|)   d|-jL                  › �<   ŒY Œj |(rDtO        jP                  |(«       |(D ]*  }+|+jS                  «       \  })}-|-|$|)   d|-jL                  › �<   Œ, ddd«       tU        |$|‰.|¬«      S c c}	w c c}w c c}w c c}w c c}w c c}w c c}w c c}"w # 1 sw Y   Œ@xY w)ar!  Evaluate existing experiment runs against each other.

    This lets you use pairwise preference scoring to generate more
    reliable feedback in your experiments.

    Args:
        experiments (Tuple[Union[str, uuid.UUID], Union[str, uuid.UUID]]):
            The identifiers of the experiments to compare.
        evaluators (Sequence[COMPARATIVE_EVALUATOR_T]):
            A list of evaluators to run on each example.
        experiment_prefix (Optional[str]): A prefix to provide for your experiment name.
        description (Optional[str]): A free-form text description for the experiment.
        max_concurrency (int): The maximum number of concurrent evaluations to run.
        client (Optional[langsmith.Client]): The LangSmith client to use.
        metadata (Optional[dict]): Metadata to attach to the experiment.
        load_nested (bool): Whether to load all child runs for the experiment.

            Default is to only load the top-level root runs.
        randomize_order (bool): Whether to randomize the order of the outputs for each evaluation.

    Returns:
        The results of the comparative evaluation.

    Examples:
        Suppose you want to compare two prompts to see which one is more effective.
        You would first prepare your dataset:

        >>> from typing import Sequence
        >>> from langsmith import Client
        >>> from langsmith.evaluation import evaluate
        >>> from langsmith.schemas import Example, Run
        >>> client = Client()
        >>> dataset = client.clone_public_dataset(
        ...     "https://smith.langchain.com/public/419dcab2-1d66-4b94-8901-0357ead390df/d"
        ... )
        >>> dataset_name = "Evaluate Examples"

        Then you would run your different prompts:
        >>> import functools
        >>> import openai
        >>> from langsmith.evaluation import evaluate
        >>> from langsmith.wrappers import wrap_openai
        >>> oai_client = openai.Client()
        >>> wrapped_client = wrap_openai(oai_client)
        >>> prompt_1 = "You are a helpful assistant."
        >>> prompt_2 = "You are an exceedingly helpful assistant."
        >>> def predict(inputs: dict, prompt: str) -> dict:
        ...     completion = wrapped_client.chat.completions.create(
        ...         model="gpt-4o-mini",
        ...         messages=[
        ...             {"role": "system", "content": prompt},
        ...             {
        ...                 "role": "user",
        ...                 "content": f"Context: {inputs['context']}"
        ...                 f"\n\ninputs['question']",
        ...             },
        ...         ],
        ...     )
        ...     return {"output": completion.choices[0].message.content}
        >>> results_1 = evaluate(
        ...     functools.partial(predict, prompt=prompt_1),
        ...     data=dataset_name,
        ...     description="Evaluating our basic system prompt.",
        ...     blocking=False,  # Run these experiments in parallel
        ... )  # doctest: +ELLIPSIS
        View the evaluation results for experiment:...
        >>> results_2 = evaluate(
        ...     functools.partial(predict, prompt=prompt_2),
        ...     data=dataset_name,
        ...     description="Evaluating our advanced system prompt.",
        ...     blocking=False,
        ... )  # doctest: +ELLIPSIS
        View the evaluation results for experiment:...
        >>> results_1.wait()
        >>> results_2.wait()

            Finally, you would compare the two prompts directly:
        >>> import json
        >>> from langsmith.evaluation import evaluate_comparative
        >>> from langsmith import schemas
        >>> def score_preferences(runs: list, example: schemas.Example):
        ...     assert len(runs) == 2  # Comparing 2 systems
        ...     assert isinstance(example, schemas.Example)
        ...     assert all(run.reference_example_id == example.id for run in runs)
        ...     pred_a = runs[0].outputs["output"] if runs[0].outputs else ""
        ...     pred_b = runs[1].outputs["output"] if runs[1].outputs else ""
        ...     ground_truth = example.outputs["answer"] if example.outputs else ""
        ...     tools = [
        ...         {
        ...             "type": "function",
        ...             "function": {
        ...                 "name": "rank_preferences",
        ...                 "description": "Saves the prefered response ('A' or 'B')",
        ...                 "parameters": {
        ...                     "type": "object",
        ...                     "properties": {
        ...                         "reasoning": {
        ...                             "type": "string",
        ...                             "description": "The reasoning behind the choice.",
        ...                         },
        ...                         "preferred_option": {
        ...                             "type": "string",
        ...                             "enum": ["A", "B"],
        ...                             "description": "The preferred option, either 'A' or 'B'",
        ...                         },
        ...                     },
        ...                     "required": ["preferred_option"],
        ...                 },
        ...             },
        ...         }
        ...     ]
        ...     completion = openai.Client().chat.completions.create(
        ...         model="gpt-4o-mini",
        ...         messages=[
        ...             {"role": "system", "content": "Select the better response."},
        ...             {
        ...                 "role": "user",
        ...                 "content": f"Option A: {pred_a}"
        ...                 f"\n\nOption B: {pred_b}"
        ...                 f"\n\nGround Truth: {ground_truth}",
        ...             },
        ...         ],
        ...         tools=tools,
        ...         tool_choice={
        ...             "type": "function",
        ...             "function": {"name": "rank_preferences"},
        ...         },
        ...     )
        ...     tool_args = completion.choices[0].message.tool_calls[0].function.arguments
        ...     loaded_args = json.loads(tool_args)
        ...     preference = loaded_args["preferred_option"]
        ...     comment = loaded_args["reasoning"]
        ...     if preference == "A":
        ...         return {
        ...             "key": "ranked_preference",
        ...             "scores": {runs[0].id: 1, runs[1].id: 0},
        ...             "comment": comment,
        ...         }
        ...     else:
        ...         return {
        ...             "key": "ranked_preference",
        ...             "scores": {runs[0].id: 0, runs[1].id: 1},
        ...             "comment": comment,
        ...         }
        >>> def score_length_difference(runs: list, example: schemas.Example):
        ...     # Just return whichever response is longer.
        ...     # Just an example, not actually useful in real life.
        ...     assert len(runs) == 2  # Comparing 2 systems
        ...     assert isinstance(example, schemas.Example)
        ...     assert all(run.reference_example_id == example.id for run in runs)
        ...     pred_a = runs[0].outputs["output"] if runs[0].outputs else ""
        ...     pred_b = runs[1].outputs["output"] if runs[1].outputs else ""
        ...     if len(pred_a) > len(pred_b):
        ...         return {
        ...             "key": "length_difference",
        ...             "scores": {runs[0].id: 1, runs[1].id: 0},
        ...         }
        ...     else:
        ...         return {
        ...             "key": "length_difference",
        ...             "scores": {runs[0].id: 0, runs[1].id: 1},
        ...         }
        >>> results = evaluate_comparative(
        ...     [results_1.experiment_name, results_2.experiment_name],
        ...     evaluators=[score_preferences, score_length_difference],
        ...     client=client,
        ... )  # doctest: +ELLIPSIS
        View the pairwise evaluation results at:...
        >>> eval_results = list(results)
        >>> assert len(eval_results) >= 10  # doctest: +SKIP
        >>> assert all(
        ...     "feedback.ranked_preference" in r["evaluation_results"]
        ...     for r in eval_results
        ... )  # doctest: +SKIP
        >>> assert all(
        ...     "feedback.length_difference" in r["evaluation_results"]
        ...     for r in eval_results
        ... )  # doctest: +SKIP
    rK   z7Comparative evaluation requires at least 2 experiments.z>At least one evaluator is required for comparative evaluation.r   z+max_concurrency must be a positive integer.r)   z5All experiments must have the same reference dataset.Nz vs. Ú-é   é   )Úexperimentsr5   r3   r[   ro   éc   Údataset_version)r¦   Úas_ofÚexample_idsc                ó€  •— t        j                  «       }‰rt        j                  | «       t	        j
                  d‰¬«      5  |j                  | |«      }‰€t        d«      ‚	 d d d «       t        j                  t        «      r.|j                  D �ci c]  }t        |«      |j                  “Œ c}n|j                  xs i }| D �ci c]  }t        |j                  «      |“Œ }	}|j                  j                  «       D ]–  \  }
}|	j                  t        |
«      «      }|j                  ‰j                   |
|j"                  ||j                  t        |
«      «      ‰j                  |j$                  ||r|j&                  nd |r|j(                  nd ¬«
       Œ˜ |j                  |fS # 1 sw Y   �ŒFxY wc c}w c c}w )Nr1   )Úproject_namer8   z&Client is required to submit feedback.)	Úrun_idÚkeyÚscoreÚcommentÚcomparative_experiment_idÚsource_run_idÚfeedback_group_idÚ
session_idÚ
start_time)rO   Úuuid4ÚrandomÚshufflerf   Útracing_contextÚcompare_runsrZ   rM   rõ   rN   Úscoresr[   rY   r²   ÚsubmitÚcreate_feedbackró   r÷   rù   rú   )Ú	runs_listr   Ú
comparatorÚexecutorrø   ÚresultÚridÚcommentsrz   Ú
runs_by_idrò   rô   r8   Úcomparative_experimentÚrandomize_orders               €€€r=   Úevaluate_and_submit_feedbackz:evaluate_comparative.<locals>.evaluate_and_submit_feedback¨  s{  ø€ ô !ŸJ™J›LÐÙÜ�N‰N˜9Ô%Ü×Ñ¨\À&ÖIØ×,Ñ,¨Y¸Ó@ˆFØˆ~Ü Ð!IÓJÐJð ÷ Jô ˜&Ÿ.™.¬#Ô.ð 28·²Ó?±¨#ŒS�‹X�v—~‘~Ñ%°Ò?à—.‘.Ò& Bð 	ñ 3<Ó<±)¨3”c˜#Ÿ&™&“k 3Ñ&°)ˆ
Ð<Ø#Ÿ]™]×0Ñ0Ö2‰MˆF�EØ—.‘.¤ V£Ó-ˆCØ�O‰OØ×&Ñ&ØØ—J‘JØØ Ÿ™¤S¨£[Ó1Ø*@×*CÑ*CØ$×2Ñ2Ø"3Ù-0˜3Ÿ>š>°dÙ-0˜3Ÿ>š>°dð õ ð 3ð �z‰z˜6Ð!Ð!÷3 JÑIüò
 @ùò
 =s   Á F)ÂF6Ã	F;Æ)F3©Úmax_workersrx   ú	feedback.)r
  r¤   )
r  úlist[schemas.Run]r   r~   r  r    r  zcf.ExecutorrÛ   z,tuple[uuid.UUID, ComparisonEvaluationResult])+ra   rZ   rq   rr   rs   rN   Úreference_dataset_idr¾   r[   Únamer×   rO   rû   ÚhexÚcreate_comparative_experimentr   rX   r   ÚTracerSessionResultÚ_build_comparative_urlÚ#_print_comparative_experiment_startrt   rv   r`   ÚrangeÚlist_examplesr3   r²   ÚcollectionsÚdefaultdictrP   rº   r%   r·   Úls_utilsÚContextThreadPoolExecutorrY   r  ró   ÚcfrØ   r  r@   )/rë   r1   r4   r5   r6   r8   r3   rp   r  r:   ÚprojectsÚpÚref_datasets_Úexperiment_idsÚexperiment_namesrš   rö   Úexperiments_tupler¬   rw   rx   Úexamples_intersectionr  rz   Úexample_ids_setÚexample_ids_nullableÚeidrï   Ú
batch_sizer0   ÚiÚexample_ids_batchrÃ   Ú	runs_dictÚ	evaluatorÚcomparatorsrÀ   r  r¿   r  ÚfuturesÚ
example_idr  ÚfutureÚ_r  r
  s/        `  `                                     @r=   rc   rc   Ÿ  s  ú€ ô~ ˆ;Ó˜!ÒÜÐRÓSÐSÙÜØLó
ð 	
ð ˜ÒÜÐFÓGÐGØÒ-”r×+Ñ+Ó-€Fñ HSÓSÁ{¸Ô  ¨VÕ4À{€HÐSÙ:BÓC¹(°Q”S˜×/Ñ/Õ0¸(€MÐCÜŒs�=Ó!Ó" aÒ'ÜÐPÓQÐQÙ$,Ó-¡H˜q�a—d“d H€NÐ-ØÐ Ù,4ÓK©H q¸¿¹Ñ8J˜AŸF›F¨HÐÐKà�L‰LÐ)Ó*¨SÑ0´3´t·z±z³|×7GÑ7GÈÈÐ7KÓ3LÑLñ 	ð ,¨cÑ1´C¼¿
¹
»×8HÑ8HÈÈ!Ð8LÓ4MÑMˆÜ $§
¡
£ÐØ#×AÑAØØ"ØØØ$ð Bó Ðô ÜŒg×)Ñ)¬7×+FÑ+FÐFÑGÜˆh‹óÐô ,Ð,=Ð?UÓV€NÜ'¨Ô7ñ  óáˆGô 	$ G¨VÀÖMØð 	ð ð
 !ÐÛˆ	Ù?HÓI¹y¸˜3×3Ó3¸yˆÐIØ Ð(Ø$3Ñ!à! _Ñ4Ñ!ð ð (=Ð'HŒÐ"Ô#Èbð ñ #7ÓJÑ"6˜3¸#¹/’3Ð"6€KÐJð €JØ€DÜ�1”c˜+Ó&¨
Ö3ˆØ'¨¨A°
©NÐ;ÐØ×%Ñ%Ø ‘{×7Ñ7Ø˜1‘+×&Ñ&×*Ñ*Ð+<Ó=Ø)ð &ö 
ˆAð
 ˆD�—‘ŠJñ
ð 4ô 5@×4KÑ4KÌDÓ4Q€IÛˆ	ÛˆCØ×'Ñ'¨4Ò/Øœ$œtŸy™y¨#×*BÑ*BÓCÑD×KÑKÈCÕPñ ð ð
 EOÒDTÐRTÐDTÓUÑDT°yÔ'¨	Õ2ÐDT€KÐUØ€Gð""Ø$ð""à ð""ð 2ð""ð ð	""ð
 
6÷""ôH ‹<€DÜ	×	+Ñ	+Ø#Ò( qõ
à	ØˆÙ%)¨)¯/©/Ó*;Ö%<Ñ!ˆJ˜	Ø#)¨9Ð"5ˆG�JÑÛ)�
Ø" QÒ&Ø%Ÿ_™_Ø4Ø!Ø˜ZÑ(Ø"Ø ó�Fð —N‘N 6Õ*á <Ø! 4¨
Ñ#3°ZÀó!‘I�A�vð EK�G˜JÑ'¨)°F·J±J°<Ð(@ÒAñ *ð &=ñ" Ü�G‰G�GÔÛ!�Ø%+§]¡]£_Ñ"�
˜FØ@F�˜
Ñ# i°·
±
¨|Ð$<Ò=ð "÷/
ô6 (ØØØ5Øô	ð ùò TùÚCùò .ùâKùò(ùò Jùò Kùò& V÷P
ð 
úsI   ÁQÁ/Q!Â3Q&ÃQ+Ã Q+ÇQ0ÈQ5È5Q:È=Q:Ì/Q?Í;C
RÒRc                  ó^   — e Zd ZdZ	 	 	 d	 	 	 	 	 	 	 d	d„Zed
d„«       Zedd„«       Zd„ Zd„ Z	y)r@   a  Represents the results of an evaluate_comparative() call.

    This class provides an iterator interface to iterate over the comparison results,
    indexed access by example ID, and properties to access the pairwise comparison
    URL and the underlying comparative experiment.

    Attributes:
        url (Optional[str]): URL of the pairwise comparison view in the LangSmith UI.
        comparative_experiment (Optional[schemas.ComparativeExperiment]): The
        comparative experiment, exposing its id, dataset, and metadata.
    Nc                ó<   — || _         || _        || _        || _        y r,   )rŠ   Ú	_examplesÚ_comparative_experimentÚ_url)r–   rÀ   Úexamplesr
  r¤   s        r=   r˜   z%ComparativeExperimentResults.__init__ý  s!   € ð  ˆŒØ!ˆŒØ'=ˆÔ$Øˆ�	r?   c                ó   — | j                   S )zCThe comparative experiment, exposing its id, dataset, and metadata.)r6  r›   s    r=   r
  z3ComparativeExperimentResults.comparative_experiment	  s   € ð ×+Ñ+Ð+r?   c                ó   — | j                   S )z8URL of the pairwise comparison view in the LangSmith UI.)r7  r›   s    r=   r¤   z ComparativeExperimentResults.url  s   € ð �y‰yÐr?   c                ó    — | j                   |   S )z0Return the result associated with the given key.)rŠ   )r–   ró   s     r=   Ú__getitem__z(ComparativeExperimentResults.__getitem__  s   € à�}‰}˜SÑ!Ð!r?   c              #  ó”   K  — | j                   j                  «       D ]'  \  }}| j                  r| j                  |   nd |dœ–— Œ) y ­w)N)r   r€   )rŠ   rY   r5  )r–   ró   Úvalues      r=   rµ   z%ComparativeExperimentResults.__iter__  sB   è ø€ ØŸ-™-×-Ñ-Ö/‰JˆC�à26·.².˜4Ÿ>™>¨#Ò.ÀdØ&+ñó ñ 0ùó   ‚AA©NNN)rÀ   Údictr8  z*Optional[dict[uuid.UUID, schemas.Example]]r
  ú'Optional[schemas.ComparativeExperiment]r¤   rÞ   )rÛ   rB  rÝ   )
r‚   rƒ   r„   rå   r˜   ræ   r
  r¤   r<  rµ   r-   r?   r=   r@   r@   ð  sr   „ ñ
ð @DØJNØ!ð
àð
ð =ð
ð !Hð	
ð
 ó
ð ò,ó ð,ð òó ðò"ór?   c                óH  — | d   j                   xs | d   j                   }|sy |j                  d«      d   }|j                  }|j                  d«      d   }|› d|› ddj                  | D �cg c]  }t	        |j
                  «      ‘Œ c}«      › d|j
                  › �S c c}w )	Nr   r)   r    r¡   r¢   r£   z%2Cz&comparativeExperiment=)r¤   r¥   r  r×   rN   r[   )rë   r
  r¤   r§   r¦   r¨   rÃ   s          r=   r  r  "  s¬   € ð �a‰.×
Ñ
Ò
2 ¨A¡× 2Ñ 2€CÙØØ—)‘)˜C“. Ñ#€KØ'×<Ñ<€JØ× Ñ  Ó0°Ñ3€Hàˆ*�J˜z˜lð +Ø!ŸJ™J¹;Ó'G¹;°a¬¨A¯D©D­	¸;Ñ'GÓHÐIØ
!Ð"8×";Ñ";Ð!<ð	>ðùâ'Gs   Á-Bc                ó(   — | rt        d| › d�«       y y )Nz)View the pairwise evaluation results at:
ú

)Úprint)r¬   s    r=   r  r  3  s!   € ñ ÜØ8¸Ð8HÈÐMõ	
ð r?   c                ó2   — t        | «      xs t        | «      S r,   )re   Ú_is_langchain_runnablerˆ   s    r=   Ú_is_callablerI  <  s   € Ü�FÓÒ=Ô5°fÓ=Ð=r?   c               óØ  — |	xs t        j                  «       }	t        | «      rd n t        t        t
        j                     | «      }t        |||	«      \  }}t        ||	||xs |||t        |«      |t        | |«      ||¬«      j                  «       }t        j                  d «      x}r&t        j                  |«      |j                   › d�z  }nd }t        j"                  ||	j$                  g¬«      5  t        | «      r!|j'                  t        t(        | «      |¬«      }|r|j+                  ||¬«      }|r|j-                  |«      }t/        ||
¬«      }|cd d d «       S # 1 sw Y   y xY w)N)
r8   r3   r:   r5   r7   Úevaluator_keysrx   Úinclude_attachmentsr;   rT   z.yaml)Úignore_hosts©r6   )r9   )rq   rr   rI  r   r   r   ÚRunÚ_resolve_experimentrÙ   Ú_collect_evaluator_keysÚ_include_attachmentsr•   r  Úget_cache_dirÚpathlibÚPathr¦   Úwith_optional_cacheÚapi_urlÚwith_predictionsÚTARGET_TÚwith_evaluatorsÚwith_summary_evaluatorsr*   )r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   rT   rx   Úexperiment_ÚmanagerÚ	cache_dirÚ
cache_pathrÀ   s                       r=   rh   rh   @  s`  € ð$ Ò-”r×+Ñ+Ó-€FÜ Ô'‰4¬T´(¼7¿;¹;Ñ2GÈÓ-P€DÜ+¨J¸¸fÓEÑ€K�ä ØØØØÒ3Ð"3ØØ'Ü.¨zÓ:àä0°¸ÓDØ%Ø%ô÷ �eƒgð ô ×*Ñ*¨4Ó0Ð0€yÐ0Ü—\‘\ )Ó,°'×2DÑ2DÐ1EÀUÐ/KÑK‰
àˆ
Ü	×	%Ñ	% jÀÇÁÐ?OÖ	PÜ˜Ôà×.Ñ.Ü”X˜vÓ&¸ð /ó ˆGñ à×-Ñ-Ø¨Oð .ó ˆGñ à×5Ñ5Ð6HÓIˆGä# G°hÔ?ˆØ÷! 
Q×	PÒ	Pús   Ã3A#E Å E)c                óN   — 	 t        j                  | «       y# t        $ r Y yw xY w©NTF©rO   rP   rZ   ©r>  s    r=   Ú_is_uuidrd  |  ó(   € ðÜ�	‰	�%ÔØøÜò Ùðúó   ‚ ˜	$£$c                óÌ   — t        | t        j                  «      r| S t        | t        j                  «      st        | «      r|j                  | ¬«      S |j                  | ¬«      S )N©Ú
project_id)rñ   )rM   r   rQ   rO   rP   rd  Úread_project)rw   r8   s     r=   rs   rs   „  sV   € ô �'œ7×0Ñ0Ô1ØˆÜ	�GœTŸY™YÔ	'¬8°GÔ+<Ø×"Ñ"¨gÐ"Ó6Ð6à×"Ñ"°Ð"Ó8Ð8r?   c                ó2  — |rdnd}t        j                  |j                  j                  «      }|t         j                  j
                  k(  r?t        | t        j                  «      st        | |«      } t        j                  | ||¬«      }n½t        | t        j                  «      r1t        «       5  |j                  | j                  |¬«      }ddd«       nrt        | t        j                  «      st!        | «      r't        «       5  |j                  | |¬«      }ddd«       n&t        «       5  |j                  | |¬«      }ddd«       |st#        «      S t%        j&                  t"        «      }g }i }D ]M  }	|	j(                  �||	j(                     j+                  |	«       n|j+                  |	«       |	||	j                  <   ŒO |j-                  «       D ]  \  }
}t/        |d„ ¬«      ||
   _        Œ |S # 1 sw Y   ŒµxY w# 1 sw Y   ŒÁxY w# 1 sw Y   ŒÍxY w)z'Load nested traces for a given project.NT)Úis_root)ri  rl  )rñ   rl  c                ó   — | j                   S r,   )Údotted_order)Úrs    r=   Ú<lambda>z-_load_traces_for_experiment.<locals>.<lambda>¸  s   € ÀqÇ~Â~r?   )ró   )r   Úget_query_backendÚinfoÚinstance_flagsÚQueryBackendÚSMITHDB_ONLYrM   r   rQ   rs   Ú_load_traces_v2r   Ú	list_runsr[   rO   rP   rd  r`   r  r  Úparent_run_idrº   rY   ÚsortedÚ
child_runs)rw   r8   rp   rl  Úbackendrx   ÚtreemaprÀ   Úall_runsrz   rò   rz  s               r=   rt   rt   �  sÄ  € ñ "‰d t€Gô "×3Ñ3°F·K±K×4NÑ4NÓO€GØÔ%×2Ñ2×?Ñ?Ò?Ü˜'¤7×#8Ñ#8Ô9Ü& w°Ó7ˆGÜ"×2Ñ2°7¸FÈGÔT‰ô 
�GœW×2Ñ2Ô	3Ü)Õ+Ø×#Ñ#¨w¯z©zÀ7Ð#ÓKˆD÷ ,Ð+ä	�GœTŸY™YÔ	'¬8°GÔ+<Ü)Õ+Ø×#Ñ#¨wÀÐ#ÓHˆD÷ ,Ð+ô *Õ+Ø×#Ñ#°À'Ð#ÓJˆD÷ ,áÜ�D‹zÐô 	×Ñ¤Ó%ð ð €GØ€HÛˆØ×ÑÐ(Ø�C×%Ñ%Ñ&×-Ñ-¨cÕ2à�N‰N˜3ÔØˆ�—‘Òð ð &Ÿm™mžoÑˆ�
Ü&,¨ZÑ=UÔ&Vˆ�ÑÕ#ð .à€N÷1 ,Ð+ú÷ ,Ð+ú÷ ,Ð+ús$   Â0G5ÄHÄ-HÇ5G>ÈH
ÈHc                óª   — | j                  |j                  |j                  j                  d«      ¬«      D �ci c]  }|j                  |“Œ c}S c c}w )Nrí   )r¦   rî   )r  r  r3   r²   r[   )r8   rw   rÃ   s      r=   ru   ru   ¼  sc   € ð
 ×%Ñ%Ø×3Ñ3Ø×"Ñ"×&Ñ&Ð'8Ó9ð &ô 
óñ
ˆAð 	
�‰ˆa‰ð
ñð ùò s   ¹AÚITc                 ó8   — 	 ddl m}  | S # t        $ r d„ cY S w xY w)Nr   ©r¿   c                ó   — | S r,   r-   )Úxs    r=   rp  z_load_tqdm.<locals>.<lambda>Ï  s   € ™r?   )Ú	tqdm.autor¿   ÚImportErrorr�  s    r=   r·   r·   Ë  s(   € ðÝ"ð €Køô ò ÙÒðús   ‚
 Š˜ÚETÚ_ExperimentManagerMixin)Úboundc                  óz   — e Zd Z	 	 	 d		 	 	 	 	 	 	 d
d„Zedd„«       Zdd„Zd„ Z	 	 	 	 	 	 dd„Zdd„Z		 	 	 	 	 	 dd„Z
y)r‡  Nc               óè  — |xs t        j                  «       | _        d | _        |€t	        «       | _        nlt        |t        «      r6|dz   t        t        j                  «       j                  d d «      z   | _        n&t        t        |j                  «      | _        || _        |xs i }|j                  d«      s'dt        j                  «       j                  d«      i|¥}|xs i | _        || _        y )Nrè   rê   Úrevision_id)rq   rr   r8   Ú_experimentÚ_get_random_nameÚ_experiment_namerM   rN   rO   rû   r  r   r  r²   Úls_envÚget_langchain_env_var_metadataÚ	_metadataÚ_description)r–   r:   r3   r8   r5   s        r=   r˜   z _ExperimentManagerMixin.__init__×  sØ   € ð Ò6¤× 4Ñ 4Ó 6ˆŒØ<@ˆÔØÐÜ$4Ó$6ˆDÕ!Ü˜
¤CÔ(Ø$.°Ñ$4´s¼4¿:¹:»<×;KÑ;KÈBÈQÐ;OÓ7PÑ$PˆDÕ!ä$(¬¨j¯o©oÓ$>ˆDÔ!Ø)ˆDÔà’>˜rˆØ�|‰|˜MÔ*àœv×DÑDÓF×JÑJØ!ó ðð ð	ˆHð "š RˆŒØ'ˆÕr?   c                óH   — | j                   �| j                   S t        d«      ‚)Nz=Experiment name not provided, and experiment not yet started.)rŽ  rZ   r›   s    r=   rš   z'_ExperimentManagerMixin.experiment_nameô  s*   € à× Ñ Ð,Ø×(Ñ(Ð(ÜØKó
ð 	
r?   c                óH   — | j                   €t        d«      ‚| j                   S )NúExperiment not started yet.)rŒ  rZ   r›   s    r=   r�   z'_ExperimentManagerMixin._get_experimentü  s&   € Ø×ÑÐ#ÜÐ:Ó;Ð;Ø×ÑÐr?   c                ó¸   — | j                   xs i }d|d<   t        j                  «       }|ri |¥d|i¥}| j                  ri | j                  j                  ¥|¥}|S )NÚpy_sdk_evaluateÚ__ls_runnerÚgit)r‘  r�  Úget_git_inforŒ  r3   )r–   Úproject_metadataÚgit_infos      r=   Ú_get_experiment_metadataz0_ExperimentManagerMixin._get_experiment_metadata  s~   € ØŸ>™>Ò/¨RÐØ*;Ð˜Ñ'Ü×&Ñ&Ó(ˆÙð Ø"ð à�xñ Ðð ×Òð Ø×"Ñ"×+Ñ+ð à"ð Ðð  Ðr?   c           
     ó´  — | j                   }d}t        | dd «      }t        | dd «      }t        | dd «      }t        |«      D ]:  }	 | j                  j	                  | j                   | j
                  |||||¬«      c S  t        d|› d	�«      ‚# t        j                  $ r7 |› dt        t        j                  «       j                  d d «      › �| _         Y Œ•w xY w)
Né
   Ú_num_examplesÚ_num_repetitionsÚ_evaluator_keys)r5   r  r3   Únum_examplesr7   rK  rè   é   z+Could not find a unique experiment name in z= attempts. Please try again with a different experiment name.)rŽ  Úgetattrr  r8   Úcreate_projectr’  r  ÚLangSmithConflictErrorrN   rO   rû   r  rZ   )	r–   r¦   r3   Ústarting_nameÚnum_attemptsr£  r7   rK  r2  s	            r=   Ú_create_experimentz*_ExperimentManagerMixin._create_experiment  sü   € ð ×-Ñ-ˆØˆÜ˜t _°dÓ;ˆÜ! $Ð(:¸DÓAˆÜ  Ð'8¸$Ó?ˆÜ�|Ö$ˆAðWØ—{‘{×1Ñ1Ø×)Ñ)Ø $× 1Ñ 1Ø)3Ø%Ø!-Ø$3Ø#1ð 2ó ò ð %ô Ø9¸,¸ð HBð Bó
ð 	
øô ×2Ñ2ò WØ+8¨/¸¼3¼t¿z¹z»|×?OÑ?OÐPRÐQRÐ?SÓ;TÐ:UÐ(V�Ö%ðWús   Á5BÂACÃCc                ó’   — | j                   €.| j                  «       }| j                  |j                  |«      }|S | j                   }|S r,   )rŒ  r�  rª  r¦   )r–   Úfirst_exampler›  rw   s       r=   Ú_get_projectz$_ExperimentManagerMixin._get_project,  sR   € Ø×ÑÐ#Ø#×<Ñ<Ó>ÐØ×-Ñ-Ø×(Ñ(Ð*:óˆGð
 ˆð ×&Ñ&ˆGØˆr?   c                ó*  — |r{|j                   ro|j                   j                  d«      d   }|j                  }|j                  d«      d   }|› d|› d|j                  › �}t	        d| j
                  › d|› d�«       y t	        d	| j
                  «       y )
Nr    r   r¡   r¢   r£   z-View the evaluation results for experiment: 'z' at:
rE  z%Starting evaluation of experiment: %s)r¤   r¥   r¦   r[   rF  rš   )r–   rw   r¬  r§   r¦   r¨   r¬   s          r=   Ú_print_experiment_startz/_ExperimentManagerMixin._print_experiment_start6  s©   € ñ �w—{’{Ø!Ÿ+™+×+Ñ+¨CÓ0°Ñ3ˆKØ&×1Ñ1ˆJØ"×(Ñ(¨Ó8¸Ñ;ˆHà�*˜J z lð 3$Ø$+§J¡J <ð1ð ô Ø?À×@TÑ@TÐ?Uð VØ'Ð(¨ð.õô Ø7¸×9MÑ9Mõr?   r@  )r:   ú+Optional[Union[schemas.TracerSession, str]]r3   úOptional[dict]r8   úOptional[langsmith.Client]r5   rÞ   rÚ   )rÛ   úschemas.TracerSession)r¦   rÜ   r3   rA  rÛ   r³  )r¬  r~   rÛ   r³  )rw   zOptional[schemas.TracerSession]r¬  r~   rÛ   rà   )r‚   rƒ   r„   r˜   ræ   rš   r�   r�  rª  r­  r¯  r-   r?   r=   r‡  r‡  Ö  s˜   „ ð
 $(Ø-1Ø%)ð(ð @ð(ð !ð	(ð
 +ð(ð #ó(ð: ò
ó ð
ó ò
 ð 
Ø#ð
Ø/3ð
à	ó
ó6ðØ6ðØGVðà	ôr?   c                  ó¨  ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dˆ fd„Z	 	 	 	 dd„Zedd„«       Zedd„«       Zedd„«       Z	edd„«       Z
dd	„Z	 d 	 	 	 	 	 d!d
„Zddœ	 	 	 	 	 d"d„Z	 	 	 	 d#d„Zd$d„Zd%d„Z	 	 d&	 	 	 	 	 	 	 d'd„Z	 	 	 	 	 	 	 	 d(d„Z	 d 	 	 	 	 	 d)d„Z	 	 	 	 d*d„Zd+d„Zd,d„Zd-d„Zd.d„Zˆ xZS )/rÙ   aQ  Manage the execution of experiments.

    Supports lazily running predictions and evaluations in parallel to facilitate
    result streaming and early debugging.

    Args:
        data (DATA_T): The data used for the experiment. Can be a dataset name or ID OR
            a generator of examples.
        num_repetitions (int): The number of times to run over the data.
        runs (Optional[Iterable[schemas.Run]]): The runs associated with the experiment
            predictions.
        experiment (Optional[schemas.TracerSession]): The tracer session
            associated with the experiment.
        experiment_prefix (Optional[str]): The prefix for the experiment name.
        metadata (Optional[dict]): Additional metadata for the experiment.
        client (Optional[langsmith.Client]): The Langsmith client used for
            the experiment.
        evaluation_results (Optional[Iterable[EvaluationResults]]): The evaluation
            sresults for the experiment.
        summary_results (Optional[Iterable[EvaluationResults]]): The aggregate results
            for the experiment.
    Nc               ó  •— t         ‰| �  ||||¬«       || _        || _        d | _        || _        || _        || _        |	| _        |
�|
nt        || j                  ¬«      | _        || _        || _        || _        || _        || _        y )N)r:   r3   r8   r5   )r8   )Úsuperr˜   Ú_datar¢  r5  Ú_runsÚ_evaluation_resultsr¼   r¡  Ú_resolve_num_examplesr8   r   rR  Ú_reuse_attachmentsÚ_upload_resultsÚ_attachment_raw_data_dictÚ_error_handling)r–   r0   r:   r3   r8   rx   r€   Úsummary_resultsr5   r7   r£  rK  rL  Úreuse_attachmentsr;   Úattachment_raw_data_dictrT   Ú	__class__s                    €r=   r˜   z_ExperimentManager.__init__d  s§   ø€ ô( 	‰ÑØ!ØØØ#ð	 	ô 	
ð ˆŒ
Ø-ˆÔØ>BˆŒØˆŒ
Ø#5ˆÔ Ø /ˆÔØ /ˆÔð Ð'ñ ä& t°D·K±KÔ@ð 	Ôð
 %8ˆÔ!Ø"3ˆÔØ-ˆÔØ)AˆÔ&Ø-ˆÕr?   c                ó|  — t        |d«      r|j                  s|S i }|j                  j                  «       D ]€  \  }}| j                  �jt	        |j
                  «      |z   | j                  v rF|d   t        j                  | j                  t	        |j
                  «      |z      «      |d   dœ||<   Œ||||<   Œ‚ t        j                  |j
                  |j                  |j                  |j                  |j                  |j                  |j                  |j                   ||j"                  |j$                  ¬«      S )aÙ  Reset attachment readers for an example.

        This is only in the case that an attachment is going to be used by more
        than 1 callable (target + evaluators). In that case we keep a single copy
        of the attachment data in `self._attachment_raw_data_dict`, and create
        readers from that data. This makes it so that we don't have to keep
        copies of the same data in memory, instead we can just create readers
        from the same data.
        ÚattachmentsÚpresigned_urlÚ	mime_type)rÅ  ÚreaderrÆ  )r[   Ú
created_atr¦   ÚinputsÚoutputsr3   Úmodified_atr÷   rÄ  Ú	_host_urlÚ
_tenant_id)ÚhasattrrÄ  rY   r½  rN   r[   ÚioÚBytesIOr   ÚExamplerÈ  r¦   rÉ  rÊ  r3   rË  r÷   rÌ  rÍ  )r–   r   Únew_attachmentsr  Ú
attachments        r=   Ú!_reset_example_attachment_readersz4_ExperimentManager._reset_example_attachment_readers�  s"  € ô �w Ô.°g×6IÒ6IØˆNà=?ˆØ '× 3Ñ 3× 9Ñ 9Ö ;ÑˆD�*à×.Ñ.Ð:Ü˜Ÿ
™
“O dÑ*¨d×.LÑ.LÑLð &0°Ñ%@Ü Ÿj™jØ×6Ñ6´s¸7¿:¹:³ÈÑ7MÑNóð ",¨KÑ!8ñ)� Ò%ð )3� Ò%ð !<ô  �‰Ø�z‰zØ×)Ñ)Ø×)Ñ)Ø—>‘>Ø—O‘OØ×%Ñ%Ø×+Ñ+Ø!×/Ñ/Ø'Ø×'Ñ'Ø×)Ñ)ô
ð 	
r?   c           	     óø  ‡ ‡— ‰ j                   �€<t        ‰ j                  ‰ j                  ‰ j                  ¬«      ‰ _         ‰ j
                  rš‰ j                  €Žt        j                  ‰ j                   «      \  }‰ _         |D ���ci c]R  }|j                  xs i j                  «       D ]/  \  }}t        |j                  «      |z   |d   j                  «       “Œ1 ŒT c}}}‰ _        ‰ j                  dkD  rVt        ‰ j                   «      Št        j                   j#                  ˆˆ fd„t%        ‰ j                  «      D «       «      ‰ _         t        j                  ‰ j                   «      \  ‰ _         }|S c c}}}w )N)r8   rL  rÇ  r)   c              3  óf   •K  — | ]#  }‰D �cg c]  }‰j                  |«      ‘Œ c}–— Œ% y c c}w ­wr,   )rÔ  )rE   r2  r   Úexamples_listr–   s      €€r=   rH   z._ExperimentManager.examples.<locals>.<genexpr>Ï  sE   øè ø€ ð ?ñ
 :˜ñ (5óá'4˜Gð ×>Ñ>¸wÕGØ'4õñ :ùò	ùs   ƒ	1Œ,¤1)r5  Ú_resolve_datar·  r8   rR  r»  r½  Ú	itertoolsÚteerÄ  rY   rN   r[   Úreadr¡  r`   ÚchainÚfrom_iterabler  )r–   Úexamples_copyrÃ   r  r>  Úexamples_iterr×  s   `     @r=   r8  z_ExperimentManager.examples¾  s7  ù€ à�>‰>Ñ!Ü*Ø—
‘
Ø—{‘{Ø$(×$=Ñ$=ôˆDŒNð
 ×&Ò&¨4×+IÑ+IÐ+QÜ09·±¸d¿n¹nÓ0MÑ-�˜tœ~ñ +õ2á*˜Ø()¯©Ò(;¸×'BÑ'BÖ'D™˜˜eô ˜Ÿ™“I Ñ$ e¨H¡o×&:Ñ&:Ó&<Ñ<à'Dð %Ø*ó2�Ô.ð
 ×$Ñ$ qÒ(Ü $ T§^¡^Ó 4�Ü!*§¡×!>Ñ!>ô ?ô
 # 4×#8Ñ#8Ô9ó?ó "�”ô )2¯©°d·n±nÓ(EÑ%ˆŒ˜ØÐùô2s   ÂAE5c                ó  — | j                   �t        | j                   dd «      s3t        t        | j                  «      «      }t        |j                  «      S t        t        t        j                  | j                   «      j                  «      S )Nr  )rŒ  r¥  ÚnextÚiterr8  rN   r¦   r   r   r  r  )r–   r   s     r=   r¦   z_ExperimentManager.dataset_idÙ  sr   € à×ÑÐ#¬7Ø×ÑÐ4°dô,
ô œ4 §¡Ó.Ó/ˆGÜ�w×)Ñ)Ó*Ð*ÜÜ”×,Ñ,¨d×.>Ñ.>Ó?×TÑTó
ð 	
r?   c                óX   — | j                   €d„ | j                  D «       S | j                   S )Nc              3  ó$   K  — | ]  }d g i–— Œ
 y­w)rÀ   Nr-   )rE   r2  s     r=   rH   z8_ExperimentManager.evaluation_results.<locals>.<genexpr>ç  s   è ø€ Ð;©]¨�Y ”O©]ùs   ‚)r¹  r8  r›   s    r=   r€   z%_ExperimentManager.evaluation_resultsä  s)   € à×#Ñ#Ð+Ù;¨T¯]ª]Ó;Ð;Ø×'Ñ'Ð'r?   c                ó‚   — | j                   €t        d«      ‚t        j                  | j                   «      \  | _         }|S )Nz;Runs not provided in this experiment. Please predict first.)r¸  rZ   rÙ  rÚ  )r–   Ú	runs_iters     r=   rx   z_ExperimentManager.runsê  s=   € à�:‰:ÐÜØMóð ô !*§¡¨d¯j©jÓ 9ÑˆŒ
�IØÐr?   c                ó"  — t        t        j                  | j                  d«      «      }| j                  r| j                  |«      nd }| j                  ||«       | j                  | j                  d<   | j                  | j                  |¬«      S )Nr)   r7   )r:   )
rá  rÙ  Úislicer8  r¼  r­  r¯  r¡  r‘  Ú_copy)r–   r¬  rw   s      r=   r•   z_ExperimentManager.startó  st   € ÜœY×-Ñ-¨d¯m©m¸QÓ?Ó@ˆØ6:×6JÒ6J�$×#Ñ# MÔ2ÐPTˆØ×$Ñ$ W¨mÔ<Ø,0×,AÑ,Aˆ�‰Ð(Ñ)Ø�z‰z˜$Ÿ-™-°GˆzÓ<Ð<r?   c               óÚ   — t        «       }|j                  | j                  ||t        |«      ¬«      }t	        j
                  |d«      \  }}| j                  d„ |D «       d„ |D «       ¬«      S )z3Lazily apply the target function to the experiment.)r6   rL  rK   c              3  ó&   K  — | ]	  }|d    –— Œ y­w©r   Nr-   ©rE   Úpreds     r=   rH   z6_ExperimentManager.with_predictions.<locals>.<genexpr>
  s   è ø€ Ð,© ˆT�)�_©ùó   ‚c              3  ó&   K  — | ]	  }|d    –— Œ y­w©rz   Nr-   rí  s     r=   rH   z6_ExperimentManager.with_predictions.<locals>.<genexpr>
  s   è ø€ Ð3OÉBÀD°D¸µKÉBùrï  )rx   )r   rz   Ú_predictÚ_target_include_attachmentsrÙ  rÚ  ré  )r–   r/   r6   ÚcontextÚ_experiment_resultsÚr1Úr2s          r=   rX  z#_ExperimentManager.with_predictionsú  sq   € ô “.ˆØ%Ÿk™kØ�M‰MØØ+Ü ;¸FÓ Cð	 *ó 
Ðô —‘Ð2°AÓ6‰ˆˆBØ�z‰zÙ,©Ó,Ñ3OÉBÓ3Oð ó 
ð 	
r?   rN  c               óî   — t        |«      }t        «       }|j                  | j                  ||¬«      }t	        j
                  |d«      \  }}}| j                  d„ |D «       d„ |D «       d„ |D «       ¬«      S )z7Lazily apply the provided evaluators to the experiment.rN  é   c              3  ó&   K  — | ]	  }|d    –— Œ y­wrì  r-   ©rE   r  s     r=   rH   z5_ExperimentManager.with_evaluators.<locals>.<genexpr>"  s   è ø€ Ð0©R 6ˆV�IÕ©Rùrï  c              3  ó&   K  — | ]	  }|d    –— Œ y­wrñ  r-   rû  s     r=   rH   z5_ExperimentManager.with_evaluators.<locals>.<genexpr>#  s   è ø€ Ð1©b F�&˜•-©bùrï  c              3  ó&   K  — | ]	  }|d    –— Œ y­w)r€   Nr-   rû  s     r=   rH   z5_ExperimentManager.with_evaluators.<locals>.<genexpr>$  s   è ø€ ÐNÉ2À Ð';Õ <É2ùrï  )rx   r€   )Ú_resolve_evaluatorsr   rz   Ú_scorerÙ  rÚ  ré  )r–   r1   r6   rô  Úexperiment_resultsrö  r÷  Úr3s           r=   rZ  z"_ExperimentManager.with_evaluators  s{   € ô )¨Ó4ˆ
Ü“.ˆØ$Ÿ[™[Ø�K‰K˜°_ð )ó 
Ðô
 —]‘]Ð#5°qÓ9‰
ˆˆB�Ø�z‰zÙ0©RÓ0Ù1©bÓ1ÙNÉ2ÓNð ó 
ð 	
r?   c                ó´   — t        |«      }t        «       }|j                  | j                  |«      }| j	                  | j
                  | j                  |¬«      S )z?Lazily apply the provided summary evaluators to the experiment.)rx   r¿  )Ú_wrap_summary_evaluatorsr   rz   Ú_apply_summary_evaluatorsré  r8  rx   )r–   r2   Úwrapped_evaluatorsrô  Úaggregate_feedback_gens        r=   r[  z*_ExperimentManager.with_summary_evaluators'  sZ   € ô
 6Ð6HÓIÐÜ“.ˆØ!(§¡Ø×*Ñ*Ð,>ó"
Ðð �z‰zØ�M‰M §	¡	Ð;Qð ó 
ð 	
r?   c              #  ó”   K  — t        | j                  | j                  | j                  «      D ]  \  }}}t	        |||¬«      –— Œ y­w)z?Return the traces, evaluation results, and associated examples.©rz   r   r€   N)Úziprx   r8  r€   r|   )r–   rz   r   r€   s       r=   r¸   z_ExperimentManager.get_results5  sJ   è ø€ ä03Ø�I‰I�t—}‘} d×&=Ñ&=ö1
Ñ,ˆC�Ð,ô &ØØØ#5ôó ñ1
ùr?  c                ó~   — | j                   €dg iS d| j                   D ��cg c]  }|d   D ]  }|‘Œ Œ c}}iS c c}}w )zEIf `summary_evaluators` were applied, consume and return the results.rÀ   )r¼   )r–   rÀ   Úress      r=   r»   z%_ExperimentManager.get_summary_scores@  s_   € à× Ñ Ð(Ø˜r�?Ð"ð à#×4Ò4ôá4�GØ" 9Ô-�Cò à-ð Ø4òð
ð 	
ùós   ¡9c             #  ó€  K  — t        |«      }|dk(  rX| j                  D ]H  }t        ||| j                  | j                  | j
                  | j                  || j                  «      –— ŒJ n±t        j                  |«      5 }| j                  D �cg c]Q  }|j                  t        ||| j                  | j                  | j
                  | j                  || j                  «	      ‘ŒS }}t        j                  |«      D ]  }|j                  «       –— Œ 	 ddd«       | j                  «        yc c}w # 1 sw Y   ŒxY w­w)z(Run the target function on the examples.r   N)Ú_ensure_traceabler8  Ú_forwardrš   r‘  r8   r¼  r¾  r  r  r  r  Úas_completedr  Ú_end)	r–   r/   r6   rL  Úfnr   r  r/  r1  s	            r=   rò  z_ExperimentManager._predictO  s!  è ø€ ô ˜vÓ&ˆà˜aÒØŸ=œ=�ÜØØØ×(Ñ(Ø—N‘NØ—K‘KØ×(Ñ(Ø'Ø×(Ñ(ó	ó 	ñ )ô ×3Ñ3°OÔDÈð $(§=¢=óñ $1˜ð —O‘OÜ ØØØ×,Ñ,ØŸ™ØŸ™Ø×,Ñ,Ø+Ø×,Ñ,õ
ð $1ð ð ô !Ÿo™o¨gÖ6�FØ Ÿ-™-›/Ó)ñ 7÷ Eð$ 	�	‰	�ùò#÷ EÐDüs1   ‚A>D>Â D2ÂAD-Ã%.D2ÄD>Ä-D2Ä2D;Ä7D>c                ó   — t        j                  «       }i |d   xs i ¥| j                  |d   j                  |d   j                  dœ¥}t        j                  di i |¥d|| j
                  sdnd| j                  dœ¥¤Ž5  |d   }|d   }|d	   }|D ]Ü  }	t        j                  «       }
	 |	j                  |||
¬
«      }|d   j                  | j                  j                  |«      «       | j
                  r7| j                  j                  ||| j                  «       j                  |¬«       |j,                  €Œ©|j,                  D ]%  }|j,                  |   d   }|j/                  d«       Œ' ŒÞ t1        |||¬«      cd d d «       S # t        $ �r/}	 |	j                  }t!        |D �cg c]  }t#        ||
t%        |«      ddi¬«      ‘Œ nc c}w c}¬«      }|d   j                  | j                  j                  |«      «       | j
                  r7| j                  j                  ||| j                  «       j                  |¬«       n.# t        $ r"}t&        j)                  d|› �«       Y d }~nd }~ww xY wt&        j+                  dt%        |	«      › d|r|j                  nd› dt%        |«      › �d¬«       Y d }~�ŒŽd }~ww xY w# 1 sw Y   y xY w)Nr3   r   rz   )r:   rv   Úreference_run_idr1   ÚlocalT)rñ   r3   Úenabledr8   r€   )rz   r   Úevaluator_run_idrÀ   )rz   ri  Ú	_executorÚerror)ró   r÷   rõ   Úextra)rÀ   zError parsing feedback keys: zError running evaluator z on run Ú ú: ©Úexc_inforÇ  r   r  r-   )rf   Úget_tracing_contextrš   r[   rþ   r¼  r8   rO   rû   Úevaluate_runÚextendÚ_select_eval_resultsÚ_log_evaluation_feedbackr�   Ú	ExceptionÚfeedback_keysr"   r!   Úreprr\   r]   r  rÄ  Úseekr|   )r–   r1   Úcurrent_resultsr  Úcurrent_contextr3   rz   r   Úeval_resultsr-  r  Úevaluator_responserÃ   r$  ró   Úerror_responseÚe2rÓ  rÇ  s                      r=   Ú_run_evaluatorsz"_ExperimentManager._run_evaluators{  s	  € ô ×0Ñ0Ó2ˆð
Ø˜zÑ*Ò0¨bð
ð #×2Ñ2Ø(7¸	Ñ(B×(EÑ(EØ$3°EÑ$:×$=Ñ$=ñð
ˆô ×Ññ 
ðØ!ðà ,Ø$Ø*.×*>Ò*>™7ÀDØŸ+™+òó
ð " %Ñ(ˆCØ% iÑ0ˆGØ*Ð+?Ñ@ˆLÛ'�	Ü#'§:¡:£<Ð ð3Ø)2×)?Ñ)?ØØ 'Ø)9ð *@ó *Ð&ð ! Ñ+×2Ñ2ØŸ™×8Ñ8Ð9KÓLôð ×+Ò+àŸ™×<Ñ<Ø.Ø #Ø'+×';Ñ';Ó'=×'@Ñ'@Ø&.ð	 =ô ðP ×&Ñ&Ñ2Ø&-×&9Ô&9˜
Ø!(×!4Ñ!4°ZÑ!@ÀÑ!J˜ØŸ™ A�ñ ':ðo (ôv 'ØØØ#/ô÷O
ñ 
øô@ !ó !ðØ(1×(?Ñ(?˜ä):ñ ,9ó%ñ ,9 Cô !1Ø(+Ø2BÜ,0°«GØ+2°D¨/ö	!"ñ ,9ùô%ô
*˜ð % YÑ/×6Ñ6Ø ŸK™K×<Ñ<¸^ÓLôð  ×/Ò/à ŸK™K×@Ñ@Ø .Ø$'Ø+/×+?Ñ+?Ó+A×+DÑ+DØ*2ð	 Aô ùô %ò ÜŸ™Ð'DÀRÀDÐ%IÔJÜûðúô —L‘LØ2´4¸	³?Ð2Cð D Ù*- §¢°2Ð6°b¼¸a»¸	ðCà!%ð !÷ ò ûð;!ú÷A
ð 
úsp   Â )KÂ*BFÄ.KÄ;AKÆ
KÆIÆ(!G
Ç	A>IÉJ<É	I3	ÉI.	É)J<É.I3	É3AJ<Ê6KÊ<KËKËKc           
   #  ó¦  K  — t        j                  |xs d¬«      5 }|dk(  r@t        «       }| j                  «       D ]"  }|j	                  | j
                  |||«      –— Œ$ nºt        «       }| j                  «       D ]o  }|j                  |j                  | j
                  |||«      «       	 t        j                  |d¬«      D ]%  }|j                  «       –— |j                  |«       Œ' Œq t        j                  |«      D ]  }|j                  «       }|–— Œ ddd«       y# t        j                  t        f$ r Y ŒÄw xY w# 1 sw Y   yxY w­w)zšRun the evaluators on the prediction stream.

        Expects runs to be available in the manager.
        (e.g. from a previous prediction step)
        r)   r  r   gü©ñÒMbP?)r¯   N)r  r  r   r¸   rz   r-  r¾   Úaddr  r  r  r  ÚremoveÚTimeoutError)	r–   r1   r6   r  rô  r'  r/  r1  r  s	            r=   rÿ  z_ExperimentManager._score×  s7  è ø€ ô ×/Ñ/Ø'Ò,¨1õ
àØ !Ò#Ü&›.�Ø'+×'7Ñ'7Ö'9�OØ!Ÿ+™+Ø×,Ñ,Ø"Ø'Ø ó	ó ñ (:ô ›%�Ø'+×'7Ñ'7Ö'9�OØ—K‘KØ Ÿ™Ø ×0Ñ0Ø&Ø+Ø$ó	ôðô ')§o¡o°gÀu×&M˜FØ"(§-¡-£/Ò1Ø#ŸN™N¨6Õ2ñ 'Nð (:ô" !Ÿo™o¨gÖ6�FØ#Ÿ]™]›_�FØ “Lð 7÷?
ð 
øô: ŸO™O¬\Ð:ò Ùðú÷;
ð 
üsA   ‚E�BEÂ.?D&Ã-0EÄ	EÄ&EÄ?EÅEÅEÅEÅ
Ec              #  ó"  K  — g g }}t        | j                  | j                  «      D ]'  \  }}|j                  |«       |j                  |«       Œ) g }t	        j
                  «       5 }| j                  r| j                  «       j                  nd }t        j                  «       }	i |	d   xs i ¥| j                  |dœ¥}
t        j                  di i |	¥d|
| j                  | j                  sdnddœ¥¤Ž5  |D ]©  }	  |||«      }| j                  j                  ||j                  ¬«      }|j!                  |«       | j                  rY|D ]T  }|j#                  dh¬	«      }|j%                  d
d «      } |j&                  | j                  j(                  fi |¤d ||dœ¤Ž ŒV Œ« 	 d d d «       d d d «       d|i–— y # t*        $ r0}t,        j/                  dt1        |«      › d|› �d¬«       Y d }~Œød }~ww xY w# 1 sw Y   ŒTxY w# 1 sw Y   ŒXxY w­w)Nr3   )r:   rž   r1   r  T)rñ   r3   r8   r  )Úfn_nameÚtarget_run_id)ÚexcludeÚevaluator_info)rò   ri  Úsource_infoz Error running summary evaluator r  r  rÀ   r-   )r	  rx   r8  rº   r  r  r¼  r�   r[   rf   r  rš   rþ   r8   r!  r‚   r   Ú
model_dumpÚpopr  r  r#  r\   r  r%  )r–   r2   rx   r8  rz   r   Úaggregate_feedbackr  ri  r(  r3   r-  Úsummary_eval_resultÚflattened_resultsr  Úfeedbackr6  rÃ   s                     r=   r  z,_ExperimentManager._apply_summary_evaluators  s.  è ø€ ð ˜RˆhˆÜ §	¡	¨4¯=©=Ö9‰LˆC�Ø�K‰K˜ÔØ�O‰O˜GÕ$ð :ð  ÐÜ×/Ñ/Ô1°XØ6:×6JÒ6J˜×-Ñ-Ó/×2Ò2ÐPTˆJÜ ×4Ñ4Ó6ˆOðØ" :Ñ.Ò4°"ðð #'×"6Ñ"6Ø%/ñðˆHô ×#Ñ#ñ ðØ%ðà$0Ø (Ø"Ÿk™kØ.2×.BÒ.B™wÈòóó "4�IðÙ.7¸¸hÓ.GÐ+à,0¯K©K×,LÑ,LØ/Ø$-×$6Ñ$6ð -Mó -Ð)ð +×1Ñ1Ð2CÔDØ×/Ò/Û*; Ø+1×+<Ñ+<ÀoÐEVÐ+<Ó+W Ø19·±Ð>NÐPTÓ1U Ø / §¡Ø$(§K¡K×$?Ñ$?ñ!"à&.ð!"ð ,0Ø/9Ø0>õ!"ð +<øñ "4÷÷ 2ðX Ð,Ð-Ó-øô %ò ÜŸ™Ø>¼tÀI»Ð>OÈrÐRSÐQTÐUØ%)ð %÷ ñ ûðú÷;ð ú÷ 2Ð1üsb   ‚A%HÁ'BHÃ4G7Ã;B&F;Æ!G7Æ$HÆ,HÆ;	G4	Ç&G/	Ç*G7Ç/G4	Ç4G7Ç7H 	Ç<HÈHÈHc                óÈ   — t        | j                  «      }|D �cg c]  }|j                  sŒ|j                  ‘Œ }}|rt        |«      nd }|r|j	                  «       S d S c c}w r,   )r`   r8  rË  ÚmaxÚ	isoformat)r–   r8  ÚexrË  Úmax_modified_ats        r=   Ú_get_dataset_versionz'_ExperimentManager._get_dataset_version:  sZ   € Ü˜Ÿ™Ó&ˆÙ08ÓK±¨"¸B¿N»N�r—~“~°ˆÐKñ /:œ#˜kÔ*¸tˆÙ.=ˆ×(Ñ(Ó*ÐGÀ4ÐGùò	 Ls
   šA¬Ac                ó|  — t        | j                  «      }t        «       }|D ]Ž  }|j                  ro|j                  j	                  d«      rTt        |j                  d   t         «      r7|j                  d   D ]$  }t        |t        «      sŒ|j                  |«       Œ& Œ~|j                  d«       Œ� t        |«      S )NÚdataset_splitÚbase)r`   r8  r¾   r3   r²   rM   rN   r/  )r–   r8  Úsplitsr   r¥   s        r=   Ú_get_dataset_splitsz&_ExperimentManager._get_dataset_splitsB  s˜   € Ü˜Ÿ™Ó&ˆÜ“ˆÛˆGà× Ò Ø×$Ñ$×(Ñ(¨Ô9Ü˜w×/Ñ/°Ñ@Ä$ÔGà$×-Ñ-¨oÔ>�EÜ! %¬Õ-ØŸ
™
 5Õ)ñ ?ð —
‘
˜6Õ"ð  ô �F‹|Ðr?   c                ó&  — | j                   sy | j                  }|€t        d«      ‚| j                  «       }| j	                  «       |d<   | j                  «       |d<   | j                  j                  |j                  i |j                  ¥|¥¬«       y )Nr•  rí   Údataset_splits)r3   )
r¼  rŒ  rZ   r�  rC  rH  r8   Úupdate_projectr[   r3   )r–   r:   r›  s      r=   r  z_ExperimentManager._endS  s�   € Ø×#Ò#ØØ×%Ñ%ˆ
ØÐÜÐ:Ó;Ð;à×8Ñ8Ó:ÐØ.2×.GÑ.GÓ.IÐÐ*Ñ+Ø-1×-EÑ-EÓ-GÐÐ)Ñ*Ø�‰×"Ñ"Ø�M‰MðØ×%Ñ%ðà"ðð 	#õ 	
r?   c                ó¶  — | j                   f}| j                  | j                  | j                  | j                  | j
                  | j                  | j                  | j                  | j                  | j                  | j                  | j                  | j                  dœ}t        |«      t        |t        |«      d  «      z   }i |¥|¥} | j                   |i |¤ŽS )N)r:   r3   rx   r8   r€   r¿  r£  rK  rL  rÀ  r;   rÁ  rT   )r·  rŒ  r‘  r¸  r8   r¹  r¼   r   r¢  rR  r»  r¼  r½  r¾  r`   ra   rÂ  )r–   Úargsr<   Údefault_argsÚdefault_kwargsÚ	full_argsÚfull_kwargss          r=   ré  z_ExperimentManager._copye  sÈ   € ØŸ
™
�}ˆà×*Ñ*ØŸ™Ø—J‘JØ—k‘kØ"&×":Ñ":Ø#×4Ñ4Ø ×.Ñ.Ø"×2Ñ2Ø#'×#<Ñ#<Ø!%×!8Ñ!8Ø"×2Ñ2Ø(,×(FÑ(FØ"×2Ñ2ñ
ˆô ˜“J¤ l´3°t³9°;Ð&?Ó!@Ñ@ˆ	Ø2˜Ð2¨6Ð2ˆØˆt�~‰~˜yÐ8¨KÑ8Ð8r?   )NNNNNNr)   NNFFTNÚlog) r:   r°  r3   r±  r8   r²  rx   úOptional[Iterable[schemas.Run]]r€   ú%Optional[Iterable[EvaluationResults]]r¿  rT  r5   rÞ   r7   rá   r£  rã   rK  úOptional[list[str]]rL  rU   rÀ  rU   r;   rU   rÁ  r±  rT   úLiteral['log', 'ignore']r0   ÚDATA_T)r   r~   rÛ   r~   )rÛ   úIterable[schemas.Example]rÚ   )rÛ   zIterable[EvaluationResults])rÛ   zIterable[schemas.Run])rÛ   rÙ   r,   )r6   rã   r/   rY  rÛ   rÙ   )r1   z*Sequence[Union[EVALUATOR_T, RunEvaluator]]r6   rã   rÛ   rÙ   )r2   úSequence[SUMMARY_EVALUATOR_T]rÛ   rÙ   )rÛ   úIterable[ExperimentResultRow])rÛ   zdict[str, list[dict]])NF)r6   rã   rL  rU   r/   rY  rÛ   z&Generator[_ForwardResults, None, None])r1   úSequence[RunEvaluator]r'  r|   r  zcf.ThreadPoolExecutorrÛ   r|   )r1   r[  r6   rã   rÛ   rZ  )r2   rY  rÛ   z(Generator[EvaluationResults, None, None]rÝ   )rÛ   rU  rß   )rM  r   r<   r   rÛ   rÙ   )r‚   rƒ   r„   rå   r˜   rÔ  ræ   r8  r¦   r€   rx   r•   rX  rZ  r[  r¸   r»   rò  r-  rÿ  r  rC  rH  r  ré  Ú__classcell__)rÂ  s   @r=   rÙ   rÙ   L  sk  ø„ ñð8 $(Ø-1Ø04ØDHØAEØ%)Ø Ø&*Ø.2Ø$)Ø"'Ø#Ø37Ø38ð%*.ð @ð	*.ð
 !ð*.ð +ð*.ð .ð*.ð Bð*.ð ?ð*.ð #ð*.ð ð*.ð $ð*.ð ,ð*.ð "ð*.ð  ð*.ð  ð!*.ð" #1ð#*.ð$ 1ð%*.àõ*.ðX,
Ø&ð,
à	ó,
ð\ òó ðð4 ò
ó ð
ð ò(ó ð(ð
 òó ðó=ð *.ð	
ð 'ð	
àð
ð
 
ó
ð8 *.ñ
ð
ð
ð 'ð
ð 
ó
ð4
à9ð
ð 
ó
ó	ó
ð& *.Ø$)ð*ð 'ð	*ð
 "ð*àð*ð 
0ó*ðXZà*ðZð -ðZð (ð	Zð
 
óZð~ *.ð+!à*ð+!ð 'ð+!ð 
'ó	+!ðZ4.Ø"?ð4.à	1ó4.ólHóó"
÷$9r?   rÙ   c                ó�   — g }| D ]>  }t        |t        «      r|j                  |«       Œ%|j                  t        |«      «       Œ@ |S r,   )rM   r#   rº   r&   )r1   rÀ   r-  s      r=   rþ  rþ  {  sA   € ð €GÛˆ	Ü�i¤Ô.Ø�N‰N˜9Õ%à�N‰Nœ=¨Ó3Õ4ð	  ð
 €Nr?   c                óN   — dd„}g }| D ]  }|j                   ||«      «       Œ |S )Nc                ó„   ‡ ‡— t        ‰ dd«      Št        ‰ «      Š t        j                  ‰ «      	 	 	 	 	 	 dˆˆ fd„«       }|S )Nr‚   ÚBatchEvaluatorc                óœ   •‡ ‡— t        j                  ‰¬«      	 	 	 	 	 	 dˆˆˆ fd„«       } |dt        ‰ «      › d�dt        ‰«      › d�«      S )N©r  c                ó:   •—  ‰t        ‰«      t        ‰«      «      S r,   )r`   )Úruns_Ú	examples_r-  r8  rx   s     €€€r=   Ú_wrapper_super_innerz]_wrap_summary_evaluators.<locals>._wrap.<locals>._wrapper_inner.<locals>._wrapper_super_inner’  s   ø€ ñ !¤ d£¬T°(«^Ó<Ð<r?   zRuns[] (Length=Ú)zExamples[] (Length=)rd  rN   re  rN   rÛ   ú*Union[EvaluationResult, EvaluationResults])rf   Ú	traceablera   )rx   r8  rf  Ú	eval_namer-  s   `` €€r=   Ú_wrapper_innerz?_wrap_summary_evaluators.<locals>._wrap.<locals>._wrapper_innerŽ  sh   ú€ ô �\‰\˜yÔ)ð=Øð=Ø'*ð=à;ö=ó *ð=ñ
 (Ø!¤# d£) ¨AÐ.Ð2EÄcÈ(ÃmÀ_ÐTUÐ0Vóð r?   )rx   zSequence[schemas.Run]r8  zSequence[schemas.Example]rÛ   rh  )r¥  r$   Ú	functoolsÚwraps)r-  rk  rj  s   ` @r=   Ú_wrapz'_wrap_summary_evaluators.<locals>._wrapŠ  sW   ù€ Ü˜I zÐ3CÓDˆ	Ü0°Ó;ˆ	ä	�‰˜Ó	#ð	Ø'ð	Ø3Lð	à7õ	ó 
$ð	ð Ðr?   )r-  r   rÛ   r   )rº   )r1   rn  rÀ   r-  s       r=   r  r  ‡  s.   € óð( €GÛˆ	Ø�‰‘u˜YÓ'Õ(ð  à€Nr?   c                  ó"   — e Zd ZU ded<   ded<   y)Ú_ForwardResultsr}   rz   r~   r   Nr�   r-   r?   r=   rp  rp  ¤  s   … Ø	ÓØÔr?   rp  c                ó   ‡‡— d Šdˆfd„}dˆfd„}	‰j                   xs ‰j                  j                  «       }
t        j                  ||i |¥d|
i¥|¬«      }|dk(  r‰j
                  |d<   n|dk(  r|	|d<   nt        d	|›�«      ‚t        j                  |sd
nd¬«      5  	 t        | «      }|D �cg c]  }t        ‰|«      ‘Œ }} | |d|iŽ |r@‰j                  �4‰j                  D ]%  }‰j                  |   d   }|j                  d«       Œ' t        t!        t"        j$                  ‰«      ‰¬«      cd d d «       S c c}w # t        $ r%}t        j                  d|› �dd¬«       Y d }~Œ\d }~ww xY w# 1 sw Y   y xY w)Nc                ó
   •— | Šy r,   r-   )ro  rz   s    €r=   Ú_get_runz_forward.<locals>._get_runµ  s   ø€ à‰r?   c                ó(   •— ‰j                   | _        y r,   )r[   rv   )ro  r   s    €r=   Ú_set_reference_example_idz+_forward.<locals>._set_reference_example_id¹  s   ø€ Ø!(§¡ˆÕr?   Úexample_version)Úon_endrñ   r3   r8   rR  rv   ÚignoreÚ_on_successz2Unrecognized error_handling value: error_handling=r  T)r  Úlangsmith_extrarÇ  r   zError running target function: r)   )r  Ú
stacklevel)rz   r   )ro  z
rt.RunTreerÛ   rà   )rË  rÈ  r@  rf   ÚLangSmithExtrar[   rZ   rþ   Ú_get_target_argsr¥  rÄ  r&  r#  r\   r  rp  r   r   rO  )r  r   rš   r3   r8   r;   rL  rT   rs  ru  rv  rz  Ú	arg_namesÚargnrM  rÓ  rÇ  rÃ   rz   s    `                @r=   r  r  ©  s‹  ù€ ð &*€Cõõ,ð ×*Ñ*Ò@¨g×.@Ñ.@×KÑKÓM€OÜ×'Ñ'ØØ$ØA�HÐAÐ/°ÑAØô	€Oð ˜ÒØ29·*±*ˆÐ.Ò/Ø	˜8Ò	#à)Bˆ˜Ò&äÐN¸~Ð>OÐPÓQÐQä	×	Ñ	±>¡GÀtÖ	Lð	Ü(¨Ó,ˆIÙ7@ÓA±y¨t”G˜G TÕ*°yˆDÐAÙ�Ð6 oÒ6á" w×':Ñ':Ð'FØ")×"5Ô"5�JØ$×0Ñ0°Ñ<¸XÑF�FØ—K‘K •Nð #6ô ¤4¬¯©°SÓ#9À7ÔK÷ 
MÑ	Lùò Bøô ò 	Ü�L‰LØ1°!°Ð5ÀÐQRð ÷ ñ ûð	ú÷ 
MÐ	LúsI   Â E4Â"EÂ1D>ÃAEÄ$E4Ä>EÅ	E1ÅE,Å'E4Å,E1Å1E4Å4E=c                óN   — 	 t        j                  | «       y# t        $ r Y yw xY wra  rb  rc  s    r=   Ú_is_valid_uuidr�  Ü  re  rf  c                ó¢   — | sg S g }	 t        | «      D ]'  }|j                  D ]  }|sŒ|j                  |«       Œ Œ) 	 |S # t        $ r |cY S w xY w)a]  Best-effort extraction of feedback keys for the progress hint.

    Used to populate ``extra.__progress.evaluator_keys`` on the experiment
    session so the UI can render per-evaluator progress. Returns an empty list
    when keys cannot be statically inferred (e.g. arbitrary callables whose
    return shape isn't visible to AST inspection).
    )rþ  r$  rº   r#  )r1   ÚkeysÚevrF   s       r=   rQ  rQ  ä  s`   € ñ Øˆ	Ø€DðÜ% jÖ1ˆBØ×%Ô%�ÚØ—K‘K •Nñ &ñ 2ð €Køô ò ØŠðús   ˆA  ¨A  Á AÁAc               ó*  — 	 t        | t        «      rt        | t        «      st        | «      S d}t        | t        j
                  «      r%| j                  �| j                  S | j                  }nMt        | t        j                  «      r| }n0t        | t        «      r t        | «      rt        j                  | «      }|�|j                  |¬«      j                  S t        | t        «      r|j                  | ¬«      j                  S y# t        $ r Y yw xY w)u`  Best-effort dataset size for the experiment progress hint.

    Returns ``None`` for lazy iterators where size cannot be inferred without
    consuming the iterable. Never raises â€” failures (e.g. transient backend
    error during ``read_dataset``) degrade to ``None`` and the backend resolves
    the count from the dataset at experiment start.
    N)r¦   )Údataset_name)rM   r
   rN   ra   r   ÚDatasetÚexample_countr[   rO   rP   r�  Úread_datasetr#  )r0   r8   r¦   s      r=   rº  rº  û  sâ   € ðä�dœEÔ"¬:°d¼CÔ+@Ü�t“9Ðà*.ˆ
Ü�dœGŸO™OÔ,Ø×!Ñ!Ð-Ø×)Ñ)Ð)ØŸ™‰JÜ˜œdŸi™iÔ(Ø‰JÜ˜œcÔ"¤~°dÔ';ÜŸ™ 4›ˆJØÐ!Ø×&Ñ&°*Ð&Ó=×KÑKÐKä�dœCÔ Ø×&Ñ&°DÐ&Ó9×GÑGÐGØøÜò Ùðús#   ‚*D ­3D Á!A7D Ã+D Ä	DÄD)rL  c               ó–  — t        | t        j                  «      r|j                  | |¬«      S t        | t        «      r1t        | «      r&|j                  t        j                  | «      |¬«      S t        | t        «      r|j                  | |¬«      S t        | t        j                  «      r|j                  | j                  |¬«      S | S )z*Return the examples for the given dataset.)r¦   rL  )r†  rL  )	rM   rO   rP   r  rN   r�  r   r‡  r[   )r0   r8   rL  s      r=   rØ  rØ    sÌ   € ô �$œŸ	™	Ô"Ø×#Ñ#ØÐ1Dð $ó 
ð 	
ô 
�Dœ#Ô	¤>°$Ô#7Ø×#Ñ#Ü—y‘y “Ð<Oð $ó 
ð 	
ô 
�Dœ#Ô	Ø×#Ñ#ØÐ3Fð $ó 
ð 	
ô 
�Dœ'Ÿ/™/Ô	*Ø×#Ñ#Ø—w‘wÐ4Gð $ó 
ð 	
ð €Kr?   c                ó   — d| v r| d   S | S )NrÉ  r-   )rÉ  s    r=   Ú_default_process_inputsrŒ  9  s   € Ø'¨6Ñ1ˆ6�(ÑÐ=°vÐ=r?   c                óð   — t        | «      st        d«      ‚t        j                  | «      r| }|S t	        | «      r| j
                  }  t        j                  dt        ¬«      t        t        | «      «      }|S )z(Ensure the target function is traceable.zÐTarget must be a callable function or a langchain/langgraph object. For example:

def predict(inputs: dict) -> dict:
    # do work, like chain.invoke(inputs)
    return {...}

evaluate(
    predict,
    ...
)ÚTarget)r  Úprocess_inputs)
rI  rZ   rf   Úis_traceable_functionrH  Úinvokeri  rŒ  r   r   )r/   r  s     r=   r  r  =  sv   € ô ˜ÔÜðó

ð 
	
ô 
×Ñ Ô'Ø6<ˆð €Iô " &Ô)Ø—]‘]ˆFØPŒR�\‰\˜xÔ8OÔPÜ”˜6Ó"ó
ˆð €Ir?   c                óD   — t        | «      xs t        t        |«      «      S r,   )ró  rU   Ú_evaluators_include_attachments)r/   r1   s     r=   rR  rR  Y  s#   € Ü& vÓ.ò ´$Ü'¨
Ó3ó3ð r?   c                ó,   — | €yt        d„ | D «       «      S )Nr   c              3  ó2   K  — | ]  }t        |«      –— Œ y ­wr,   )Ú_evaluator_uses_attachments)rE   rÃ   s     r=   rH   z2_evaluators_include_attachments.<locals>.<genexpr>c  s   è ø€ ÐB±z°!Ô*¨1×-±zùs   ‚)Úsum)r1   s    r=   r“  r“  _  s   € ØÐØäÑB±zÓBÓBÐBr?   c                ó   — t        | «      syt        j                  | «      }t        |j                  j                  «       «      }|D �cg c])  }|j                  |j                  |j                  fv sŒ(|‘Œ+ }}t        d„ |D «       «      S c c}w )NFc              3  ó:   K  — | ]  }|j                   d k(  –— Œ y­w)rÄ  Nrb  )rE   r   s     r=   rH   z._evaluator_uses_attachments.<locals>.<genexpr>n  s   è ø€ ÐBÑ0A¨1ˆq�v‰v˜Õ&Ñ0Aùs   ‚)
re   ÚinspectÚ	signaturer`   Ú
parametersrW   ÚkindÚPOSITIONAL_ONLYÚPOSITIONAL_OR_KEYWORDrV   )r-  ÚsigÚparamsr   Úpositional_paramss        r=   r–  r–  f  s…   € Ü�IÔØÜ
×
Ñ
˜IÓ
&€CÜ�#—.‘.×'Ñ'Ó)Ó*€FáóÙˆa˜QŸV™V¨×(9Ñ(9¸1×;RÑ;RÐ'SÒSŠ�6ð ð ô ÑBÑ0AÓBÓBÐBùòs   Á	)BÁ3Bc                ó   — dt        | «      v S )ú0Whether the target function accepts attachments.rÄ  )r}  rˆ   s    r=   ró  ró  q  s   € àÔ,¨VÓ4Ð4Ð4r?   c                ó&  — t        | «      sg S t        | «      rdgS t        j                  | «      }t	        |j
                  j                  «       «      }|D �cg c])  }|j                  |j                  |j                  fv sŒ(|‘Œ+ }}|D �cg c]  }|j                  |j                  u sŒ|‘Œ }}t        |«      dk(  rt        d«      ‚t        |«      dkD  rt        d«      ‚t        |«      dkD  rQ|D �ch c]  }|j                  ’Œ c}j                  g d¢«      r&t        d|D �cg c]  }|j                  ‘Œ c}› �«      ‚g }|d	d D ]-  }|j                  d
v r|j!                  |j                  «       Œ- n |xs dgS c c}w c c}w c c}w c c}w )r¤  rÉ  r   zFTarget function must accept at least one positional argument (inputs).rù  zlTarget function must accept at most three arguments without default values: (inputs, attachments, metadata).r)   )rÉ  rÄ  r3   zˆWhen passing multiple positional arguments without default values, they must be named 'inputs', 'attachments', or 'metadata'. Received: N>   rÉ  r3   rÄ  )re   rH  rš  r›  r`   rœ  rW   r�  rž  rŸ  Údefaultr±   ra   rZ   r  Ú
differencerº   )r/   r   r¡  r   r¢  Úpositional_no_defaultrM  s          r=   r}  r}  v  s´  € ä�FÔØˆ	Ü˜fÔ%ØˆzÐä
×
Ñ
˜FÓ
#€CÜ�#—.‘.×'Ñ'Ó)Ó*€FáóÙˆa˜QŸV™V¨×(9Ñ(9¸1×;RÑ;RÐ'SÒSŠ�6ð ð ñ ):ÓRÑ(9 1¸Q¿Y¹YÈ!Ï'É'Ò=QšQÐ(9ÐÐRä
ÐÓ Ò"ÜØTó
ð 	
ô 
Ð"Ó	# aÒ	'ÜðQó
ð 	
ô 
Ð"Ó	# aÒ	'Ù-ó-Ù-�1ˆ�‹Ð-ñ-ç�jÒ6Ó7ð-8ô ðOá 5Ó6Ñ 5˜1�—“Ð 5Ñ6Ð7ð9ó
ð 	
ð ˆØ" 2 AÓ&ˆAØ�v‰vÐ>Ñ>Ø—‘˜AŸF™FÕ#áð	 'ð
 Ò!˜�zÐ!ùò;ùò Sùò-ùò 7s$   Á)E?ÂE?ÂFÂ*FÃ4F	Ä%F
c                óv  — | �[t        | t        j                  «      r| }nt        | |«      }|j                  st        d«      ‚|j                  st        d«      ‚||fS |�Zt        j                  |«      \  }}t        |«      }|j                  |j                  ¬«      }|j                  st        d«      ‚||fS y)Nz,Experiment name must be defined if provided.zOExperiment must have an associated reference_dataset_id, but none was provided.rh  z,Experiment name not found for provided runs.)NN)rM   r   rQ   rs   r  rZ   r  rÙ  rÚ  rá  rj  rù   )r:   rx   r8   r\  rd  Ú	first_runs         r=   rP  rP  Ÿ  sÅ   € ð ÐÜ�j¤'×"7Ñ"7Ô8Ø$‰Kä*¨:°vÓ>ˆKà×ÒÜÐKÓLÐLØ×/Ò/Üð)óð ð ˜DÐ Ð àÐÜ—m‘m DÓ)‰ˆˆtÜ˜“Kˆ	Ø×)Ñ)°Y×5IÑ5IÐ)ÓJˆØ×ÒÜÐKÓLÐLØ˜DÐ Ð Ør?   c                 ó   — ddl m}   | «       S )Nr   ©Úrandom_name)Ú%langsmith.evaluation._name_generationr­  r¬  s    r=   r�  r�  À  s   € ÝAá‹=Ðr?   c                ó‚   — 	 dd l } |j                  t        | ||¬«      «      S # t        $ r}t        d«      |‚d }~ww xY w)Nr   z¤The 'pandas' library is required to use the 'to_pandas' function. Please install it using 'pip install pandas' or 'conda install pandas' before calling this method.rÇ   )rÌ   r…  rä   Ú_flatten_experiment_results)rÀ   r•   rÈ   ÚpdrÃ   s        r=   rÉ   rÉ   Æ  sQ   € ð
Ûð ˆ2�<‰<Ô3°GÀ5ÈcÔRÓSÐSøô ò ÜðAó
ð ð		ûðús   ‚$ ¤	>­9¹>c                óF  — | || D ����cg c�]m  }i |d   j                   xs i j                  «       D ��ci c]  \  }}d|› �|“Œ c}}¥|d   j                  xs i j                  «       D ��ci c]  \  }}d|› �|“Œ c}}¥d|d   j                  i¥|d   j                  �4|d   j                  j                  «       D ��ci c]  \  }}d|› �|“Œ c}}ni ¥|d   d   D �ci c]4  }d	|j                  › �|j
                  �|j
                  n|j                  “Œ6 c}¥|d   j                  r-|d   j                  |d   j                  z
  j                  «       nd |d   j                  |d   j                  d
œ¥‘�Œp c}}}}S c c}}w c c}}w c c}}w c c}w c c}}}}w )Nr   zinputs.rz   zoutputs.r  z
reference.r€   rÀ   r  )Úexecution_timer0  r[   )rÉ  rY   rÊ  r  ró   rô   r>  Úend_timerú   Útotal_secondsrv   r[   )rÀ   r•   rÈ   rƒ  rF   rG   ro  s          r=   r°  r°  ×  sÒ  € ð6 ˜˜sÑ#ö-ò, $ˆAð+	
Ø-.¨y©\×-@Ñ-@Ò-FÀB×,MÑ,MÔ,OÔPÑ,O¡D A q�˜˜ˆ}˜aÑÐ,OÒPð	
à./°©h×.>Ñ.>Ò.DÀ"×-KÑ-KÔ-MÔNÑ-M¡T Q¨�˜!˜ˆ~˜qÑ Ð-MÒNð	
ð �Q�u‘X—^‘^ñ	
ð �Y‘<×'Ñ'Ð3ð 23°9±×1EÑ1E×1KÑ1KÔ1MÔNÑ1M©¨¨A�:˜a˜SÐ! 1Ñ$Ð1MÓNàð	
ð Ð/Ñ0°Ò;óá;�Að ˜AŸE™E˜7Ð#°·±Ð0C Q§W¢WÈÏÉÑPØ;ñð	
ð �U‘8×$Ò$ð �5‘×"Ñ" Q u¡X×%8Ñ%8Ñ8×GÑGÔIàà˜E™(×7Ñ7Ø�E‘(—+‘+õ'	
ð* $ô-ð ùãPùÛNùó Oùòùõs<   ‹+F
¶FÁ(F
Á.F
Á>AF
ÃFÃF
Ã"9FÄA#F
ÆF
)Úmaxsizec                 ó2   — 	 ddl m}  | S # t        $ r Y y w xY w)Nr   r'   )Úlangchain_core.runnablesr(   r…  r'   s    r=   Ú_import_langchain_runnabler¹  ö  s!   € ðÝ5àˆøÜò Ùðús   ‚
 Š	•c                óH   — t        t        «       x}xr t        | |«      «      S r,   )rU   r¹  rM   )Úor(   s     r=   rH  rH   	  s"   € ÜÔ7Ó9Ð9�ÒV¼zÈ!ÈXÓ?VÓWÐWr?   )NNNNNNr   r)   NTNT)r0   úOptional[DATA_T]r1   úOptional[Sequence[EVALUATOR_T]]r2   ú'Optional[Sequence[SUMMARY_EVALUATOR_T]]r3   r±  r4   rÞ   r5   rÞ   r6   rã   r7   rá   r8   r²  r9   rU   r:   úOptional[EXPERIMENT_T]r;   rU   r/   z'Union[TARGET_T, Runnable, EXPERIMENT_T]r<   r   rÛ   r*   )r0   r¼  r1   z+Optional[Sequence[COMPARATIVE_EVALUATOR_T]]r2   r¾  r3   r±  r4   rÞ   r5   rÞ   r6   rã   r7   rá   r8   r²  r9   rU   r:   r¿  r;   rU   r/   z(Union[tuple[EXPERIMENT_T, EXPERIMENT_T]]r<   r   rÛ   r@   )NNNNNNr   r)   NTNTrR  ) r0   r¼  r1   zIOptional[Union[Sequence[EVALUATOR_T], Sequence[COMPARATIVE_EVALUATOR_T]]]r2   r¾  r3   r±  r4   rÞ   r5   rÞ   r6   rã   r7   rá   r8   r²  r9   rU   r:   r¿  r;   rU   rT   rV  r/   zJUnion[TARGET_T, Runnable, EXPERIMENT_T, tuple[EXPERIMENT_T, EXPERIMENT_T]]r<   r   rÛ   z6Union[ExperimentResults, ComparativeExperimentResults])NNNr   NFT)r1   r½  r2   r¾  r3   r±  r6   rã   r8   r²  rp   rU   r9   rU   r:   ú,Union[str, uuid.UUID, schemas.TracerSession]rÛ   r*   )NNé   NNFF)r1   z!Sequence[COMPARATIVE_EVALUATOR_T]r4   rÞ   r5   rÞ   r6   rá   r8   r²  r3   r±  rp   rU   r  rU   rë   z!tuple[EXPERIMENT_T, EXPERIMENT_T]rÛ   r@   )rë   z3tuple[schemas.TracerSession, schemas.TracerSession]r
  zschemas.ComparativeExperimentrÛ   rÞ   )r¬   rÞ   rÛ   rà   )r/   ú0Union[TARGET_T, Iterable[schemas.Run], Runnable]rÛ   rU   )NNNNNNr)   NTNTrR  )r0   rW  r1   r½  r2   r¾  r3   r±  r4   rÞ   r5   rÞ   r6   rã   r7   rá   r8   r²  r9   rU   r:   ú6Optional[Union[schemas.TracerSession, str, uuid.UUID]]r;   rU   rT   rV  r/   rÂ  rÛ   r*   )r>  rN   rÛ   rU   )rw   ÚEXPERIMENT_Tr8   úlangsmith.ClientrÛ   r³  )F)rw   rÀ  r8   rÅ  rp   rU   rÛ   r  )r8   rÅ  rw   r³  rÛ   z dict[uuid.UUID, schemas.Example])rÛ   zCallable[[IT], IT])r1   z8Sequence[Union[EVALUATOR_T, RunEvaluator, AEVALUATOR_T]]rÛ   r[  )r1   rY  rÛ   zlist[SUMMARY_EVALUATOR_T])FrR  )r  zrh.SupportsLangsmithExtrar   r~   rš   rN   r3   rA  r8   rÅ  r;   rU   rL  rU   rT   rV  rÛ   rp  )r1   zBOptional[Sequence[Union[EVALUATOR_T, RunEvaluator, AEVALUATOR_T]]]rÛ   ú	list[str])r0   z-Union[DATA_T, AsyncIterable[schemas.Example]]r8   rÅ  rÛ   rã   )r0   rW  r8   rÅ  rL  rU   rÛ   rX  )rÉ  rA  rÛ   rA  )r/   z=TARGET_T | rh.SupportsLangsmithExtra[[dict], dict] | RunnablerÛ   z'rh.SupportsLangsmithExtra[[dict], dict])r/   r   r1   úOptional[Sequence]rÛ   rU   )r1   rÇ  rÛ   rá   )r-  r   rÛ   rU   )r/   r   rÛ   rU   )r/   r   rÛ   rÆ  )r:   rÃ  rx   rS  r8   rÅ  rÛ   zStuple[Optional[Union[schemas.TracerSession, str]], Optional[Iterable[schemas.Run]]]rÚ   râ   )rÀ   zlist[ExperimentResultRow]r•   rã   rÈ   rã   )rÛ   zOptional[type])r»  r   rÛ   rU   )~rå   Ú
__future__r   r  Úconcurrent.futuresr/  r  rl  rš  rÏ  rÙ  ÚloggingrT  r‹   rü   rŽ   rO   Úcollections.abcr   r   r   r   r   r	   r
   Úcontextvarsr   Útypingr   r   r   r   r   r   r   r   Útyping_extensionsr   r   Ú	langsmithr   r�  r   rf   r   rq   r   r   r  Úlangsmith._internalr   Ú#langsmith._internal._beta_decoratorr   r   Úlangsmith.evaluation.evaluatorr   r   r    r!   r"   r#   r$   r%   r&   rÌ   r±  r¸  r(   rä   Ú	getLoggerr‚   r\   rA  rY  rN   rP   rÑ  r‡  rW  rO  r_   ÚAEVALUATOR_TrQ   rÄ  r>   r^   r|   r*   rd   rc   r@   r  r  rI  rh   rd  rs   rt   ru   r  r·   r†  r‡  rÙ   rþ  r  rp  r  r�  rQ  rº  rØ  rŒ  r  rR  r“  r–  ró  r}  rP  r�  rÉ   r°  Ú	lru_cacher¹  rH  r-   r?   r=   Ú<module>rÖ     sX	  ðÙ å "ã Ý Û Û Û 	Û Û Û Û Û Û Û ÷÷ ñ õ %÷	÷ 	ó 	÷ 2ã Ý #Ý 'Ý %Ý Ý 'Ý 3÷÷
÷ 
õ 
ñ ÛÝ1à—‘�Ià€IØ	ˆ×	Ñ	˜8Ó	$€à�˜4˜& $˜,Ñ'¨°4¸°,ÀÐ2DÑ)EÐEÑF€à	ˆs�D—I‘I˜x¨¯©Ñ8¸'¿/¹/ÐIÑ	J€ð ØØØ	�‰�h˜wŸ™Ñ/Ð0ØÐÐ 1Ð1Ñ2ð	4ñð ˆS�%˜Ð/Ð1AÐAÑBÐBÑCðEñ€ð ØØ	�‰�h˜wŸ™Ñ/Ð0Ø�%Ð(Ð*;Ð;Ñ<Ñ=ð	?ñð ñ€ð �S˜$Ÿ)™) W×%:Ñ%:Ð:Ñ;€ð 
ð "Ø26ØBFØ#Ø'+Ø!%Ø%&ØØ)-ØØ)-Øðð ðð 0ð	ð
 @ðð ðð %ðð ðð #ðð ðð 'ðð ðð 'ðð ðØ3ðð ðð  ò!ó 
ðð& 
ð "Ø>BØBFØ#Ø'+Ø!%Ø%&ØØ)-ØØ)-Øð'ð ð'ð <ð	'ð
 @ð'ð ð'ð %ð'ð ð'ð #ð'ð ð'ð 'ð'ð ð'ð 'ð'ð ð'Ø4ð'ð ð'ð  "ò!'ó 
ð'ð, "ð 	ØBFØ#Ø'+Ø!%Ø%&ØØ)-ØØ)-ØØ/4ð#V
ð ðV
ðð	V
ð @ðV
ð ðV
ð %ðV
ð ðV
ð #ðV
ð ðV
ð 'ðV
ð ðV
ð 'ðV
ð  ð!V
ð" -ð#V
ØVðV
ð$ ð%V
ð& <ó'V
ðx 37ØBFØ#Ø%&Ø)-ØØðmð 0ðmð @ð	mð
 ðmð #ðmð 'ðmð ðmð ðmØ<ðmð ómô`*˜)ô *÷z)ñ z)ð@ #Øˆg�k‰kÑ˜H W§_¡_Ñ5Ð6Ø	ØÐ(¨$Ð.Ñ/Ø�%Ð2°DÐ8Ñ9Ñ:ð	<ñðñÐ ð (,Ø!%ØØ)-Ø#ØØ!ðNð 2ðNð %ð	Nð
 ðNð ðNð 'ðNð ðNð ðNð ðNØ2ðNð "óN÷b
,ñ ,ðdØDðà9ðð óð"
Ø!ð
à	ó
ó>ð 37ØBFØ#Ø'+Ø!%Ø%)ØØ)-ØØIMØØ/4ð9ð ð9ð 0ð	9ð
 @ð9ð ð9ð %ð9ð ð9ð #ð9ð ð9ð 'ð9ð ð9ð Gð9ð ð9ð -ð9Ø<ð9ð  ó!9óxð9Øð9Ø#3ð9àó9ð ð*Ø9ð*àð*ð ð*ð ó	*ðZ	Øð	Ø'<ð	à%ó	ñ ˆTƒ]€óñ ˆTÐ2Ô3€÷sñ sôll9Ð0ô l9ð^	ØHð	àó	ðØ-ðàóô:�iô ð !&Ø/4ð0LØ!ð0Làð0Lð ð0Lð ð	0Lð
 ð0Lð ð0Lð ð0Lð -ð0Lð ó0LófðØRðàóð.!Ø
7ð!ð ð!ð ó	!ðP !&ñ	Ø
ðð ðð ð	ð
 óó4>ðØIðà,óó8óCóCó5ó
&"ðRØFðà
)ðð ððó	óBð ØðTØ&ðTàðTð 
óTð& ØðØ&ðàðð 
óð> €×Ñ˜QÔòó  ðôXr?   