§
    ~ŠtjH#  ã                   óX  — d Z ddlZddlZddlZddlZddlmZ ddlmZm	Z	 ddl
mZ ddlmZ ddlmZ ddlmZ ddlmZmZmZ ddlmZ d	ed
edefd„Zdej        dedee         fd„Zedddddœdeej                 de dee          dee         de!de!dej"        fd„¦   «         Z#de dedeej                 fd„Z$ e	d¦  «        Z% e	d¦  «        Z&dee%         dee&         dee'e%e&f                  fd„Z( ed ¦  «        ed!dd"œde d#ed$ee)         dee         ddf
d%„¦   «         ¦   «         Z*dS )&zfBeta utility functions to assist in common eval workflows.

These functions may change in the future.
é    N)ÚSequence)ÚOptionalÚTypeVar)Ú
evaluation)Ú_v2_migration_utils)Ú
deprecatedÚsuppress_deprecation_warningÚ	warn_beta)ÚClientÚrun_dictÚid_mapÚreturnc                 ó4  — | d         }|                      ¦   «         D ]5\  }}|                     t          |¦  «        t          |¦  «        ¦  «        }Œ6|| d<   |                      d¦  «        r|| d                  | d<   |                      d¦  «        si | d<   | S )zøConvert the IDs in the run dictionary using the provided ID map.

    Parameters:
    - run_dict: The dictionary representing a run.
    - id_map: The dictionary mapping old IDs to new IDs.

    Returns:
    - dict: The updated run dictionary.
    Údotted_orderÚparent_run_idÚextra)ÚitemsÚreplaceÚstrÚget)r   r   ÚdoÚkÚvs        úS/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/langsmith/beta/_evals.pyÚ_convert_idsr      s    € ð 
�.Ô	!€BØ—’‘”ð (ð (‰ˆˆ1Ø�ZŠZ�˜A™œ¥ A¡¤Ñ'Ô'ˆˆØ!€Hˆ^Ñà‡|‚|�OÑ$Ô$ð FØ$*¨8°OÔ+DÔ$Eˆ�Ñ!Ø�<Š<˜Ñ Ô ð Øˆ�ÑØ€Oó    ÚrootÚrun_to_example_mapc                 ó  ‡— | g}t          j        ¦   «         }| j        |iŠg }|r¼|                     ¦   «         }|                     h d£¬¦  «        }‰                     |d         t          j        ¦   «         ¦  «        ‰|d         <   ‰|d                  |d<   ‰|d                  |d<   |j        r|                     |j        ¦  «         |                     |¦  «         |°¼ˆfd„|D ¦   «         }|| j	                 |d         d<   |S )a  Convert the root run and its child runs to a list of dictionaries.

    Parameters:
    - root: The root run to convert.
    - run_to_example_map: The dictionary mapping run IDs to example IDs.

    Returns:
    - The list of converted run dictionaries.
    >   Ú
session_idÚchild_run_idsÚparent_run_ids)ÚexcludeÚidÚtrace_idc                 ó0   •— g | ]}t          |‰¦  «        ‘ŒS © )r   )Ú.0Úrr   s     €r   ú
<listcomp>z%_convert_root_run.<locals>.<listcomp>F   s#   ø€ Ð7Ð7Ð7¨!�l˜1˜fÑ%Ô%Ð7Ð7Ð7r   r   Úreference_example_id)
ÚuuidÚuuid4r%   ÚpopÚdictr   Ú
child_runsÚextendÚappendr$   )	r   r   Úruns_r%   ÚresultsÚsrcÚsrc_dictÚresultr   s	           @r   Ú_convert_root_runr8   /   s  ø€ ð ˆF€EÝŒz‰|Œ|€HØŒm˜XÐ&€FØ€GØ
ð !Ø�iŠi‰kŒkˆØ—8’8Ð$UÐ$UÐ$U�8ÑVÔVˆØ!'§¢¨H°T¬N½D¼J¹L¼LÑ!IÔ!Iˆˆx˜Œ~ÑØ ¨¤Ô/ˆ�‰Ø% h¨zÔ&:Ô;ˆ�ÑØŒ>ð 	)Ø�LŠL˜œÑ(Ô(Ð(Ø�Š�xÑ Ô Ð ð ð !ð 8Ð7Ð7Ð7¨wÐ7Ñ7Ô7€FØ(:¸4¼7Ô(C€Fˆ1„IÐ$Ñ%Ø€Mr   F)Útest_project_nameÚclientÚload_child_runsÚinclude_outputsÚrunsÚdataset_namer9   r:   r;   r<   c                óª  ‡‡— | st          d| › �¦  «        ‚‰pt          j        ¦   «         Š‰                     |¬¦  «        }|rd„ | D ¦   «         nd}‰                     d„ | D ¦   «         |d„ | D ¦   «         |j        ¬¦  «         |s| }nˆfd„| D ¦   «         }|p"d	t          j        ¦   «         j        dd
…         › �}t          ‰ 
                    |¬¦  «        ¦  «        }	d„ |	D ¦   «         Š|	d         j        r|	d         j        n|	d         j        }
ˆfd„|D ¦   «         }‰                     ||j        d|
                     ¦   «         dœ¬¦  «        }|D ]d}|d         |d         z
  }t          j                             t          j        j        ¬¦  «        |d<   |d         |z   |d<    ‰j        di |¤d|i¤Ž Œe‰                     |j        ¦  «        }|S )aÔ  Convert the following runs to a dataset + test.

    This makes it easy to sample prod runs into a new regression testing
    workflow and compare against a candidate system.

    Internally, this function does the following:
        1. Create a dataset from the provided production run inputs.
        2. Create a new test project.
        3. Clone the production runs and re-upload against the dataset.

    Parameters:
    - runs: A sequence of runs to be executed as a test.
    - dataset_name: The name of the dataset to associate with the test runs.
    - client: An optional LangSmith client instance. If not provided, a new client will
        be created.
    - load_child_runs: Whether to load child runs when copying runs.

    Returns:
    - The project containing the cloned runs.

    Example:
    --------
    ```python
    import langsmith
    import random

    client = langsmith.Client()

    # Randomly sample 100 runs from a prod project
    runs = list(client.list_runs(project_name="My Project", execution_order=1))
    sampled_runs = random.sample(runs, min(len(runs), 100))

    runs_as_test(runs, dataset_name="Random Runs")

    # Select runs named "extractor" whose root traces received good feedback
    runs = client.list_runs(
        project_name="<your_project>",
        filter='eq(name, "extractor")',
        trace_filter='and(eq(feedback_key, "user_score"), eq(feedback_score, 1))',
    )
    runs_as_test(runs, dataset_name="Extraction Good")
    ```
    z1Expected a non-empty sequence of runs. Received: )r>   c                 ó   — g | ]	}|j         ‘Œ
S r'   )Úoutputs©r(   r)   s     r   r*   z(convert_runs_to_test.<locals>.<listcomp>„   s   € Ð'Ð'Ð'˜QˆqŒyÐ'Ð'Ð'r   Nc                 ó   — g | ]	}|j         ‘Œ
S r'   )ÚinputsrB   s     r   r*   z(convert_runs_to_test.<locals>.<listcomp>†   s   € Ð'Ð'Ð'˜Q�”Ð'Ð'Ð'r   c                 ó   — g | ]	}|j         ‘Œ
S r'   )r$   rB   s     r   r*   z(convert_runs_to_test.<locals>.<listcomp>ˆ   s   € Ð+Ð+Ð+ ˜œÐ+Ð+Ð+r   )rD   rA   Úsource_run_idsÚ
dataset_idc                 ó:   •— g | ]}‰                      |¦  «        ‘ŒS r'   )Ú_load_child_runs)r(   r)   r:   s     €r   r*   z(convert_runs_to_test.<locals>.<listcomp>�   s'   ø€ ÐAÐAÐA°q˜×/Ò/°Ñ2Ô2ÐAÐAÐAr   zprod-baseline-é   c                 ó(   — i | ]}|j         |j        “ŒS r'   )Úsource_run_idr$   )r(   Úes     r   ú
<dictcomp>z(convert_runs_to_test.<locals>.<dictcomp>”   s   € ÐBÐBÐB°A˜!œ/¨1¬4ÐBÐBÐBr   r   c                 ó:   •— g | ]}t          |‰¦  «        D ]}|‘ŒŒS r'   )r8   )r(   Úroot_runr   r   s      €r   r*   z(convert_runs_to_test.<locals>.<listcomp>™   sK   ø€ ð ð ð àÝ)¨(Ð4FÑGÔGðð ð ð 	ðð ð ð r   zprod-baseline)ÚwhichÚdataset_version)Úproject_nameÚreference_dataset_idÚmetadataÚend_timeÚ
start_time)ÚtzrS   r'   )Ú
ValueErrorÚrtÚget_cached_clientÚcreate_datasetÚcreate_examplesr$   r,   r-   ÚhexÚlistÚlist_examplesÚmodified_atÚ
created_atÚcreate_projectÚ	isoformatÚdatetimeÚnowÚtimezoneÚutcÚ
create_runÚupdate_project)r=   r>   r9   r:   r;   r<   ÚdsrA   Úruns_to_copyÚexamplesrR   Ú	to_createÚprojectÚnew_runÚlatencyÚ_r   s      `            @r   Úconvert_runs_to_testrs   K   sp  øø€ ðj ð YÝÐWÈtÐWÐWÑXÔXÐXØÐ-•rÔ+Ñ-Ô-€FØ	×	Ò	¨LÐ	Ñ	9Ô	9€BØ+:ÐDÐ'Ð' $Ð'Ñ'Ô'Ð'À€GØ
×ÒØ'Ð' $Ð'Ñ'Ô'ØØ+Ð+ dÐ+Ñ+Ô+Ø”5ð	 ñ ô ð ð ð BØˆˆàAÐAÐAÐA¸DÐAÑAÔAˆà)ÐTÐ-T½d¼j¹l¼lÔ>NÈrÐPQÈrÔ>RÐ-TÐ-TÐå�F×(Ò(°lÐ(ÑCÔCÑDÔD€HØBÐB¸ÐBÑBÔBÐà#+¨A¤;Ô#:ÐVˆ�ŒÔÐÀÈÄÔ@Vð ðð ð ð à$ðñ ô €Ið ×#Ò#Ø&ØœUà$Ø.×8Ò8Ñ:Ô:ð
ð 
ð $ñ ô €Gð ð Eð EˆØ˜*Ô%¨°Ô(=Ñ=ˆÝ (Ô 1× 5Ò 5½Ô9JÔ9NÐ 5Ñ OÔ Oˆ�ÑØ% lÔ3°gÑ=ˆ�
ÑØˆÔÐDÐD˜GÐDÐDÐ2CÐDÐDÐDÐDÐDà×ÒØŒ
ñ	ô 	€Að €Nr   rS   c                 óN  — t          j        |j        j        ¦  «        }|t           j        j        k    rt          j        | |¦  «        S t          ¦   «         5  |                     | ¬¦  «        }d d d ¦  «         n# 1 swxY w Y   t          j
        t          ¦  «        }g }i }|D ]I}|j        �!||j                                      |¦  «         n|                     |¦  «         |||j        <   ŒJ|                     ¦   «         D ]"\  }}	t!          |	d„ ¬¦  «        ||         _        Œ#|S )N)rS   c                 ó   — | j         S ©N)r   )r)   s    r   ú<lambda>z%_load_nested_traces.<locals>.<lambda>Ë   s   € ÀqÄ~€ r   )Úkey)r   Úget_query_backendÚinfoÚinstance_flagsÚQueryBackendÚSMITHDB_ONLYÚ_load_nested_traces_v2r	   Ú	list_runsÚcollectionsÚdefaultdictr_   r   r2   r$   r   Úsortedr0   )
rS   r:   Úbackendr=   Útreemapr4   Úall_runsÚrunÚrun_idr0   s
             r   Ú_load_nested_tracesrˆ   ´   st  € õ "Ô3°F´KÔ4NÑOÔO€GØÕ%Ô2Ô?Ò?Ð?Ý"Ô9¸,ÈÑOÔOÐOõ 
&Ñ	'Ô	'ð ;ð ;Ø×Ò¨\ÐÑ:Ô:ˆð;ð ;ð ;ñ ;ô ;ð ;ð ;ð ;ð ;ð ;ð ;øøøð ;ð ;ð ;ð ;õ 	Ô¥Ñ%Ô%ð ð €GØ€HØð ð ˆØÔÐ(Ø�CÔ%Ô&×-Ò-¨cÑ2Ô2Ð2Ð2à�NŠN˜3ÑÔÐØˆ�”ÑÐØ%Ÿmšm™oœoð Wð WÑˆ�
Ý&,¨ZÐ=UÐ=UÐ&VÑ&VÔ&Vˆ�ÔÔ#Ð#Ø€Ns   ÁA:Á:A>ÂA>ÚTÚUÚlist1Úlist2c                 óF   — t          t          j        | |¦  «        ¦  «        S rv   )r_   Ú	itertoolsÚproduct)r‹   rŒ   s     r   Ú_outer_productr�   Ó   s   € Ý•	Ô! %¨Ñ/Ô/Ñ0Ô0Ð0r   z´compute_test_metrics() is deprecated and will be removed after Jan 31, 2027. There is no replacement: run the evaluators yourself and log the results with client.create_feedback().é
   )Úmax_concurrencyr:   Ú
evaluatorsr’   c          
      ó  — ddl m} g }|D ]‡}t          |t          j        ¦  «        r|                     |¦  «         Œ2t          |¦  «        r(|                     t          j        |¦  «        ¦  «         Œit          dt          |¦  «        › �¦  «        ‚|pt          j        ¦   «         }t          | |¦  «        } ||¬¦  «        5 } |j        |j        gt          t!          ||¦  «        Ž ¢R Ž }	ddd¦  «         n# 1 swxY w Y   |	D ]}
ŒdS )aê  Compute test metrics for a given test name using a list of evaluators.

    .. admonition:: Deprecated

        There is no replacement: run the evaluators yourself and log the results
        with :meth:`langsmith.Client.create_feedback`.
        Will be removed after Jan 31, 2027.

    Args:
        project_name (str): The name of the test project to evaluate.
        evaluators (list): A list of evaluators to compute metrics with.
        max_concurrency (Optional[int], optional): The maximum number of concurrent
            evaluations. Defaults to 10.
        client (Optional[Client], optional): The client to use for evaluations.
            Defaults to None.

    Returns:
        None: This function does not return any value.
    r   )ÚContextThreadPoolExecutorz5Evaluation not yet implemented for evaluator of type )Úmax_workersN)Ú	langsmithr•   Ú
isinstanceÚls_evalÚRunEvaluatorr2   ÚcallableÚrun_evaluatorÚNotImplementedErrorÚtyperZ   r[   rˆ   ÚmapÚevaluate_runÚzipr�   )rS   r“   r’   r:   r•   Úevaluators_ÚfuncÚtracesÚexecutorr4   rr   s              r   Úcompute_test_metricsr¦   ×   s‡  € ð@ 4Ð3Ð3Ð3Ð3Ð3à.0€KØð ð ˆÝ�d�GÔ0Ñ1Ô1ð 	Ø×Ò˜tÑ$Ô$Ð$Ð$Ý�d‰^Œ^ð 	Ø×Ò�wÔ4°TÑ:Ô:Ñ;Ô;Ð;Ð;å%ØTÍÈTÉ
Ì
ÐTÐTñô ð ð Ð-•rÔ+Ñ-Ô-€FÝ  ¨vÑ6Ô6€FØ	"Ð	"¨Ð	?Ñ	?Ô	?ð 
À8Ø�(”,ØÔð
Ý"%¥~°f¸kÑ'JÔ'JÐ"Kð
ð 
ð 
ˆð
ð 
ð 
ñ 
ô 
ð 
ð 
ð 
ð 
ð 
ð 
øøøð 
ð 
ð 
ð 
ð ð ð ˆØðð s   Ã)C9Ã9C=Ä C=)+Ú__doc__r€   re   rŽ   r,   Úcollections.abcr   Útypingr   r   Úlangsmith.run_treesÚ	run_treesrZ   Úlangsmith.schemasÚschemasÚ
ls_schemasr—   r   r™   Úlangsmith._internalr   Ú#langsmith._internal._beta_decoratorr   r	   r
   Úlangsmith.clientr   r/   r   ÚRunr_   r8   r   ÚboolÚTracerSessionrs   rˆ   r‰   rŠ   Útupler�   Úintr¦   r'   r   r   ú<module>r·      s  ððð ð
 Ð Ð Ð Ø €€€Ø Ð Ð Ð Ø €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø $Ð $Ð $Ð $Ð $Ð $Ð $Ð $à  Ð  Ð  Ð  Ð  Ð  Ø &Ð &Ð &Ð &Ð &Ð &Ø +Ð +Ð +Ð +Ð +Ð +Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3ðð ð ð ð ð ð ð ð ð ð
 $Ð #Ð #Ð #Ð #Ð #ð˜4ð ¨ð °$ð ð ð ð ð,˜JœNð Àð ÈÈdÌð ð ð ð ð8 ð
 (,Ø#Ø!Ø!ðeð eð eØ
�:”>Ô
"ðeð ðeð   ”}ð	eð
 �VÔðeð ðeð ðeð Ôðeð eð eñ „ðeðP cð °6ð ¸dÀ:Ä>Ô>Rð ð ð ð ð6 €GˆC�L„L€Ø€GˆC�L„L€ð1˜$˜qœ'ð 1¨$¨q¬'ð 1°d¸5ÀÀAÀ¼;Ô6Gð 1ð 1ð 1ð 1ð €ð%ñô ð
 ð
 &(Ø#ð-ð -ð -Øð-ð ð-ð ˜c”]ð	-ð
 �VÔð-ð 
ð-ð -ð -ñ „ñô ð-ð -ð -r   