~tj,;dZddlmZddlZddlZddlmZmZmZm Z m Z m Z m Z ddl mZddl mZddl mZddl mZer dd lmZdd lmZGd d ZeZGd dZGddZeZdgZdS)a<Make approximate assertions as "expectations" on test results. This module is designed to be used within test cases decorated with the `@pytest.mark.decorator` decorator It allows you to log scores about a test case and optionally make assertions that log as "expectation" feedback to LangSmith. Example: ```python import pytest from langsmith import expect @pytest.mark.langsmith def test_output_semantically_close(): response = oai_client.chat.completions.create( model="gpt-3.5-turbo", messages=[ {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Say hello!"}, ], ) response_txt = response.choices[0].message.content # Intended usage expect.embedding_distance( prediction=response_txt, reference="Hello!", ).to_be_less_than(0.9) # Score the test case matcher = expect.edit_distance( prediction=response_txt, reference="Hello!", ) # Apply an assertion and log 'expectation' feedback to LangSmith matcher.to_be_less_than(1) # You can also directly make assertions on values directly expect.value(response_txt).to_contain("Hello!") # Or using a custom check expect.value(response_txt).against(lambda x: "Hello" in x) # You can even use this for basic metric logging within tests expect.score(0.8) expect.score(0.7, key="similarity").to_be_greater_than(0.7) ``` ) annotationsN) TYPE_CHECKINGAnyCallableLiteralOptionalUnionoverloadclient) run_helpers) run_trees)utils)EditDistanceConfig)EmbeddingConfigc"eZdZdZddZd dZdS) _NULL_SENTRYzA sentinel singleton class used to distinguish omitted keyword arguments from those passed in with the value None (which may have different behavior). returnLiteral[False]cdS)NFselfs O/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/langsmith/_expect.py__bool__z_NULL_SENTRY.__bool__QsustrcdS)N NOT_GIVENrrs r__repr__z_NULL_SENTRY.__repr__Ts{rN)rr)rr)__name__ __module__ __qualname____doc__rr rrrrrLsFrrcteZdZdZ d'd(d Zd)d*dZd+dZd,dZd,dZd-dZ d.d/d Z d,d!Z d0d"Z d1d#Z d2d&ZdS)3_Matcherz4A class for making assertions on expectation values.Nr Optional[ls_client.Client]keyrvaluer _executor,Optional[ls_utils.ContextThreadPoolExecutor]run_id Optional[str]c||_||_||_|ptjd|_t j|_|jr |jj n||_ dS)N max_workers) _clientr(r)ls_utilsContextThreadPoolExecutorr*rhget_current_run_tree_rttrace_id_run_id)rr r(r)r*r,s r__init__z_Matcher.__init__^s`  "Wh&HUV&W&W&W*,,,0H@tx((& rscoreintmessagerNonec tjsw|jstj|_|j|jj|jd|||j r |j j nd|j r |j j nddSdS)N expectation)r,r(r;comment session_id start_time) r3test_tracking_is_disabledr2rtget_cached_clientr*submitcreate_feedbackr9r7rBrC)rr;r=s r_submit_feedbackz_Matcher._submit_feedbackms133 < 6!355 N ! ! ,|!26(D48..26(D48.. "       r conditionbool method_namec |s J||dd|jd|dS#t$r+}|dt||dd}~wwxYw)Nz Success: .)r=r)rIr(AssertionErrorrepr)rrJr=rLes r_assertz_Matcher._assert{s  % %g % %9  ! !!-Q-Q-QK-Q-Q ! R R R R R     ! !!T!WW - - -  s.2 A'&A""A'floatc l||j|kd|jd|d|jddS)zAssert that the expectation value is less than the given value. Args: value: The value to compare against. Raises: AssertionError: If the expectation value is not less than the given value. Expected z to be less than , but got to_be_less_thanNrSr)r(rr)s rrXz_Matcher.to_be_less_thansP J  P P P5 P PDJ P P      rc l||j|kd|jd|d|jddS)aAssert that the expectation value is greater than the given value. Args: value: The value to compare against. Raises: AssertionError: If the expectation value is not greater than the given value. rVz to be greater than rWto_be_greater_thanNrYrZs rr\z_Matcher.to_be_greater_thansP J  S S Se S Stz S S      r min_value max_valuec |||jcxko|kncd|jd|d|d|jddS)a4Assert that the expectation value is between the given min and max values. Args: min_value: The minimum value (exclusive). max_value: The maximum value (exclusive). Raises: AssertionError: If the expectation value is not between the min and max. rVz to be between z and rW to_be_betweenNrY)rr]r^s rr`z_Matcher.to_be_betweens  . . . .Y . . . . % % % % % % %  % %       r precisionc |t|j|t||kd|jd|d|jddS)akAssert that the expectation value is approximately equal to the given value. Args: value: The value to compare against. precision: The number of decimal places to round to for comparison. Raises: AssertionError: If the rounded expectation value does not equal the rounded given value. rVz to be approximately rWto_be_approximatelyN)rSroundr)r()rr)rbs rrdz_Matcher.to_be_approximatelysd $*i ( (E%,C,C C T T Tu T T T T !     rc l||j|kd|jd|d|jddS)aAssert that the expectation value equals the given value. Args: value: The value to compare against. Raises: AssertionError: If the expectation value does not exactly equal the given value. rVz to be equal to rWto_equalNrYrZs rrgz_Matcher.to_equalsP J%  O O O% O O4: O O      rcb||jdud|jd|jddS)zAssert that the expectation value is `None`. Raises: AssertionError: If the expectation value is not `None`. NrVz to be None, but got to_be_nonerYrs rriz_Matcher.to_be_nonesH J$  C C Ctz C C      rcZ|||jvd|jd|dddS)zAssert that the expectation value contains the given value. Args: value: The value to check for containment. Raises: AssertionError: If the expectation value does not contain the given value. rVz to contain z, but it does not to_containNrYrZs rrkz_Matcher.to_containsI TZ  F F Fe F F F      rfuncrctj|}|||jd|d|jddS)zAssert the expectation value against a custom function. Args: func: A custom function that takes the expectation value as input. Raises: AssertionError: If the custom function returns False. z Assertion z failed for againstN)inspect signaturerSr)r()rrlfunc_signatures rrnz_Matcher.againsts[!*400 D   ? ? ?TX ? ?      r)NN) r r'r(rr)rr*r+r,r-N)r;r<r=r-rr>)rJrKr=rrLrrr>)r)rTrr>)r]rTr^rTrr>)ra)r)rTrbr<rr>)rr>)r)rrr>)rlrrr>)r!r"r#r$r:rIrSrXr\r`rdrgrirkrnrrrr&r&[s>>CG $ A A A A A                  "     "                   rr&ceZdZdZddd'dZddd(dZddd)dZd*dZddddd+dZe d*dZ e d,d Z e dfd-d#Z d.d&Z dS)/_Expectz1A class for setting expectations on test results.Nr r r'c||_tjd|_t j|jjddS)Nr/r0T)wait)r2r3r4executoratexitregistershutdownrr s rr:z_Expect.__init__s>  :qIII  .T::::::rconfig predictionr referencer}Optional[EmbeddingConfig]rr&c &ddlm}|pi}|drdnd}||}|||}||jd}|d ||d |d |jd t |jd ||j S)aCompute the embedding distance between the prediction and reference. This logs the embedding distance to LangSmith and returns a `_Matcher` instance for making assertions on the distance value. By default, this uses the OpenAI API for computing embeddings. Args: prediction: The predicted string to compare. reference: The reference string to compare against. config: Optional configuration for the embedding distance evaluator. Supported options: - `encoder`: A custom encoder function to encode the list of input strings to embeddings. Defaults to the OpenAI API. - `metric`: The distance metric to use for comparison. Supported values: `'cosine'`, `'euclidean'`, `'manhattan'`, `'chebyshev'`, `'hamming'`. Returns: A `_Matcher` instance for the embedding distance value. Example: ```python expect.embedding_distance( prediction="hello", reference="hi", ).to_be_less_than(1.0) ``` r)EmbeddingDistanceencodercustomopenair|r~r)rmetricembedding_distanceUsing z , Metric: r; source_inforAr*) 'langsmith._internal._embedding_distancergetevaluatedistancerIr&r2rw) rr~rr}r encoder_func evaluatorr;src_infos rrz_Expect.embedding_distance sT NMMMMM2#)::i#8#8Fxxh %%V444 ""jI"NN+y7IJJ  'PLPPI)>JJ0$77  L/// ""jI"NN$9==  'DFDDDD      L  m     rr)rc<t|jd||jS)aDCreate a `_Matcher` instance for making assertions on the given value. Args: value: The value to make assertions on. Returns: A `_Matcher` instance for the given value. Example: ```python expect.value(10).to_be_less_than(20) ``` r)r)r&r2rwrZs rr)z _Expect.values gu NNNNrr;)r( source_run_idrAUnion[float, int, bool]r(rOptional[ls_client.ID_TYPE]rAr-cv|||ddi||dt|j|||jS)aLog a numeric score to LangSmith. Args: score: The score value to log. key: The key to use for logging the score. Defaults to `'score'`. Example: ```python expect.score(0.8) # doctest: +ELLIPSIS expect.score(0.8, key="similarity").to_be_greater_than(0.7) ``` methodz expect.score)r;rrrAr)rIr&r2rw)rr;r(rrAs rr;z _Expect.scoresW,   (.9!."       c5DMJJJJrcdSrrrrZs r__call__z_Expect.__call__s363rls_client.ClientcdSrrrr{s rrz_Expect.__call__sCF3r Optional[Any]Union[_Expect, _Matcher]cbt|}|tur||S|S)Nr )rtrr))rr)r expecteds rrz_Expect.__call__s6 &)))  ! !>>%(( (rresultsdictctj}|r|jnd}tjsA|jst j|_|jj |jj f||d|dSdS)N)r,r() r5r6r8r3rDr2rErFrwrGrH)rr(r current_runr,s rrIz_Expect._submit_feedbacks-// )4>%%$133 < 6!355 DM  , 5;  HO       r)r r')r~rrrr}rrr&)r~rrrr}rrr&)r)rrr&) r;rr(rrrrAr-rr&)r rrrt)r r'r)rrr)r(rrr) r!r"r#r$r:rrr)r;r rrrIrrrrtrts.;;?C;;;;;;-1 ; ; ; ; ; ; D04 8 8 8 8 8 8 tOOOO(59!% KKKKKKF666X6 FFFXF )-1     rrtexpect) r$ __future__rrxrotypingrrrrrr r langsmithr ls_clientr r5rrErr3rrrrrrr&rtr__all__rrrrs00d#""""" *)))))''''''%%%%%%''''''HEEEEEEGGGGGG         LNN c c c c c c c c LKKKKKKKK\  *r