$h)SSKJr SSKJrJrJrJrJrJr SSK J r SSK J r SSK JrJr \(a SSKJr SSK Jr "SS \5r"S S 5rg ) ) annotations) TYPE_CHECKINGAnyCallableOptional TypedDictUnion)DynamicRunEvaluator) traceable)ExampleRunStringEvaluator) RunEvaluatorc<\rSrSr%SrS\S'S\S'S\S'S rg ) SingleEvaluatorInputz!The input to a `StringEvaluator`.str predictionz Optional[Any] referencez Optional[str]inputN)__name__ __module__ __qualname____firstlineno____doc____annotations____static_attributes__rd/var/www/html/shao/venv/lib/python3.13/site-packages/langsmith/evaluation/integrations/_langchain.pyrrs+O  r rcD\rSrSrSrSSS.SSjjrS SjrSrg) LangChainStringEvaluatora"A class for wrapping a LangChain StringEvaluator. Requires the `langchain` package to be installed. Attributes: evaluator (StringEvaluator): The underlying StringEvaluator OR the name of the evaluator to load. Methods: as_run_evaluator() -> RunEvaluator: Convert the LangChainStringEvaluator to a RunEvaluator. Examples: Creating a simple LangChainStringEvaluator: >>> evaluator = LangChainStringEvaluator("exact_match") Converting a LangChainStringEvaluator to a RunEvaluator: >>> from langsmith.evaluation import LangChainStringEvaluator >>> from langchain_openai import ChatOpenAI >>> evaluator = LangChainStringEvaluator( ... "criteria", ... config={ ... "criteria": { ... "usefulness": "The prediction is useful if" ... " it is correct and/or asks a useful followup question." ... }, ... "llm": ChatOpenAI(model="gpt-4o"), ... }, ... ) >>> run_evaluator = evaluator.as_run_evaluator() >>> run_evaluator # doctest: +ELLIPSIS Customizing the LLM model used by the evaluator: >>> from langsmith.evaluation import LangChainStringEvaluator >>> from langchain_anthropic import ChatAnthropic >>> evaluator = LangChainStringEvaluator( ... "criteria", ... config={ ... "criteria": { ... "usefulness": "The prediction is useful if" ... " it is correct and/or asks a useful followup question." ... }, ... "llm": ChatAnthropic(model="claude-3-opus-20240229"), ... }, ... ) >>> run_evaluator = evaluator.as_run_evaluator() >>> run_evaluator # doctest: +ELLIPSIS Using the `evaluate` API with different evaluators: >>> def prepare_data(run: Run, example: Example): ... # Convert the evaluation data into the format expected by the evaluator ... # Only required for datasets with multiple inputs/output keys ... return { ... "prediction": run.outputs["prediction"], ... "reference": example.outputs["answer"], ... "input": str(example.inputs), ... } >>> import re >>> from langchain_anthropic import ChatAnthropic >>> import langsmith >>> from langsmith.evaluation import LangChainStringEvaluator, evaluate >>> criteria_evaluator = LangChainStringEvaluator( ... "criteria", ... config={ ... "criteria": { ... "usefulness": "The prediction is useful if it is correct" ... " and/or asks a useful followup question." ... }, ... "llm": ChatAnthropic(model="claude-3-opus-20240229"), ... }, ... prepare_data=prepare_data, ... ) >>> embedding_evaluator = LangChainStringEvaluator("embedding_distance") >>> exact_match_evaluator = LangChainStringEvaluator("exact_match") >>> regex_match_evaluator = LangChainStringEvaluator( ... "regex_match", config={"flags": re.IGNORECASE}, prepare_data=prepare_data ... ) >>> scoring_evaluator = LangChainStringEvaluator( ... "labeled_score_string", ... config={ ... "criteria": { ... "accuracy": "Score 1: Completely inaccurate\nScore 5: Somewhat accurate\nScore 10: Completely accurate" ... }, ... "normalize_by": 10, ... "llm": ChatAnthropic(model="claude-3-opus-20240229"), ... }, ... prepare_data=prepare_data, ... ) >>> string_distance_evaluator = LangChainStringEvaluator( ... "string_distance", ... config={"distance_metric": "levenshtein"}, ... prepare_data=prepare_data, ... ) >>> from langsmith import Client >>> client = Client() >>> results = evaluate( ... lambda inputs: {"prediction": "foo"}, ... data=client.list_examples(dataset_name="Evaluate Examples", limit=1), ... evaluators=[ ... embedding_evaluator, ... criteria_evaluator, ... exact_match_evaluator, ... regex_match_evaluator, ... scoring_evaluator, ... string_distance_evaluator, ... ], ... ) # doctest: +ELLIPSIS View the evaluation results for experiment:... N)config prepare_datacSSKJn [X5(aXlOJ[U[5(aSSKJn U"U40U=(d 0D6UlO[S[U535eX0l g)aInitialize a LangChainStringEvaluator. See: https://api.python.langchain.com/en/latest/evaluation/langchain.evaluation.schema.StringEvaluator.html#langchain-evaluation-schema-stringevaluator Args: evaluator (StringEvaluator): The underlying StringEvaluator. rr)load_evaluatorzUnsupported evaluator type: N) langchain.evaluation.schemar isinstance evaluatorrlangchain.evaluationr(NotImplementedErrortype _prepare_data)selfr+r%r&rr(s r!__init__!LangChainStringEvaluator.__init__s[ @ i 1 1&N  3 ' ' ;+IH&,BHDN%(DT)_DU&VW W)r c^^^TRR(aSOSnTRR(aSOSnSUUS3m[S S UU4Sjjj5m[TRRS9S S UU4Sjjj5n[TRRS9S S UU4S jjj5n[ X45$) zConvert the LangChainStringEvaluator to a RunEvaluator. This is the object used in the LangSmith `evaluate` API. Returns: RunEvaluator: The converted RunEvaluator. z) "input": example.inputs['input'],z0 "reference": example.outputs['expected']z] def prepare_data(run, example): return { "prediction": run.outputs['my_output'],zL } evaluator = LangChainStringEvaluator(..., prepare_data=prepare_data) c >UR(a4[UR5S:a[STRST35eTRR(aLU(aEUR(a4[UR5S:a[STRST35eTRR (aLU(aEUR (a4[UR 5S:a[STRST35e[[[URR555TRR(aDU(a=UR(a,[[URR555OSTRR (aFU(a?UR (a.[[UR R555S9$SS9$)Nz Evaluator z{ only supports a single prediction key. Please ensure that the run has a single output. Or initialize with a prepare_data: z nly supports a single reference key. Please ensure that the example has a single output. Or create a custom evaluator yourself: zy only supports a single input key. Please ensure that the example has a single input. Or initialize with a prepare_data: )rrr) outputslen ValueErrorr+requires_referencerequires_inputinputsrnextitervalues)runexamplecustomization_error_strr0s r!prepare_evaluator_inputsKLangChainStringEvaluator.as_run_evaluator..prepare_evaluator_inputss{{s3;;/!3   01<//111OO(1,   01@//1--NN'!+   01<//1(S[[%7%7%9 :;99##OO goo44678 55'gnngnn33567  r )namec>TRcT"X5OTRX5nTRR"S0UD6nSTRR0UE$Nkeyr)r/r+evaluate_stringsevaluation_namer@rA eval_inputsresultsrCr0s r!evaluate;LangChainStringEvaluator.as_run_evaluator..evaluatesa%%-)6''5  nn55D DG4>>99EWE Er c># TRcT"X5OTRX5nTRR"S0UD6IShvN nSTRR0UE$N7frG)r/r+aevaluate_stringsrJrKs r! aevaluate.aevaluatesm%%-)6''5  !NN<<K{KKG4>>99EWE ELsAA* A( A*)N)r@r rAOptional[Example]returnr)r@r rArTrUdict)r+r;r:r rJr )r0 input_str reference_strrNrRrBrCs` @@r!as_run_evaluator)LangChainStringEvaluator.as_run_evaluators~~,, ; ~~00 B  '01>ykJ# 373 3 03  !3 3  3 j 66 7 F F 8 F 66 7 F F 8 F#877r )r/r+)r+zUnion[StringEvaluator, str]r%zOptional[dict]r&zBOptional[Callable[[Run, Optional[Example]], SingleEvaluatorInput]])rUr)rrrrrr1rYrrr r!r#r#sCqn"& *.* *  *:f8 f8r r#N) __future__rtypingrrrrrr langsmith.evaluation.evaluatorr langsmith.run_helpersr langsmith.schemasr r r)rrrr#rr r!r`s8"KK>+*;;9w8w8r