a ag7*@sdZddlmZddlZddlZddlmZmZmZm Z m Z ddl m Z ddl mZddlmZddlmZdd lmZdd lmZmZmZdd lmZmZdd lmZd ddddZd ddddZGdddeeeZ GdddeeeZ!Gddde!Z"dS)z-LLM Chains for evaluating question answering.) annotationsN)AnyListOptionalSequenceTuple) Callbacks)BaseLanguageModel)PromptTemplate) ConfigDict)LLMChain)CONTEXT_PROMPT COT_PROMPTPROMPT) LLMEvalChainStringEvaluator)RUN_KEYstrzOptional[Tuple[str, int]])textreturncCstd|tj}|rD|ddkr.dS|ddkrDdSz|dt ddt j }|dkrzWdS|dkrWdS|d t ddt j }|dkrWdS|dkrWdSWnt yYn0dS) Nzgrade:\s*(correct|incorrect)CORRECT)rr INCORRECT)rrr) researchstrip IGNORECASEgroupuppersplit translater maketransstring punctuation IndexError)rmatchZ first_word last_wordr)p/var/www/html/cobodadashboardai.evdpl.com/venv/lib/python3.9/site-packages/langchain/evaluation/qa/eval_chain.py _get_scores4        r+dictcCs6|}t|}|dur"d\}}n|\}}|||dS)zParse the output text. Args: text (str): The output text to parse. Returns: Any: The parsed output. N)NN) reasoningvaluescore)rr+)rr-Z parsed_scoresr.r/r)r)r*_parse_string_eval_output1s  r0c @seZdZUdZdZded<eddZeddd d Z e ddd d Z e ddd dZ e ddddZ ed.dddddddZd/ddddddddddd d!Zd"d"d#d$d%Zdddd&d'dd(d(dddd"d)d*d+Zdddd&d'dd(d(dddd"d)d,d-ZdS)0 QAEvalChainz,LLM Chain for evaluating question answering.resultsr output_keyignoreextraboolrcCsdSNFr)clsr)r)r*is_lc_serializablePszQAEvalChain.is_lc_serializablecCsdS)NZ correctnessr)selfr)r)r*evaluation_nameTszQAEvalChain.evaluation_namecCsdSNTr)r=r)r)r*requires_referenceXszQAEvalChain.requires_referencecCsdSr@r)r=r)r)r*requires_input\szQAEvalChain.requires_inputNr Optional[PromptTemplate]rllmpromptkwargsrcKsH|pt}hd}|t|jkr4td|d|j|f||d|S)aLoad QA Eval Chain from LLM. Args: llm (BaseLanguageModel): the base language model to use. prompt (PromptTemplate): A prompt template containing the input_variables: 'input', 'answer' and 'result' that will be used as the prompt for evaluation. Defaults to PROMPT. **kwargs: additional keyword arguments. Returns: QAEvalChain: the loaded QA eval chain. >resultanswerqueryInput variables should be , but got rErF)rsetinput_variables ValueError)r;rErFrGexpected_input_varsr)r)r*from_llm`szQAEvalChain.from_llmrJrIrH callbackszSequence[dict]r List[dict])examples predictions question_key answer_keyprediction_keyrTrcs*fddt|D}|j||dS)5Evaluate question answering examples and predictions.cs,g|]$\}}|||dqS)rJrIrHr).0iZexamplerYrZrWrXr)r* s  z(QAEvalChain.evaluate..rS enumerateapply)r>rVrWrXrYrZrTinputsr)r`r*evaluates  zQAEvalChain.evaluater,rHrcCs&t||j}t|vr"|t|t<|SNr0r3rr>rH parsed_resultr)r)r*_prepare_outputs zQAEvalChain._prepare_outputF referenceinputrTinclude_run_info Optional[str] predictionrnrorTrprGrcKs ||||d||d}||S)aEvaluate Chain or LLM output, based on optional input and label. Args: prediction (str): the LLM or chain prediction to evaluate. reference (Optional[str], optional): the reference label to evaluate against. input (Optional[str], optional): the input to consider during evaluation callbacks (Callbacks, optional): the callbacks to use for tracing. include_run_info (bool, optional): whether to include run info in the returned results. **kwargs: additional keyword arguments, including callbacks, tags, etc. Returns: dict: The evaluation results containing the score or value. r\rTrprlr>rsrnrorTrprGrHr)r)r*_evaluate_stringss zQAEvalChain._evaluate_stringscs(|j|||d||dIdH}||S)Nr\rerTrpZacallrlrvr)r)r*_aevaluate_stringss   zQAEvalChain._aevaluate_strings)N)rJrIrH)__name__ __module__ __qualname____doc__r3__annotations__r model_config classmethodr<propertyr?rArBrRrfrlrwrzr)r)r)r*r1GsB  " 'r1c @seZdZdZeddddZeddddZedddd Ze d d Z ed d dddZ eddddZ ed0dddddddZ d1dddddddd dd!d"d#Zd$d$d%d&d'Zdddd(d)dd*d*d ddd$d+d,d-Zdddd(d)dd*d*d ddd$d+d.d/ZdS)2ContextQAEvalChainz3LLM Chain for evaluating QA w/o GT based on contextr7r8cCsdSr9r)r:r)r)r*r<sz%ContextQAEvalChain.is_lc_serializablecCsdS)z.Whether the chain requires a reference string.Tr)r=r)r)r*rAsz%ContextQAEvalChain.requires_referencecCsdS)z+Whether the chain requires an input string.Tr)r=r)r)r*rBsz!ContextQAEvalChain.requires_inputr4r5r None)rFrcCs0hd}|t|jkr,td|d|jdS)N>contextrHrJrKrL)rNrOrP)r;rFrQr)r)r*_validate_input_varssz'ContextQAEvalChain._validate_input_varsrcCsdS)NzContextual Accuracyr)r=r)r)r*r?sz"ContextQAEvalChain.evaluation_nameNr rCrrDcKs&|pt}|||f||d|S)aLoad QA Eval Chain from LLM. Args: llm (BaseLanguageModel): the base language model to use. prompt (PromptTemplate): A prompt template containing the input_variables: 'query', 'context' and 'result' that will be used as the prompt for evaluation. Defaults to PROMPT. **kwargs: additional keyword arguments. Returns: ContextQAEvalChain: the loaded QA eval chain. rM)r rr;rErFrGr)r)r*rRs zContextQAEvalChain.from_llmrJrrHrSrUr)rVrWrX context_keyrZrTrcs*fddt|D}|j||dS)r[cs,g|]$\}}|||dqS)rJrrHr)r]rrZrWrXr)r*ras  z/ContextQAEvalChain.evaluate..rSrb)r>rVrWrXrrZrTrer)rr*rf s  zContextQAEvalChain.evaluater,rgcCs&t||j}t|vr"|t|t<|Srhrirjr)r)r*rl"s z"ContextQAEvalChain._prepare_outputFrmrqrrcKs ||||d||d}||S)Nrrtrurvr)r)r*rw(s  z$ContextQAEvalChain._evaluate_stringscs(|j|||d||dIdH}||S)Nrrxryrvr)r)r*rz=s   z%ContextQAEvalChain._aevaluate_strings)N)rJrrH)r{r|r}r~rr<rrArBr rrr?rRrfrlrwrzr)r)r)r*rsD rc@sNeZdZdZeddddZeddddZedd d d dd ddZd S)CotQAEvalChainz=LLM Chain for evaluating QA using chain of thought reasoning.r7r8cCsdSr9r)r:r)r)r*r<Rsz!CotQAEvalChain.is_lc_serializablercCsdS)NzCOT Contextual Accuracyr)r=r)r)r*r?VszCotQAEvalChain.evaluation_nameNr rCrrDcKs&|pt}|||f||d|S)zLoad QA Eval Chain from LLM.rM)rrrr)r)r*rRZs zCotQAEvalChain.from_llm)N) r{r|r}r~rr<rr?rRr)r)r)r*rOsr)#r~ __future__rrr$typingrrrrrZ langchain_core.callbacks.managerrZlangchain_core.language_modelsr Zlangchain_core.promptsr Zpydanticr Zlangchain.chains.llmr Z#langchain.evaluation.qa.eval_promptr rrZlangchain.evaluation.schemarrZlangchain.schemarr+r0r1rrr)r)r)r*s$