a dg K@sJddlmZddlZddlZddlmZmZddlmZm Z m Z m Z m Z m Z mZmZmZmZddlmZddlmZddlmZddlmZmZmZmZdd lmZmZm Z dd l!m"Z"dd l#m$Z$m%Z%dd l&m'Z'dd l(m)Z)erddl*Z+e,e-Z.GdddZ/dddddZ0GdddZ1dddddZ2d?ddddddZ3dddddZ4dd d!d"d#d$Z5d%d&d'd(d)Z6d%d%d'd*d+Z7Gd,d-d-eZ8ed.e8d/Z9Gd0d1d1e e9eZ:ed2e:d/Z;Gd3d4d4e e;eZGd9d:d:e e;eeZ?Gd;d<dd>ee e;ZAdS)@) annotationsN)ABCabstractmethod) TYPE_CHECKINGAnyDictGenericListOptionalTupleTypeTypeVarUnion)Chain)LLMChain)CallbackManagerForChainRun)BasePromptTemplateChatPromptTemplateHumanMessagePromptTemplateSystemMessagePromptTemplate) BaseModel ConfigDictmodel_validator)_Embed)MetricsTrackerAverageMetricsTrackerRollingWindow)ModelRepository)VwLoggerc@s,eZdZddddZddddZeZd S) _BasedOnrvaluecCs ||_dSNrselfr r$r/var/www/html/cobodadashboardai.evdpl.com/venv/lib/python3.9/site-packages/langchain_experimental/rl_chain/base.py__init__-sz_BasedOn.__init__strreturncCs t|jSr!r'r r#r$r$r%__str__0sz_BasedOn.__str__N__name__ __module__ __qualname__r&r,__repr__r$r$r$r%r,srr)anythingr)cCst|S)z4Wrap a value to indicate that it should be based on.)rr2r$r$r%BasedOn6sr4c@s,eZdZddddZddddZeZd S) _ToSelectFromrrcCs ||_dSr!rr"r$r$r%r&=sz_ToSelectFrom.__init__r'r(cCs t|jSr!r*r+r$r$r%r,@sz_ToSelectFrom.__str__Nr-r$r$r$r%r5<sr5cCst|tstdt|S)z9Wrap a value to indicate that it should be selected from.z*ToSelectFrom must be a list to select from) isinstancelist ValueErrorr5r3r$r$r% ToSelectFromFs r9Fbool)r2keepr)cst|trtt|jdSt|tr8tt|jdSt|trTfdd|DSt|trtfdd| DSt|t r|St |dS)z4Wrap a value to indicate that it should be embedded.r;csg|]}t|dqSr<Embed).0vr<r$r% VzEmbed..csi|]\}}|t|dqSr=r>)r@krAr<r$r% XrCzEmbed..) r6r5r9r?r rr4r7dictitemsr)r2r;r$r<r%r?Ns     r?cCs t|ddS)z=Wrap a value to indicate that it should be embedded and kept.Tr<r>r3r$r$r% EmbedAndKeep^srHz'vw.TextFormatParser'r'zList['vw.Example'])parser input_strr)csfdd|dDS)z/Parse the input string into a list of examples.csg|]}|qSr$) parse_line)r@linerIr$r%rBjrCzparse_lines.. )split)rIrJr$rMr% parse_linesgsrPDict[str, Any]zTuple[Dict, Dict]inputsr)cs@fddD}|s"tdfddD}||fS)z1Get the BasedOn and ToSelectFrom from the inputs.cs&i|]}t|tr||jqSr$)r6r5r r@rDrSr$r%rEosz3get_based_on_and_to_select_from..z}No variables using 'ToSelectFrom' found in the inputs. Please include at least one variable containing a list to select from.csBi|]:}t|tr|t|jtr2|jn |jgqSr$)r6rr r7rTrUr$r%rEzs)keysr8)rSZto_select_fromZbased_onr$rUr%get_based_on_and_to_select_fromms  rWcCsN|}|D]8\}}t|ts,t|trt|jtst|j||_q|S)aPrepare the inputs for auto embedding. Go over all the inputs and if something is either wrapped in _ToSelectFrom or _BasedOn, and if their inner values are not already _Embed, then wrap them in EmbedAndKeep while retaining their _ToSelectFrom or _BasedOn status )copyrGr6r5rr rrH)rSZ next_inputsrDrAr$r$r%prepare_inputs_for_autoembeds  rYc@seZdZdZdS)Selectedz.Abstract class to represent the selected item.N)r.r/r0__doc__r$r$r$r%rZsrZ TSelected)boundc@s4eZdZUdZded<ded<d ddddd ZdS) Eventz%Abstract class to represent an event.rQrSzOptional[TSelected]selectedNrSr_cCs||_||_dSr!r`)r#rSr_r$r$r%r&szEvent.__init__)N)r.r/r0r[__annotations__r&r$r$r$r%r^s r^TEventc@sheZdZdZddddZeddddd Zedd dd d Zedd dd dZd dddZ dS)Policyz%Abstract class to represent a policy.r)kwargscKsdSr!r$)r#rdr$r$r%r&szPolicy.__init__rbeventr)cCsdSr!r$r#rfr$r$r%predictszPolicy.predictNonecCsdSr!r$rgr$r$r%learnsz Policy.learncCsdSr!r$rgr$r$r%logsz Policy.logr(cCsdSr!r$r+r$r$r%savesz Policy.saveN) r.r/r0r[r&rrhrjrkrlr$r$r$r%rcsrccsneZdZdZdddddddfdd Zd dd d d Zd dd ddZd dd ddZddddZZ S)VwPolicyzVowpal Wabbit policy.r List[str]Embedderrr) model_repovw_cmdfeature_embedder vw_loggerargsrdcs6tj|i|||_|j||_||_||_dSr!)superr&rpload workspacerrrs)r#rprqrrrsrtrd __class__r$r%r&s zVwPolicy.__init__rbrecCs.ddl}||j}|jt||j|SNr)vowpal_wabbit_nextTextFormatParserrwZ predict_onerPrrformat)r#rfvw text_parserr$r$r%rhs  zVwPolicy.predictricCs:ddl}|j|}||j}t||}|j|dSrz)r{rrr}r|rwrPZ learn_one)r#rfr~vw_exrZmulti_exr$r$r%rjs    zVwPolicy.learncCs&|jr"|j|}|j|dSr!)rsZlogging_enabledrrr}rk)r#rfrr$r$r%rks  z VwPolicy.logr(cCs|j|jdSr!)rprlrwr+r$r$r%rlsz VwPolicy.save) r.r/r0r[r&rhrjrkrl __classcell__r$r$rxr%rms rmc@s4eZdZdZdddddZedddd d Zd S) roz(Abstract class to represent an embedder.r)rtrdcOsdSr!r$)r#rtrdr$r$r%r&szEmbedder.__init__rbr'recCsdSr!r$rgr$r$r%r}szEmbedder.formatN)r.r/r0r[r&rr}r$r$r$r%rosroc@s(eZdZdZedddddddZd S) SelectionScorerzHAbstract class to grade the chosen selection or the response of the llm.rQr'rbfloatrS llm_responserfr)cCsdSr!r$)r#rSrrfr$r$r%score_responseszSelectionScorer.score_responseN)r.r/r0r[rrr$r$r$r%rsrc@seZdZUdZded<dZded<dZded<ed d d d Zed d ddZ e dde dddddZ dddddddZ dS)AutoSelectionScorerzAuto selection scorer.r llm_chainNzUnion[BasePromptTemplate, None]promptz Optional[str]scoring_criteria_template_strrr(cCs tdS)NaPLEASE RESPOND ONLY WITH A SINGLE FLOAT AND NO OTHER TEXT EXPLANATION You are a strict judge that is called on to rank a response based on given criteria. You must respond with your ranking by providing a single float within the range [0, 1], 0 being very bad response and 1 being very good response.)r from_templater$r$r$r%get_default_system_promptsz-AutoSelectionScorer.get_default_system_promptrcCs(d}t|}t}t||g}|S)NzGiven this based_on "{rl_chain_selected_based_on}" as the most important attribute, rank how good or bad this text is: "{rl_chain_selected}".)rrrrr from_messages)Zhuman_templatehuman_message_promptdefault_system_promptZ chat_promptr$r$r%get_default_prompts z&AutoSelectionScorer.get_default_promptbefore)moderQr)valuesr)cCs|d}|d}|d}|dur8|dur8t}n0|durh|durht|}t}t||g}||d<t||d|d<|S)Nllmrr)rrr) getrrrrrrrr)clsrrrrrrr$r$r%set_prompt_and_llm_chains     z,AutoSelectionScorer.set_prompt_and_llm_chainr'r^rrc Csd|jjfd|i|}|}zt|}|WSty^}ztd|WYd}~n d}~00dS)NrzThe auto selection scorer did not manage to score the response, there is always the option to try again or tweak the reward prompt. Error: )rrhstripr Exception RuntimeError)r#rSrrfZrankingresper$r$r%r.sz"AutoSelectionScorer.score_response)r.r/r0r[rarr staticmethodrrr classmethodrrr$r$r$r%rs     rc seZdZUdZGdddeZded<dZded<d ed <d ed <eZd ed<dZ ded<dZ ded<dZ ded<dZ ded<dZ ded<dddedddfdddddd d!d!d"d"d# fd$d% Zedd&d'Zed(d)d*d+Zed(d)d,d-ZdSd.d/dd0d1d2d3Zd0d)d4d5Zd0d)d6d7Zd0d)d8d9Zd/d0d:fd;d< Zdd)d=d>Zed/d?d:d@dAZed/d?d"dBdCdDdEZedd?dBdFdGdHZed?dId?dJdKdLZdTd/dMd/dNdOdPZ edd)dQdRZ!Z"S)URLChaina?Chain that leverages the Vowpal Wabbit (VW) model as a learned policy for reinforcement learning. Attributes: - llm_chain (Chain): Represents the underlying Language Model chain. - prompt (BasePromptTemplate): The template for the base prompt. - selection_scorer (Union[SelectionScorer, None]): Scorer for the selection. Can be set to None. - policy (Optional[Policy]): The policy used by the chain to learn to populate a dynamic prompt. - auto_embed (bool): Determines if embedding should be automatic. Default is False. - metrics (Optional[Union[MetricsTrackerRollingWindow, MetricsTrackerAverage]]): Tracker for metrics, can be set to None. Initialization Attributes: - feature_embedder (Embedder): Embedder used for the `BasedOn` and `ToSelectFrom` inputs. - model_save_dir (str, optional): Directory for saving the VW model. Default is the current directory. - reset_model (bool): If set to True, the model starts training from scratch. Default is False. - vw_cmd (List[str], optional): Command line arguments for the VW model. - policy (Type[VwPolicy]): Policy used by the chain. - vw_logs (Optional[Union[str, os.PathLike]]): Path for the VW logs. - metrics_step (int): Step for the metrics tracker. Default is -1. If set without metrics_window_size, average metrics will be tracked, otherwise rolling window metrics will be tracked. - metrics_window_size (int): Window size for the metrics tracker. Default is -1. If set, rolling window metrics will be tracked. Notes: The class initializes the VW model using the provided arguments. If `selection_scorer` is not provided, a warning is logged, indicating that no reinforcement learning will occur unless the `update_with_delayed_score` method is called. c@s@eZdZdZdddddZddddd Zdddd d Zd S) zRLChain._NoOpPolicyz$Placeholder policy that does nothingrbrrecCsdSr!r$rgr$r$r%rhYszRLChain._NoOpPolicy.predictricCsdSr!r$rgr$r$r%rj\szRLChain._NoOpPolicy.learncCsdSr!r$rgr$r$r%rk_szRLChain._NoOpPolicy.logN)r.r/r0r[rhrjrkr$r$r$r% _NoOpPolicyVsrrrresultr' output_keyrrzUnion[SelectionScorer, None]selection_scorerrc active_policyFr: auto_embedTselection_scorer_activatedZrl_chain_selectedselected_input_keyZrl_chain_selected_based_onselected_based_on_input_keyNzCOptional[Union[MetricsTrackerRollingWindow, MetricsTrackerAverage]]metricsz./rozOptional[List[str]]z Type[Policy]z!Optional[Union[str, os.PathLike]]intr) rrmodel_save_dir reset_modelrqpolicyvw_logs metrics_stepmetrics_window_sizertrdc stj| i| |jdur&tdt|jtjrX|t |d|d|pHg|t |d|_|dkrpt ||d|_ n t |d|_ dS)NzNo selection scorer provided, which means that no reinforcement learning will be done in the RL chain unless update_with_delayed_score is called.T)Z with_historyreset)rprqrrrsr)stepZ window_size)r)rur&rloggerwarningr6rrrrrrrr) r#rrrrrqrrrrrtrdrxr$r%r&ns&    zRLChain.__init__Zforbid)Zarbitrary_types_allowedextrarnr(cCsgS)z1Expect input key. :meta private: r$r+r$r$r% input_keysszRLChain.input_keyscCs|jgS)z3Expect output key. :meta private: )rr+r$r$r% output_keysszRLChain.output_keysrrQri)scorechain_response force_scorer)cCs\|r|std|jr&|j||d}|j||d|jj|d|jj|ddS)z Updates the learned policy with the score provided. Will raise an error if selection_scorer is set, and force_score=True was not provided during the method call zsThe selection scorer is set, and force_score was not set to True. Please set force_score=True to use this function.selection_metadata)rfrrfN)_can_use_selection_scorerrr on_feedback#_call_after_scoring_before_learningrrjrk)r#rrrrfr$r$r%update_with_delayed_scores  z!RLChain.update_with_delayed_scorecCs d|_dS)z Deactivates the selection scorer, meaning that the chain will no longer attempt to use the selection scorer to score responses. FNrr+r$r$r%deactivate_selection_scorersz#RLChain.deactivate_selection_scorercCs d|_dS)z Activates the selection scorer, meaning that the chain will attempt to use the selection scorer to score responses. TNrr+r$r$r%activate_selection_scorersz!RLChain.activate_selection_scorercCs|jdS)z_ This function should be called to save the state of the learned policy model. N)rrlr+r$r$r% save_progressszRLChain.save_progressrRcsFt||j|vs(|j|vrBtd|jd|jddS)NzThe rl chain does not accept 'z' or 'zG' as input keys, they are reserved for internal use during auto reward.)ru_validate_inputsrrVrr8r#rSrxr$r%rs   zRLChain._validate_inputscCs|jduo|jS)zc Returns whether the chain can use the selection scorer to score responses or not. N)rrr+r$r$r%rsz!RLChain._can_use_selection_scorerrbcCsdSr!r$rr$r$r%_call_before_predictszRLChain._call_before_predictzTuple[Dict[str, Any], TEvent])rSrf predictionr)cCsdSr!r$)r#rSrfrr$r$r%_call_after_predict_before_llmsz&RLChain._call_after_predict_before_llm)rrfr)cCsdSr!r$)r#rrfr$r$r%_call_after_llm_before_scoringsz&RLChain._call_after_llm_before_scoringzOptional[float])rfrr)cCsdSr!r$)r#rfrr$r$r%rsz+RLChain._call_after_scoring_before_learningz$Optional[CallbackManagerForChainRun])rS run_managerr)c Cs|p t}|j|d}|jj|d}|jr6|j|j|||d\}}|jj fi|d| i}|j |d|j d| }|j r|j d|j d|}|j d |j d|j |d |j d|j||d \}}d} z|r|jj|||d } Wn6ty(} ztd | WYd} ~ n d} ~ 00|jrH| durH|j| |j| |d}|jj|d|jj|d|j||diS)NrUr)rSrfr callbacksgreen)colorverbosez Code: )rz Answer: yellow)rrf)rSrrfzzThe selection scorer was not able to score, and the chain was not able to adjust to this response, error: )rrf)responser)rZget_noop_managerrrrhrZ on_decisionrrrunZ get_childZon_textrrrrrrrrinforrrjrkr) r#rSrZ _run_managerrfrZnext_chain_inputstoutputrrr$r$r%_callsJ       z RLChain._callcCsdS)NZllm_personalizer_chainr$r+r$r$r% _chain_type!szRLChain._chain_type)F)N)#r.r/r0r[rcrrarrrrrrrrmr&rZ model_configpropertyrrrrrrrrrrrrrrrrr$r$rxr%r<s^        && 3r)F)B __future__rloggingosabcrrtypingrrrrr r r r r rZlangchain.chains.baserZlangchain.chains.llmrZ langchain_core.callbacks.managerrZlangchain_core.promptsrrrrZpydanticrrrZ'langchain_experimental.rl_chain.helpersrZ'langchain_experimental.rl_chain.metricsrrZ0langchain_experimental.rl_chain.model_repositoryrZ)langchain_experimental.rl_chain.vw_loggerrr{r~ getLoggerr.rrr4r5r9r?rHrPrWrYrZr\r^rbrcrmrorrrr$r$r$r%sF 0            +  ?