a bg,@sNddlZddlmZmZmZmZddlmZddlm Z ddl m Z ddl m Z ddlmZdZd Zd Zd Zd Zd ZdZdZdZdZdZdZdZdZdZdZdZ dZ!dZ"eeegZ#edgdedgdedgde!dgde"dgdedgdedgdedegdedegdedegde degdi Z$ed d!d"Z%Gd#d$d$e Z&dS)%N)AnyDictListOptional)UUID)BaseCallbackHandler) LLMResult guard_import) import_pandasZ token_usageZ total_tokensZ prompt_tokensZcompletion_tokensrun_id model_nameZgoodbadZneutralsuccessZfailureiipromptresponsecontextdurationZfeedback llm_statusfiddler z$123e4567-e89b-12d3-a456-426614174000returncCs tdddS)zLImport the fiddler python package and raise an error if it is not installed.rzfiddler-client)Zpip_namer rr|/var/www/html/cobodadashboardai.evdpl.com/venv/lib/python3.9/site-packages/langchain_community/callbacks/fiddler_callback.pyimport_fiddler8src seZdZeeeeeddfdd ZeedddZdee ee ee ee e ee fdd d d Ze ee fe ee e d d dZee ddddZee ddddZZS)FiddlerCallbackHandlerN)urlorgprojectmodelapi_keyrc stt|_t|_||_||_||_||_ ||_ |j t |_ i|_i|_i|_|jj|||d|_|j|jvrtd|jdz|j|jWn:ty}z"td|jd|WYd}~n d}~00|jjj|j dd}tt|jD]h}|j|jtkr4|jjj|j|_ t!|j|_"q|j|jt#kr|jjj|j|_ t$t%g|j|_"q|j |j&|jvr|j |j'|jvrtd |j d |jdz"|jj(|j|j d |j i|d WnBty}z(td |j d|d|WYd}~n d}~00|jj)j*|d |jj+j,t-t.t/gtt0t1t2t3t4t5g|j6d} td|j d |jdz|jj7|j|j |j | dWnBty}z(td|j d|d|WYd}~n d}~00dS)a Initialize Fiddler callback handler. Args: url: Fiddler URL (e.g. https://demo.fiddler.ai). Make sure to include the protocol (http/https). org: Fiddler organization id project: Fiddler project name to publish events to model: Fiddler model name to publish events to api_key: Fiddler authentication token )Zorg_idZ auth_tokenzadding project z.This only has to be done once.zError adding project z(:{e}. Fiddler integration will not work.Nr)Zmax_inferred_cardinalityzadding dataset z to project train) project_id dataset_idZdatasetinfozError adding dataset z: z#.Fiddler integration will not work.) dataset_infor&Z model_taskfeaturestargetZ metadata_colscustom_featuresz adding model )r%r&Zmodel_id model_infozError adding model )8super__init__rfdlr pdrr r!r"r# DataFrame _dataset_dictZ_dfrun_id_promptsZrun_id_responserun_id_starttimeZ FiddlerApifiddler_clientZget_project_namesprintZ add_project ExceptionZ DatasetInfoZfrom_dataframerangelencolumnsnameFEEDBACKZDataTypeCATEGORYZ data_typeFEEDBACK_POSSIBLE_VALUESZpossible_values LLM_STATUSSUCCESSFAILUREZget_model_namesZget_dataset_namesZupload_datasetZ ModelInfoZfrom_dataset_infoZ ModelTaskZLLMPROMPTCONTEXTRESPONSERUN_ID TOTAL_TOKENS PROMPT_TOKENSCOMPLETION_TOKENS MODEL_NAMEDURATIONr+Z add_model) selfrr r!r"r#er(ir, __class__rrr.?s     zFiddlerCallbackHandler.__init__rc Cs|jjddtgd|jjdtdd|jjddtgd|jjdtdd|jjdd ttgd gd id |jjd dttgd|jjddttgdgS)a[ Define custom features for the model to automatically enrich the data with. Here, we enable the following enrichments: - Automatic Embedding generation for prompt and response - Text Statistics such as: - Automated Readability Index - Coleman Liau Index - Dale Chall Readability Score - Difficult Words - Flesch Reading Ease - Flesch Kincaid Grade - Gunning Fog - Linsear Write Formula - PII - Personal Identifiable Information - Sentiment Analysis zPrompt EmbeddingZ embedding)r; enrichmentr:z Prompt CF)r;Z source_columncolumnzResponse Embeddingz Response CFzText StatisticsZtextstat statistics)Zautomated_readability_indexZcoleman_liau_indexZdale_chall_readability_scoreZdifficult_wordsZflesch_reading_easeZflesch_kincaid_gradeZ gunning_fogZlinsear_write_formula)r;rPr:configZPIIZpiiZ SentimentZ sentiment)r/Z EnrichmentrBZ TextEmbeddingrD)rKrrrr+sLz&FiddlerCallbackHandler.custom_features)r prompt_responsesrrr token_usage_dictrc Cst|j|}|jt|j|t|tt|g|t|g|t |g|t |g|i}|r| D]$\} } t | t r|| g|n| || <q`zN|jddkr|j|j|j|n&|jdd} |jj|j|j| ddWn6ty } ztd| dWYd} ~ n d} ~ 00dS) z+ Publish events to fiddler rrrecords)Zorient)eventz$Error publishing events to fiddler: z. continuing...N)r9r3r0r1rBrDrEstrrJr?rIitems isinstanceintshaper5Zpublish_events_batchr!r"to_dictZ publish_eventr7r6) rKr rUrrr rVZ prompt_countZdfkeyvalueZdf_dictrLrrr_publish_eventss0         z&FiddlerCallbackHandler._publish_events) serializedpromptskwargsrcKs,|t}||j|<ttd|j|<dS)N)rEr3r\timer4)rKrbrcrdr rrr on_llm_start&s z#FiddlerCallbackHandler.on_llm_start)rrdrc Ks|}|t}ttd|j|}d}i}t|jtr`dd|jD}|j t d}dd|D}| |||t ||dS)NrerTcSs$i|]\}}|tttfvr||qSr)rFrGrH).0kvrrr 5sz5FiddlerCallbackHandler.on_llm_end..cSsg|]}|jddjqS)r)Z generationstext)rhZ llmresultrrr <sz5FiddlerCallbackHandler.on_llm_end..) flattenrEr\rfr4r[Z llm_outputdictrZgetrIrar@) rKrrdZflattened_llmresultr Z run_durationr rVrUrrr on_llm_end-s* z!FiddlerCallbackHandler.on_llm_end)errorrdrcKsF|t}ttd|j|}||dgt|j||tdS)NrerT)rEr\rfr4rar9r3rA)rKrrrdr rrrr on_llm_errorIs z#FiddlerCallbackHandler.on_llm_error)rTN)__name__ __module__ __qualname__rYr.propertylistr+rrr\rrrrargrrq BaseExceptionrs __classcell__rrrNrr>s2wK + r)'rftypingrrrruuidrZlangchain_core.callbacksrZlangchain_core.outputsrZlangchain_core.utilsr Z#langchain_community.callbacks.utilsr Z TOKEN_USAGErFrGrHrErIZGOODBADZNEUTRALr@rAZDEFAULT_MAX_TOKENZDEFAULT_MAX_DURATIONrBrDrCrJr<r?r>r2rrrrrrsP