a bg<@sddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl m Z ddl m Z mZmZmZmZmZmZddlmZddlmZddlmZe rddlmZeeZGd d d e ZGd d d Z Gd ddeZ!GdddeZ"dS)) annotationsN) HTMLParser) TYPE_CHECKINGAnyDictListOptionalTupleUnion) BaseLoader)Document) TextSplitter) ConnectioncsVeZdZdZddfdd Zddddd d Zddd d d ZddddZZS)ParseOracleDocMetadatazParse Oracle doc metadata...Nonereturncs"t|d|_i|_dS)NF)super__init__resetmatchmetadataself __class__{/var/www/html/cobodadashboardai.evdpl.com/venv/lib/python3.9/site-packages/langchain_community/document_loaders/oracleai.pyr%s zParseOracleDocMetadata.__init__strzList[Tuple[str, Optional[str]]])tagattrsrcCsP|dkr>d}|D]*\}}|dkr$|}|dkr|r||j|<qn|dkrLd|_dS)NmetanamecontenttitleT)rr)rrr entryr#valuerrrhandle_starttag+s z&ParseOracleDocMetadata.handle_starttag)datarcCs|jr||jd<d|_dS)Nr%F)rr)rr)rrr handle_data7s z"ParseOracleDocMetadata.handle_dataDict[str, Any]cCs|jSN)rrrrr get_metadata<sz#ParseOracleDocMetadata.get_metadata) __name__ __module__ __qualname____doc__rr(r*r- __classcell__rrrrr"s  rc@s>eZdZdZeddddddZeddd d d d d ZdS)OracleDocReaderz Read a fileNzUnion[str, None]r) input_stringrc Csd}d}|dur$dtjddd}tt}td|}t|  }|d|}tdt d}|||}| }| |}|d|}|S)N r"Z>abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789)kz>I)joinrandomchoicesinttimestructpackhashlibsha256encodedigest getrandbitshexzfill) r4Z out_lengthZhash_len timestampZ timestamp_binZ hashval_binZ counter_binZ object_idZ object_id_hexrrrgenerate_object_idFs*     z"OracleDocReader.generate_object_idrdictzUnion[Document, None])conn file_pathparamsrc Csi}z ddl}Wn.ty>}ztd|WYd}~n d}~00z2d|j_|}t|d}|}Wdn1s|0Y|durtd|dWS||j } ||j } |j d|t || | d | | duri}n$_oid_fileAn exception occurred :: zSkip processing )oracledb ImportErrordefaults fetch_lobscursoropenreadr varZ DB_TYPE_CLOBexecutejsondumpsclosergetvalue startswithrfeedr-r3rHusername Exceptionloggerinfo)rJrKrLrrXer\fr)rPrQZdoc_datapdoc_idexrrr read_fileis\   &     zOracleDocReader.read_file)N)r.r/r0r1 staticmethodrHrprrrrr3Cs "r3cs8eZdZdZddddfdd Zdd d d ZZS) OracleDocLoaderzwRead documents using OracleDocLoader Args: conn: Oracle Connection, params: Loader parameters. rr+r)rJrLkwargsc s.||_tt||_tjfi|dSr,)rJraloadsrbrLrr)rrJrLrsrrrrszOracleDocLoader.__init__zList[Document]rc Csz ddl}Wn.ty:}ztd|WYd}~n d}~00d}g}i}ddi}z|jdur|jd|_|jd|_|jd|_|jd |_|jd |_nt d d |j _ |jrt |j|j|}|dur|WS|||jr`d}t|jD]`} tj|j| } tj| rt |j| |}|durT|d }td|dq||q|jrfz|jdus|jdurt d|j} |jd|_|jdur"t|jdkrt dd} | j| |j|jd| } | D],}|d|jvr|d dvrt dqd|_|jdurR|jD]}|jd||_q:d|jdt|d|jd|jd|jd|jd } | | | D]r}i}|durt |jj!d!|jd!|jd!|j}||d"<|t"d#|d$q|ddurLt#|d}|$d%s4|$d&rLt%}|&||'}t |jj!d!|jd!|jd!|jd!t#|d'}||d"<|d'|d(<|jdurt|j}t(d|D]}||d'||j|<q|d dur|t"d#|d$n|t"t#|d |d$qWnHt yd}z.td)|t)*| +WYd}~n d}~00|WSt y}z&td)|t)*WYd}~n d}~00dS)*z,Load data into LangChain Document objects...rNrMZ plaintextfalsefiledirowner tablenamecolnamezMissing loader parametersFzTotal skipped:  z%Missing owner or column name or both. mdata_colsz?Exceeds the max number of columns you can request for metadata.zgselect column_name, data_type from all_tab_columns where owner = :ownername and table_name = :tablename)Z ownernamery)NUMBERZ BINARY_DOUBLEZ BINARY_FLOATLONGZDATE TIMESTAMPZVARCHAR2zDThe datatype for the column requested for metadata is not supported.z, rowidz, z'select dbms_vector_chain.utl_to_text(t.z, json('z+')) mdata, dbms_vector_chain.utl_to_text(t.z) textz from .z trTrUr"rOrRrSZ_rowidrW),rXrYrLgetrvrwrxryrzrhrZr[r3rprJappendoslistdirpathr9isfilerirjr\r}lenr`upperZfetchallZmdata_cols_sqlrarbrHrgr rrerrfr-range traceback print_excrc)rrXrkZncolsresultsrZm_paramsdocZ skip_count file_namerKr\sqlrowsrowcolrnr)rmirorrrloadsH                             zOracleDocLoader.load)r.r/r0r1rrr2rrrrrrsrrcs<eZdZdZdddddfdd Zd d d d d ZZS)OracleTextSplitterz$Splitting text using Oracle chunker.rr+rr)rJrLrsrc s||_||_tjfi|zTddl}z ddl}Wn.tyb}ztd|WYd}~n d}~00||_||_WntytdYn0dS)z Initialize.rNrMzjoracledb or json or both are not installed. Please install them. Recommendations: `pip install oracledb`. ) rJrLrrrarXrY _oracledb_json)rrJrLrsrarXrkrrrrs&   zOracleTextSplitter.__init__rz List[str])rQrc Csz ddl}Wn.ty:}ztd|WYd}~n d}~00g}ztd|jj_|j}|j|jd|j d||j |j d| }|durq|j |d}||dq~|WSty}z&td |tWYd}~n d}~00dS) z&Split incoming text and return chunks.rNrMF)r$zUselect t.column_value from dbms_vector_chain.utl_to_chunks(:content, json(:params)) t)r$rLZ chunk_datarW)rXrYrrZr[rJr\Z setinputsizesZCLOBr`rrbrLZfetchonertrrhrirjrr) rrQrXrkZsplitsr\rdrorrr split_texts8    zOracleTextSplitter.split_text)r.r/r0r1rrr2rrrrr|sr)# __future__rr@raloggingrr:r>r=r html.parserrtypingrrrrrr r Zlangchain_core.document_loadersr Zlangchain_core.documentsr Zlangchain_text_splittersr rXr getLoggerr.rirr3rrrrrrr s*  $     !uE