a bgY@sddlmZddlZddlZddlmZddlmZddlm Z m Z m Z m Z m Z mZddlmZddlmZddlmZdd lmZmZeZd d d d ddZGdddeZGdddeZGdddeZdS)) annotationsN)sha1)Thread)AnyDictIterableListOptionalTuple)Document) Embeddings) VectorStore) BaseSettingsSettingsConfigDictstrrbool)sargsreturncGs|D]}||vrdSqdS)z Check if a string contains multiple substrings. Args: s: string to check. *args: substrings to check. Returns: True if all substrings are in the string, False otherwise. FT)rrarrv/var/www/html/cobodadashboardai.evdpl.com/venv/lib/python3.9/site-packages/langchain_community/vectorstores/myscale.pyhas_mul_sub_strs rc@seZdZUdZdZded<dZded<dZd ed <dZd ed <d Z ded <dZ ded<dddddZ ded<dZ ded<dZ ded<dZded<ddddd Zed!d"d#d$d%ZdS)&MyScaleSettingsaMyScale client configuration. Attribute: myscale_host (str) : An URL to connect to MyScale backend. Defaults to 'localhost'. myscale_port (int) : URL port to connect with HTTP. Defaults to 8443. username (str) : Username to login. Defaults to None. password (str) : Password to login. Defaults to None. index_type (str): index type string. index_param (dict): index build parameter. database (str) : Database name to find the table. Defaults to 'default'. table (str) : Table name to operate on. Defaults to 'vector_table'. metric (str) : Metric to compute distance, supported are ('L2', 'Cosine', 'IP'). Defaults to 'Cosine'. column_map (Dict) : Column type map to project column name onto langchain semantics. Must have keys: `text`, `id`, `vector`, must be same size to number of columns. For example: .. code-block:: python { 'id': 'text_id', 'vector': 'text_embedding', 'text': 'text_plain', 'metadata': 'metadata_dictionary_in_json', } Defaults to identity map. localhostrhosti intportN Optional[str]usernamepasswordZMSTG index_typezOptional[Dict[str, str]] index_paramidtextvectormetadata)r#r$r%r&zDict[str, str] column_mapdefaultdatabaseZ langchaintableZCosinemetricr)itemrcCs t||SN)getattr)selfr,rrr __getitem__UszMyScaleSettings.__getitem__z.envutf-8Zmyscale_ignore)Zenv_fileZenv_file_encodingZ env_prefixextra)__name__ __module__ __qualname____doc__r__annotations__rrr r!r"r'r)r*r+r0rZ model_configrrrrr!s,          rc sReZdZdZd?dddddfdd Zedd d d Zd d dddZddd dddZddddddZ d@dddddddddZ e dAddd dddddd!d"d#Z d d d$d%Z dBd&dd'd d(d)d*ZdCd dd'dd,d-d.d/ZdDd&dd'dd,d0d1d2ZdEd dd'dd3d-d4d5Zdd d6d7ZdFd8d'dd9d:d;d<Zed d d=d>ZZS)GMyScalea`MyScale` vector store. You need a `clickhouse-connect` python package, and a valid account to connect to MyScale. MyScale can not only search with simple vector indexes. It also supports a complex query with multiple conditions, constraints and even sub-queries. For more information, please visit [myscale official site](https://docs.myscale.com/en/overview/) Nr Optional[MyScaleSettings]rNone) embeddingconfigkwargsrc  szddlm}Wnty*tdYn0zddlm}||_Wnty^dd|_Yn0t|durz||_nt|_|jsJ|jj r|jj sJ|jj r|jj r|jj r|jjsJdD]}||jj vsJq|jjd vsJ|jjd vr td t|d }|jjrBd ddd|jjDnd}d|jj d|jj d|jj dd|jj dd|jj dd|jj dd|jj dd|d|jj dd|jjd|jjd |d!|jj dd"} ||_d#|_d$|_||_|jjd%vrd&nd'|_|f|jj |jj |jj|jjd(||_z|j d)Wn<t!y} z"t"d*|jj#d+WYd} ~ n d} ~ 00|j d,|j | dS)-zMyScale Wrapper to LangChain embedding (Embeddings): config (MyScaleSettings): Configuration to MyScale Client Other keyword arguments will pass into [clickhouse-connect](https://docs.myscale.com/) r) get_clientzlCould not import clickhouse connect python package. Please install it with `pip install clickhouse-connect`.)tqdmcSs|Sr-r)xrrrz"MyScale.__init__..N)r#r%r$r&)ZIPCOSINEL2)ipZcosinel2z_Lower case metric types will be deprecated the future. Please use one of ('IP', 'Cosine', 'L2')z try this out, ,cSs"g|]\}}d|d|dqS)'=r).0kvrrr rCz$MyScale.__init__..z( CREATE TABLE IF NOT EXISTS .z( r#z String, r$r%z! Array(Float32), r&zP JSON, CONSTRAINT cons_vec_len CHECK length( z) = z$, VECTOR INDEX vidx z TYPE z&( 'metric_type=rJz,) ) ENGINE = MergeTree ORDER BY z \)rRrJ)rDrEASCZDESC)rrrr z"SET allow_experimental_json_type=1zClickhouse version=z6 - There is no allow_experimental_json_type parameter.z$SET allow_experimental_object_type=1)$Zclickhouse_connectr? ImportErrorr@pgbarsuper__init__r=rrrr'r)r*r+upperloggerwarninglen embed_queryr"joinitemsr!dimBS must_escape _embeddings dist_orderrr clientcommand ExceptiondebugZserver_version) r/r<r=r>r?r@rMr_Z index_paramsZschema__ __class__rrrWns        $            zMyScale.__init__rcCs|jSr-)rbr/rrr embeddingsszMyScale.embeddingsr)valuercsdfdd|DS)NrPc3s*|]"}|jvrj|n|VqdSr-)rar`)rLcrlrr rCz%MyScale.escape_str..)r])r/rnrrlr escape_strszMyScale.escape_strrz Iterable[str])transac column_namesrc srd|}g}|D].}dfdd|D}|d|dqdjjdjjd|dd|d }|S) NrIcs"g|]}dt|dqSrJ)rqr)rLZ_nrlrrrOrCz'MyScale._build_istr..()z8 INSERT INTO TABLE rQz)) VALUES z )r]appendr=r)r*)r/rrrsks_datanZi_strrrlr _build_istrs zMyScale._build_istrcCs|||}|j|dSr-)r{rdre)r/rrrsZ_i_strrrr_inserts zMyScale._insert zOptional[List[dict]]rzOptional[Iterable[str]] List[str])texts metadatas batch_sizeidsr>rc Ks|pdd|D}|jj}g}|d||d||dt|jj|i}|pTdd|D}ttj|||d<tt|t|dksJt | \} } zd } |j t | d t|d D]j} t| | |jjd|j ksJ|| t||kr| r| t|j|| gd } | g}qt|dkrH| r<| ||| d d|DWSty} z0tdt| dt| dgWYd } ~ Sd } ~ 00d S)aRun more texts through the embeddings and add to the vectorstore. Args: texts: Iterable of strings to add to the vectorstore. ids: Optional list of ids to associate with the texts. batch_size: Batch size of insertion metadata: Optional column data to be inserted Returns: List of ids from adding the texts into the vectorstore. cSsg|]}t|dqS)r1)rencode hexdigest)rLtrrrrOrCz%MyScale.add_texts..r#r$r%cSsg|]}iqSrr)rLrhrrrrOrCr&rNzInserting data...)desctotal)targetrcSsg|]}|qSrr)rLirrrrOrC   )r=r'maprbr\jsondumpsr[setzipr^rUindexr_rwr]rr|startrfrYerrortyper)r/rrrrr>Zcolmap_rrrskeysvaluesrrNerrr add_textss@ $    zMyScale.add_textszOptional[List[Dict[Any, Any]]])rr<rr=text_idsrr>rc Ks(|||fi|}|j||||d|S)aZCreate Myscale wrapper with existing texts Args: texts (Iterable[str]): List or tuple of strings to be added embedding (Embeddings): Function to extract text embedding config (MyScaleSettings, Optional): Myscale configuration text_ids (Optional[Iterable], optional): IDs for the texts. Defaults to None. batch_size (int, optional): Batchsize when transmitting data to MyScale. Defaults to 32. metadata (List[dict], optional): metadata to texts. Defaults to None. Other keyword arguments will pass into [clickhouse-connect](https://clickhouse.com/docs/en/integrations/python#clickhouse-connect-driver-api) Returns: MyScale Index )rrr)r) clsrr<rr=rrr>ctxrrr from_textsszMyScale.from_textscCsd|jjd|jjd}||jjd|jjd7}|d|jjd7}|d7}|jd |jjd|jjD]&}|d |d d d |dd d7}qv|d7}|S)zText representation for myscale, prints backends, username and schemas. Easy to use with `str(Myscale())` Returns: repr: string to show connection info and data schema z rQz @ :z zusername: z Table Schema: z4--------------------------------------------------- zDESC z|nameZ24sz |rz| ) r=r)r*rrrrdquery named_results)r/_reprrrrr__repr__>s zMyScale.__repr__ List[float]rq_embtopk where_strrcCsdtt|}|r d|}nd}d|jjdd|jjdd|jjd |jjd |d |jjd d |d|jd|d }|S)NrI PREWHERE rP SELECT r$z, r&z, dist FROM rQ  ORDER BY distance(r%, []) AS dist  LIMIT )r]rrr=r'r)r*rcr/rrrZ q_emb_strq_strrrr _build_qstrRs0     zMyScale._build_qstrList[Document]rrMrr>rcKs|j|j|||fi|S)aPerform a similarity search with MyScale Args: query (str): query string k (int, optional): Top K neighbors to retrieve. Defaults to 4. where_str (Optional[str], optional): where condition string. Defaults to None. NOTE: Please do not let end-user to fill this and always be aware of SQL injection. When dealing with metadatas, remember to use `{self.metadata_column}.attribute` instead of `attribute` alone. The default name for it is `metadata`. Returns: List[Document]: List of Documents )similarity_search_by_vectorrbr\)r/rrMrr>rrrsimilarity_searchfs zMyScale.similarity_searchr<rMrr>rc s||||}z fddj|DWStyv}z0tdt|dt|dgWYd}~Sd}~00dS)Perform a similarity search with MyScale by vectors Args: query (str): query string k (int, optional): Top K neighbors to retrieve. Defaults to 4. where_str (Optional[str], optional): where condition string. Defaults to None. NOTE: Please do not let end-user to fill this and always be aware of SQL injection. When dealing with metadatas, remember to use `{self.metadata_column}.attribute` instead of `attribute` alone. The default name for it is `metadata`. Returns: List[Document]: List of (Document, similarity) cs0g|](}t|jjd|jjddqS)r$r&Z page_contentr&r r=r'rLrrlrrrOs z7MyScale.similarity_search_by_vector..rrrN rrdrrrfrYrrrr/r<rMrr>rrrrlrr}s  z#MyScale.similarity_search_by_vectorList[Tuple[Document, float]]c sj|||}z fddj|DWSty~}z0tdt |dt |dgWYd}~Sd}~00dS)/Perform a similarity search with MyScale Args: query (str): query string k (int, optional): Top K neighbors to retrieve. Defaults to 4. where_str (Optional[str], optional): where condition string. Defaults to None. NOTE: Please do not let end-user to fill this and always be aware of SQL injection. When dealing with metadatas, remember to use `{self.metadata_column}.attribute` instead of `attribute` alone. The default name for it is `metadata`. Returns: List[Document]: List of documents most similar to the query text and cosine distance in float for each. Lower score represents more similarity. cs8g|]0}t|jjd|jjdd|dfqS)r$r&rdistrrrlrrrOszCMyScale.similarity_search_with_relevance_scores..rrrN rrbr\rdrrrfrYrrrr/rrMrr>rrrrlr'similarity_search_with_relevance_scoress   z/MyScale.similarity_search_with_relevance_scorescCs$|jd|jjd|jjdS)z, Helper function: Drop data zDROP TABLE IF EXISTS rQN)rdrer=r)r*rlrrrdropsz MyScale.dropzOptional[List[str]]zOptional[bool])rrr>rc Ks|dur|durJdg}|r^t|dkr^ddd|D}||jjdd|d |rl||t|dks|Jd |}d |jjd |jjd |}z|j|WdSt y}zt t |WYd}~dSd}~00dS)a3Delete by vector ID or other criteria. Args: ids: List of ids to delete. **kwargs: Other keyword arguments that subclasses might use. Returns: Optional[bool]: True if deletion is successful, False otherwise, None if not implemented. NzIYou need to specify where to be deleted! Either with `ids` or `where_str`rrHcSsg|]}d|dqSrtr)rLr#rrrrOrCz"MyScale.delete..r#z IN (rvz AND z DELETE FROM rQz WHERE TF) r[r]rwr=r'r)r*rdrerfrYrr) r/rrr>ZcondsZid_listZwhere_str_finalZqstrrrrrdeletes0   zMyScale.deletecCs |jjdS)Nr&)r=r'rlrrrmetadata_columnszMyScale.metadata_column)N)Nr}N)NNNr})N)rN)rN)rN)NN)r4r5r6r7rWpropertyrmrqr{r|r classmethodrrrrrrrrr __classcell__rrrirr9`sD`8 %% &r9cseZdZdZdgfddddddfd d Zdd d d ddddZdd d d dddddZd dd d dddddZeddddZ Z S)!MyScaleWithoutJSONzsMyScale vector store without metadata column This is super handy if you are working to a SQL-native table Nr r:r~rr;)r<r=must_have_colsr>rc s tj||fi|||_dS)agBuilding a myscale vector store without metadata column embedding (Embeddings): embedding model config (MyScaleSettings): Configuration to MyScale Client must_have_cols (List[str]): column names to be included in query Other keyword arguments will pass into [clickhouse-connect](https://docs.myscale.com/) N)rVrWr)r/r<r=rr>rirrrWszMyScaleWithoutJSON.__init__rrrrrcCsdtt|}|r d|}nd}d|jjddd|jd|jjd|jjd |d |jjd d |d |jd|d }|S)NrIrrPrr$z, dist, z FROM rQrrr%rrr) r]rrr=r'rr)r*rcrrrrrs0     zMyScaleWithoutJSON._build_qstrrrrc s||||}z fddj|DWStyv}z0tdt|dt|dgWYd}~Sd}~00dS)rcs4g|],tjjdfddjDdqS)r$csi|]}||qSrrrLrMrrr ArCzMMyScaleWithoutJSON.similarity_search_by_vector...rr r=r'rrLrlrrrO>s zBMyScaleWithoutJSON.similarity_search_by_vector..rrrNrrrrlrr%s  z.MyScaleWithoutJSON.similarity_search_by_vectorrrc sj|||}z fddj|DWSty~}z0tdt |dt |dgWYd}~Sd}~00dS)rcs<g|]4tjjdfddjDddfqS)r$csi|]}||qSrrrrrrrdrCzYMyScaleWithoutJSON.similarity_search_with_relevance_scores...rrrrrlrrrO`szNMyScaleWithoutJSON.similarity_search_with_relevance_scores..rrrNrrrrlrrIs   z:MyScaleWithoutJSON.similarity_search_with_relevance_scoresrkcCsdS)NrPrrlrrrrnsz"MyScaleWithoutJSON.metadata_column)N)rN)rN) r4r5r6r7rWrrrrrrrrrirrs%%r) __future__rrlogginghashlibr threadingrtypingrrrrr r Zlangchain_core.documentsr Zlangchain_core.embeddingsr Zlangchain_core.vectorstoresr Zpydantic_settingsrr getLoggerrYrrr9rrrrrs"       ?