a
    bgC                     @  s   d dl mZ d dlZd dlZd dlmZ d dlmZ d dlm	Z	m
Z
mZmZmZmZ d dlmZ d dlmZ d dlmZ d d	lmZmZ e Zd
ZddddddZdddddZddddddZG dd deZG dd deZdS )    )annotationsN)sha1)Thread)AnyDictIterableListOptionalTuple)Document)
Embeddings)VectorStore)BaseSettingsSettingsConfigDictFstrr   bool)sargsreturnc                 G  s   |D ]}|| vr dS qdS )z
    Check if a string has multiple substrings.
    Args:
        s: The string to check
        *args: The substrings to check for in the string

    Returns:
        bool: True if all substrings are present in the string, False otherwise
    FT )r   r   ar   r   x/var/www/html/cobodadashboardai.evdpl.com/venv/lib/python3.9/site-packages/langchain_community/vectorstores/starrocks.pyhas_mul_sub_str   s    
r   None)r   r   c                 C  s   t rt|  dS )z[
    Print a debug message if DEBUG is True.
    Args:
        s: The message to print
    N)DEBUGprint)r   r   r   r   debug_output"   s    r   zList[dict[str, Any]])
connectionqueryr   c           
      C  sr   |   }|| |j}g }| D ]8}i }t|D ]\}}|| d }	|||	< q4|| q$t| |  |S )z
    Get a named result from a query.
    Args:
        connection: The connection to the database
        query: The query to execute

    Returns:
        List[dict[str, Any]]: The result of the query
    r   )cursorexecutedescriptionZfetchall	enumerateappendr   close)
r   r   r   columnsresultvalueridxZdatumkr   r   r   get_named_result,   s    


r+   c                   @  s   e Zd ZU dZdZded< dZded< dZded	< d
Zded< dddddZ	ded< dZ
ded< dZded< dddddZedddddZd S )!StarRocksSettingsa  StarRocks client configuration.

    Attribute:
        StarRocks_host (str) : An URL to connect to MyScale backend.
                             Defaults to 'localhost'.
        StarRocks_port (int) : URL port to connect with HTTP. Defaults to 8443.
        username (str) : Username to login. Defaults to None.
        password (str) : Password to login. Defaults to None.
        database (str) : Database name to find the table. Defaults to 'default'.
        table (str) : Table name to operate on.
                      Defaults to 'vector_table'.

        column_map (Dict) : Column type map to project column name onto langchain
                            semantics. Must have keys: `text`, `id`, `vector`,
                            must be same size to number of columns. For example:
                            .. code-block:: python

                                {
                                    'id': 'text_id',
                                    'embedding': 'text_embedding',
                                    'document': 'text_plain',
                                    'metadata': 'metadata_dictionary_in_json',
                                }

                            Defaults to identity map.
    	localhostr   hostiF#  intportrootusername passwordiddocument	embeddingmetadata)r5   r6   r7   r8   zDict[str, str]
column_mapdefaultdatabaseZ	langchaintabler   )itemr   c                 C  s
   t | |S N)getattr)selfr=   r   r   r   __getitem__p   s    zStarRocksSettings.__getitem__z.envutf-8Z
starrocks_ignore)Zenv_fileZenv_file_encodingZ
env_prefixextraN)__name__
__module____qualname____doc__r.   __annotations__r0   r2   r4   r9   r;   r<   rA   r   Zmodel_configr   r   r   r   r,   E   s&   
r,   c                      s<  e Zd ZdZd:ddddd fdd	Zd
d
dddZeddddZddd
dddZddddddZ	d;dddddddddZ
ed<ddd ddddd d!d"d#Zd
dd$d%Zd=d&dd'd
d(d)d*Zd>d
dd'dd,d-d.d/Zd?d&dd'dd,d0d1d2Zd@d
dd'dd3d-d4d5Zddd6d7Zed
dd8d9Z  ZS )A	StarRocksa  `StarRocks` vector store.

    You need a `pymysql` python package, and a valid account
    to connect to StarRocks.

    Right now StarRocks has only implemented `cosine_similarity` function to
    compute distance between two vectors. And there is no vector inside right now,
    so we have to iterate all vectors and compute spatial distance.

    For more information, please visit
        [StarRocks official site](https://www.starrocks.io/)
        [StarRocks github](https://github.com/StarRocks/starrocks)
    Nr   zOptional[StarRocksSettings]r   r   )r7   configkwargsr   c                   s  zddl }W n ty&   tdY n0 zddlm} || _W n tyZ   dd | _Y n0 t   |durv|| _nt | _| jsJ | jjr| jj	sJ | jj
r| jjr| jjsJ dD ]}|| jj
v sJ qt|d}d	| jj d
| jj d| jj
d  d| jj
d  d| jj
d  d| jj
d  d| _|| _d| _d| _|| _d| _t| j |jf | jj| jj	| jj| jj| jjd|| _t| j t| j| j dS )zStarRocks Wrapper to LangChain

        embedding_function (Embeddings):
        config (StarRocksSettings): Configuration to StarRocks Client
        r   NzVCould not import pymysql python package. Please install it with `pip install pymysql`.)tqdmc                 [  s   | S r>   r   )xrL   r   r   r   <lambda>       z$StarRocks.__init__.<locals>.<lambda>)r5   r7   r6   r8   testzCREATE TABLE IF NOT EXISTS .z
(    
    r5   z string,
    r6   r7   z array<float>,
    r8   zf string
) ENGINE = OLAP PRIMARY KEY(id) DISTRIBUTED BY HASH(id)   PROPERTIES ("replication_num" = "1")\)rS   'ZDESC)r.   r0   userr4   r;   )pymysqlImportErrorrM   pgbarsuper__init__rK   r,   r.   r0   r9   r;   r<   lenembed_queryZschemadimBSmust_escapeembedding_function
dist_orderr   connectr2   r4   r   r+   )r@   r7   rK   rL   rV   rM   r*   r]   	__class__r   r   rZ      sf    








	
	
zStarRocks.__init__r   )r'   r   c                   s   d  fdd|D S )Nr3   c                 3  s*   | ]"}| j v r j | n|V  qd S r>   )r_   r^   ).0cr@   r   r   	<genexpr>   rP   z'StarRocks.escape_str.<locals>.<genexpr>)join)r@   r'   r   rg   r   
escape_str   s    zStarRocks.escape_str)r   c                 C  s   | j S r>   )r`   rg   r   r   r   
embeddings   s    zStarRocks.embeddingsr   zIterable[str])transaccolumn_namesr   c              
     s   d |}t|jjd  g }|D ]4}d  fddt|D }|d| d q(djj djj d| d	d | d
	}|S )N,r7   c                   s<   g | ]4\}}| kr*d  t| d ndt| qS )rT   zarray<float>)rj   r   )re   r)   Z_nZembed_tuple_indexr@   r   r   
<listcomp>   s   z/StarRocks._build_insert_sql.<locals>.<listcomp>()z1
                INSERT INTO
                    rR   z))
                VALUES
                z
                )	ri   tupleindexrK   r9   r"   r#   r;   r<   )r@   rl   rm   ks_datanZi_strr   ro   r   _build_insert_sql   s,    


zStarRocks._build_insert_sqlc                 C  s$   |  ||}t| t| j| d S r>   )rx   r   r+   r   )r@   rl   rm   Z_insert_queryr   r   r   _insert   s    zStarRocks._insert    zOptional[List[dict]]r/   zOptional[Iterable[str]]z	List[str])texts	metadatas
batch_sizeidsrL   r   c              
   K  s  |pdd |D }| j j}g }|d ||d ||d | jt|i}|pVdd |D }ttj|||d < tt	|t	| dksJ t
|  \}	}
zd	}| jt
|
 d
t|dD ]j}t||	| j jd  | jksJ || t||kr|r|  t| j||	gd}|  g }qt|dkrJ|r>|  | ||	 dd |D W S  ty } z0tdt| dt| d g W  Y d	}~S d	}~0 0 d	S )a  Insert more texts through the embeddings and add to the VectorStore.

        Args:
            texts: Iterable of strings to add to the VectorStore.
            ids: Optional list of ids to associate with the texts.
            batch_size: Batch size of insertion
            metadata: Optional column data to be inserted

        Returns:
            List of ids from adding the texts into the VectorStore.

        c                 S  s   g | ]}t |d  qS )rB   )r   encode	hexdigest)re   tr   r   r   rp   	  rP   z'StarRocks.add_texts.<locals>.<listcomp>r5   r6   r7   c                 S  s   g | ]}i qS r   r   )re   _r   r   r   rp     rP   r8   r   NzInserting data...)desctotal)targetr   c                 S  s   g | ]}|qS r   r   )re   ir   r   r   rp   (  rP   	[91m[1m
[0m [95m[0m)rK   r9   r`   Zembed_documentslistmapjsondumpsr[   setzipitemsrX   rt   r]   r#   ri   r   ry   start	Exceptionloggererrortyper   )r@   r{   r|   r}   r~   rL   Zcolmap_rl   rm   keysvaluesr   ver   r   r   	add_texts   sB    

 zStarRocks.add_textszOptional[List[Dict[Any, Any]]])r{   r7   r|   rK   text_idsr}   rL   r   c           	      K  s(   | ||fi |}|j ||||d |S )a  Create StarRocks wrapper with existing texts

        Args:
            embedding_function (Embeddings): Function to extract text embedding
            texts (Iterable[str]): List or tuple of strings to be added
            config (StarRocksSettings, Optional): StarRocks configuration
            text_ids (Optional[Iterable], optional): IDs for the texts.
                                                     Defaults to None.
            batch_size (int, optional): Batchsize when transmitting data to StarRocks.
                                        Defaults to 32.
            metadata (List[dict], optional): metadata to texts. Defaults to None.
        Returns:
            StarRocks Index
        )r~   r}   r|   )r   )	clsr{   r7   r|   rK   r   r}   rL   ctxr   r   r   
from_texts-  s    zStarRocks.from_textsc                 C  sH  d| j j d| j j d}|| j j d| j j d7 }|d| j j d7 }d}d	}|d
|| d  d 7 }g d}|d|d dd|d d7 }|d|d dd7 }|d
|| d  d 7 }d| j j d| j j }t| t| j|}|D ]:}|d|d dd|d d7 }|d|d dd7 }q|d
|| d  d 7 }|S )zText representation for StarRocks Vector Store, prints backends, username
            and schemas. Easy to use with `str(StarRocks())`

        Returns:
            repr: string to show connection info and data schema
        z	[92m[1mrR   z @ :z[0m

z[1musername: z[0m

Table Schema:
      -   
)namer   keyz|[94mr   Z24sz
[0m|[96m   z[0m|
zDESC FieldTypeKey)	rK   r;   r<   r.   r0   r2   r   r+   r   )r@   _reprwidthfieldsr%   q_strrsr(   r   r   r   __repr__J  s$      zStarRocks.__repr__zList[float]zOptional[str])q_embtopk	where_strr   c                 C  s   d tt|}|r d| }nd}d| jjd  d| jjd  d| d	| jjd
  d| jj d| jj d| d| j d| d}t| |S )Nrn   zWHERE r3   z
            SELECT r6   z, 
                r8   z7, 
                cosine_similarity_norm(array<float>[z],
                  r7   z) as dist
            FROM rR   z
            z
            ORDER BY dist z
            LIMIT )	ri   r   r   rK   r9   r;   r<   ra   r   )r@   r   r   r   Z	q_emb_strr   r   r   r   _build_query_sqld  s2    


zStarRocks._build_query_sql   zList[Document])r   r*   r   rL   r   c                 K  s   | j | j|||fi |S )a  Perform a similarity search with StarRocks

        Args:
            query (str): query string
            k (int, optional): Top K neighbors to retrieve. Defaults to 4.
            where_str (Optional[str], optional): where condition string.
                                                 Defaults to None.

            NOTE: Please do not let end-user to fill this and always be aware
                  of SQL injection. When dealing with metadatas, remember to
                  use `{self.metadata_column}.attribute` instead of `attribute`
                  alone. The default name for it is `metadata`.

        Returns:
            List[Document]: List of Documents
        )similarity_search_by_vectorr`   r\   )r@   r   r*   r   rL   r   r   r   similarity_search{  s
    zStarRocks.similarity_search)r7   r*   r   rL   r   c              
     sx     |||}z fddt j|D W S  tyr } z0tdt| dt| d g W  Y d}~S d}~0 0 dS )a  Perform a similarity search with StarRocks by vectors

        Args:
            query (str): query string
            k (int, optional): Top K neighbors to retrieve. Defaults to 4.
            where_str (Optional[str], optional): where condition string.
                                                 Defaults to None.

            NOTE: Please do not let end-user to fill this and always be aware
                  of SQL injection. When dealing with metadatas, remember to
                  use `{self.metadata_column}.attribute` instead of `attribute`
                  alone. The default name for it is `metadata`.

        Returns:
            List[Document]: List of (Document, similarity)
        c              	     s6   g | ].}t | jjd   t| jjd  dqS )r6   r8   Zpage_contentr8   r   rK   r9   r   loadsre   r(   rg   r   r   rp     s
   z9StarRocks.similarity_search_by_vector.<locals>.<listcomp>r   r   r   N)r   r+   r   r   r   r   r   r   )r@   r7   r*   r   rL   r   r   r   rg   r   r     s    

 z%StarRocks.similarity_search_by_vectorzList[Tuple[Document, float]]c              
     s      j|||}z fddt j|D W S  tyz } z0tdt| dt	| d g W  Y d}~S d}~0 0 dS )a  Perform a similarity search with StarRocks

        Args:
            query (str): query string
            k (int, optional): Top K neighbors to retrieve. Defaults to 4.
            where_str (Optional[str], optional): where condition string.
                                                 Defaults to None.

            NOTE: Please do not let end-user to fill this and always be aware
                  of SQL injection. When dealing with metadatas, remember to
                  use `{self.metadata_column}.attribute` instead of `attribute`
                  alone. The default name for it is `metadata`.

        Returns:
            List[Document]: List of documents
        c              	     s>   g | ]6}t | jjd   t| jjd  d|d fqS )r6   r8   r   distr   r   rg   r   r   rp     s   zEStarRocks.similarity_search_with_relevance_scores.<locals>.<listcomp>r   r   r   N)
r   r`   r\   r+   r   r   r   r   r   r   )r@   r   r*   r   rL   r   r   r   rg   r   'similarity_search_with_relevance_scores  s    


 z1StarRocks.similarity_search_with_relevance_scoresc                 C  s$   t | jd| jj d| jj  dS )z,
        Helper function: Drop data
        zDROP TABLE IF EXISTS rR   N)r+   r   rK   r;   r<   rg   r   r   r   drop  s    zStarRocks.dropc                 C  s   | j jd S )Nr8   )rK   r9   rg   r   r   r   metadata_column  s    zStarRocks.metadata_column)N)Nrz   N)NNNrz   )N)r   N)r   N)r   N)rE   rF   rG   rH   rZ   rj   propertyrk   rx   ry   r   classmethodr   r   r   r   r   r   r   r   __classcell__r   r   rc   r   rJ   {   s>    D   9         % %	rJ   ) 
__future__r   r   logginghashlibr   	threadingr   typingr   r   r   r   r	   r
   Zlangchain_core.documentsr   Zlangchain_core.embeddingsr   Zlangchain_core.vectorstoresr   Zpydantic_settingsr   r   	getLoggerr   r   r   r   r+   r,   rJ   r   r   r   r   <module>   s     
6