a bg#a@sddlmZddlZddlZddlZddlZddlmZmZm Z m Z m Z m Z m Z ddlZddlmZddlmZddlmZddlmZddlmZd Zd d d d ZdddddZGdddeZdS)) annotationsN)AnyCallableDictIterableListOptionalType)Document) Embeddings guard_import) VectorStore)maximal_marginal_relevancerreturncCstdS)zImport lancedb package.lancedbr rrv/var/www/html/cobodadashboardai.evdpl.com/venv/lib/python3.9/site-packages/langchain_community/vectorstores/lancedb.pyimport_lancedbsrzDict[str, str]str)filterrcCsddd|DS)z2Converts a dict filter to a LanceDB filter string.z AND cSs g|]\}}|d|dqS)z = ''r).0kvrrr z#to_lance_filter..)joinitems)rrrrto_lance_filtersr!c@s eZdZdZdddddddddddd ddefd d d d d d d d d d d d d d ddddZdZddddddZed dddZd[ddddddd d!Z d\d d"dd#d$d%Z d]d d d)d)d)d d d*d+d,d-Z d.d.d/d0d1Z d^dddddd2d3d4Z d_dd)d d ddd5d6d7Zd8dd9d:Zd`d;d)dd?Zdad;d)d.csJg|]B}tj|r.d|nid|fqSrYrZr[r]rVZ score_colrSrrrs )Zschemanamesrangelen)rSrVrWcolumnsrr^rresults_to_docss   zLanceDB.results_to_docsrcCs|jS)N)r<rSrrr embeddingsszLanceDB.embeddingsz Iterable[str]zOptional[List[dict]]zOptional[List[str]]z List[str])texts metadatasidskwargsrc Ksg}|pdd|D}|jt|}t|D]L\}}||} |rJ||n d||i} ||j| |j|||j|d| iq.|} | dur|j j |j |d} | |_ n&|j dur| j||jdn | |d|_|S)aTurn texts into embedding and add it to the database Args: texts: Iterable of strings to add to the vectorstore. metadatas: Optional list of metadatas associated with the texts. ids: Optional list of ids to associate with the texts. ids: Optional list of ids to associate with the texts. Returns: List of ids of the added texts. cSsg|]}ttqSrruuiduuid4r_rrrrrz%LanceDB.add_texts..r$rXNdata)r3)r<Zembed_documentslist enumerateappendr=r>r?rRrI create_tablerPrNr1addr3rC) rSrfrgrhridocsrer\r%r+rXtblrrr add_textss,    zLanceDB.add_textszOptional[bool])r:r;rcCsN|dur |r||_|j}q&|}n|j}z|j|WStyHYdS0dS)a  Fetches a table object from the database. Args: name (str, optional): The name of the table to fetch. Defaults to None and fetches current table object. set_default (bool, optional): Sets fetched table as the default table. Defaults to False. Returns: Any: The fetched table object. Raises: ValueError: If the specified table is not found in the database. N)rPrIZ open_table Exception)rSr:r;_namerrrrRs zLanceDB.get_table`L2z Optional[int]None)col_name vector_colnum_partitionsnum_sub_vectorsindex_cache_sizemetricr:rc Cs@||}|r$|j|||||dn|r4||ntddS)aO Create a scalar(for non-vector cols) or a vector index on a table. Make sure your vector column has enough data before creating an index on it. Args: vector_col: Provide if you want to create index on a vector column. col_name: Provide if you want to create index on a non-vector column. metric: Provide the metric to use for vector index. Defaults to 'L2' choice of metrics: 'L2', 'dot', 'cosine' num_partitions: Number of partitions to use for the index. Defaults to 256. num_sub_vectors: Number of sub-vectors to use for the index. Defaults to 96. index_cache_size: Size of the index cache. Defaults to None. name: Name of the table to create index on. Defaults to None. Returns: None )rvector_column_namerrrz%Provide either vector_col or col_nameN)rR create_indexZcreate_scalar_indexrF) rSrrrrrrr:rwrrrrs  zLanceDB.create_indexr)r,rcCsBt|d$}t|dWdS1s40YdS)z!Get base64 string from image URI.rbzutf-8N)openbase64 b64encodereaddecode)rSr,Z image_filerrr encode_image2s zLanceDB.encode_image)urisrgrhrirc s}fdd|D}|dur0dd|D}d}jdurZtjdrZjj|d}ntdg}t|D]H\} } |r|| n d|| i} |j| j|| j || d | iqn|durڈj j j |d }|_ n |||S) asRun more images through the embeddings and add to the vectorstore. Args: uris List[str]: File path to the image. metadatas (Optional[List[dict]], optional): Optional list of metadatas. ids (Optional[List[str]], optional): Optional list of IDs. Returns: List[str]: List of IDs of the added images. csg|]}j|dqS))r,)r)rr,rdrrrKrz&LanceDB.add_images..NcSsg|]}ttqSrrjrmrrrrNr embed_image)rzEembedding object should be provided and must have embed_image method.r$rXro)rRr<rOrrFrrrsr=r>r?rIrtrPrNru) rSrrgrhrirwZ b64_textsrerpr\ZembrXrrdr add_images7s2   zLanceDB.add_images)queryrrr:rirc Ks|dur|j}||}t|tr*t|}|dd}|dd}|d} rx|j||jd|| j ||d} n |j||jd|j ||d} |dkr|j dur| j |j d | } t | d krtd | S) N prefilterF query_typer#metrics)rr)rhybrid)r6rzNo results found for the query.)r8rRrDdictr!getsearchr=rwhererEZrerankZto_arrowrarKrL) rSrrrr:rirwrrrZ lance_queryrvrrr_queryks8      zLanceDB._queryzCallable[[float], float]cCsR|jr |jS|jdkr|jS|jdkr,|jS|jdkr<|jStd|jddS)a8 The 'correct' relevance function may differ depending on a few things, including: - the distance / similarity metric used by the VectorStore - the scale of your embeddings (OpenAI's are unit normed. Many others are not!) - embedding dimensionality - etc. Zcosiner(ipzANo supported normalization function for distance metric of type: z=.Consider providing relevance_score_fn to Chroma constructor.N)rBr5Z_cosine_relevance_score_fnZ_euclidean_relevance_score_fnZ%_max_inner_product_relevance_score_fnrFrdrrr_select_relevance_score_fns    z"LanceDB._select_relevance_score_fnz List[float]zOptional[Dict[str, str]])r+rrr:rircKs>|dur|j}|j||f||d|}|j||dddS)zD Return documents most similar to the query vector. Nrr:rWFrW)r8rrcpop)rSr+rrr:riresrrrsimilarity_search_by_vectors z#LanceDB.similarity_search_by_vectorc s@|dur|j}||j||fddi|}fdd|DS)zZ Return documents most similar to the query vector with relevance scores. NrWTcs g|]\}}|t|fqSr)float)rdocrWr7rrrszMLanceDB.similarity_search_by_vector_with_relevance_scores..)r8rr)rSr+rrr:riZdocs_and_scoresrrr1similarity_search_by_vector_with_relevance_scoress  z9LanceDB.similarity_search_by_vector_with_relevance_scores)rrrrirc Ks|dur|j}|dd}|dd}|dd}|jdurDtd|dksT|d kr|jdur|jdur||}|j|jdd |_|d kr|j |} | |f} n|} |j | |f||d |} |j | |d St d n2|j |} |j | |fd|i|} |j | |d SdS)zAReturn documents most similar to the query with relevance scores.NrWTr:rr#z4search needs an emmbedding function to be specified.ftsr)replacerrz?Full text/ Hybrid search is not supported in LanceDB Cloud yet.r) r8rr<rFr1rCrRZcreate_fts_indexr? embed_queryrrcNotImplementedError) rSrrrrirWr:rrwr+rrrrrsimilarity_search_with_scores.        z$LanceDB.similarity_search_with_scorezList[Document])rrr:rrrirc Ks"|jf|||||dd|}|S)apReturn documents most similar to the query Args: query: String to query the vectorstore with. k: Number of documents to return. filter (Optional[Dict]): Optional filter arguments sql_filter(Optional[string]): SQL filter to apply to the query. prefilter(Optional[bool]): Whether to apply the filter prior to the vector search. Raises: ValueError: If the specified table is not found in the database. Returns: List of documents most similar to the query. F)rrr:rrrW)r)rSrrr:rrrirrrrsimilarity_searchs  zLanceDB.similarity_search?r)rrfetch_k lambda_multrrirc KsD|dur|j}|jdur td|j|}|j|||||d}|S)a?Return docs selected using the maximal marginal relevance. Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents. Args: query: Text to look up documents similar to. k: Number of Documents to return. Defaults to 4. fetch_k: Number of Documents to fetch to pass to MMR algorithm. lambda_mult: Number between 0 and 1 that determines the degree of diversity among the results with 0 corresponding to maximum diversity and 1 to minimum diversity. Defaults to 0.5. filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. Returns: List of Documents selected by maximal marginal relevance. NzBFor MMR search, you must specify an embedding function oncreation.)rr)r8r<rFr'max_marginal_relevance_search_by_vector) rSrrrrrrir+rvrrrmax_marginal_relevance_searchs  z%LanceDB.max_marginal_relevance_search)r+rrrrrirc  sf|jf|||d|}ttj|tjd|d|p:|j|d||}fddt|D} | S)aHReturn docs selected using the maximal marginal relevance. Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents. Args: embedding: Embedding to look up documents similar to. k: Number of Documents to return. Defaults to 4. fetch_k: Number of Documents to fetch to pass to MMR algorithm. lambda_mult: Number between 0 and 1 that determines the degree of diversity among the results with 0 corresponding to maximum diversity and 1 to minimum diversity. Defaults to 0.5. filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. Returns: List of Documents selected by maximal marginal relevance. )rrr)Zdtyper#)rrcsg|]\}}|vr|qSrr)rirZ mmr_selectedrrrlrzCLanceDB.max_marginal_relevance_search_by_vector..) rrnparrayZfloat32Z to_pylistr8rcrr) rSr+rrrrrirV candidatesZselected_resultsrrrrBs   z/LanceDB.max_marginal_relevance_search_by_vectorz Type[LanceDB]r )clsrfr+rgr*r-r.r/r0r1r2r3r5r6r7rircKs:tf||||||| | | | | |d |}|j||d|S)N) r*r+r-r.r/r0r1r2r3r5r6r7)rg)r"rx)rrfr+rgr*r-r.r/r0r1r2r3r5r6r7riinstancerrr from_textsos$ zLanceDB.from_texts)rh delete_allr drop_columnsr:rircKs~||}|r||n`|r>||jdd|n<|rb|jdurVtdqz||n|rr|dntddS)a Allows deleting rows by filtering, by ids or drop columns from the table. Args: filter: Provide a string SQL expression - "{col} {operation} {value}". ids: Provide list of ids to delete from the table. drop_columns: Provide list of columns to drop from the table. delete_all: If True, delete all rows from the table. z in ('{}'),Nz;Column operations currently not supported in LanceDB Cloud.truez6Provide either filter, ids, drop_columns or delete_all) rRdeleter>formatrr1rrrF)rSrhrrrr:rirwrrrrs      zLanceDB.delete)F)NN)NF)NNr{r|Nr}N)NN)NNN)NNN)NNN)NN)NNNF)NrrN)NrrN) NNr#r$r%r&NNr'r(NN)NNNNN)__name__ __module__ __qualname____doc__ DEFAULT_KrTrcpropertyrerxrRrrrrrrrrrrr classmethodrrrrrrr"s,W"1#*7&+ /-2'r") __future__rrr@rkrKtypingrrrrrrr numpyrZlangchain_core.documentsr Zlangchain_core.embeddingsr Zlangchain_core.utilsr Zlangchain_core.vectorstoresrZ&langchain_community.vectorstores.utilsrrrr!r"rrrrs $