a bg@sddlmZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z ddl m Z mZmZmZmZmZmZmZmZddlmZddlmZddlmZe rdd lmZdd lmZ Gd d d eZ!dS) ) annotationsN)contextmanager)StringIO) TYPE_CHECKINGAnyDict GeneratorIterableListOptionalTupleType) Embeddings) VectorStore)Document) connection)cursorc @s<eZdZdZGdddeejZGdddZddddd d d d d d ddddZ GdddZ dddddZ dddddZ djd d dddddZ dkdd d dddd Zd d!d"d#Zdld$d%d&d'd(d)d*Zdd+d+dd,d-d.Zedmd2d'd d%d d d d d&dd3 d4d5Zdnd6d7d&dd8d9d:Zdodd d d d;dd?d@dAdBZdpd>dDd&dEdFdGdHZdqd dDd&dIdJdKdLZdrd dDd&dEdJdMdNZdsd>dDd&dIdFdOdPZdtddQddRdSdTZdd d ddUdVdWZddDddXdYdZZdddd[d\Zdddd]d^Zd_dd`dadbZ d_dd`dcddZ!d_d?ddedfdgZ"dd!dhdiZ#dS)u YellowbrickzYellowbrick as a vector database. Example: .. code-block:: python from langchain_community.vectorstores import Yellowbrick from langchain_community.embeddings.openai import OpenAIEmbeddings ... c@seZdZdZdZdZdS)zYellowbrick.IndexTypez Helper function: create schema if not exists rsqlzE CREATE SCHEMA IF NOT EXISTS {s} )sN)r5rkrHexecuteSQLformat Identifier)r"rrkrrrrNs  zYellowbrick._create_schemac Csddlm}|jr|jfnd}|jg||j|jR}||j|jd}||dj||d|jrx|jfnd}|jg||jR}|jg||j|jR}||j|jd}||j|jd} ||d j|||| d d S) z= Helper function: create table if not exists rrjr _pk_doc_ida0 CREATE TABLE IF NOT EXISTS {t} ( doc_id UUID NOT NULL, text VARCHAR(60000) NOT NULL, metadata VARCHAR(1024) NOT NULL, CONSTRAINT {c} PRIMARY KEY (doc_id)) DISTRIBUTE ON (doc_id) SORT ON (doc_id) tcZ_pk_doc_id_embedding_id _fk_doc_ida CREATE TABLE IF NOT EXISTS {t1} ( doc_id UUID NOT NULL, embedding_id SMALLINT NOT NULL, embedding FLOAT NOT NULL, CONSTRAINT {c1} PRIMARY KEY (doc_id, embedding_id), CONSTRAINT {c2} FOREIGN KEY (doc_id) REFERENCES {t2}(doc_id)) DISTRIBUTE ON (doc_id) SORT ON (doc_id) t1t2c1c2N) r5rkrHrprIrCrmrnro) r"rrk schema_prefixrsrtrwrxryrzrrrrOs:   zYellowbrick._create_tablezOptional['PgCursor'])r2r+rr'cCsX|durD|j }|j|||dWdqT1s80Yn|j|||ddS)z Helper function: Drop data. If a cursor is provided, use it; otherwise, obtain a new cursor for the operation. N)r+)rrL _drop_table)r"r2r+rrrrr-s  0zYellowbrick.drop)rr2r+r'cCsFddlm}|r|||}n ||}|d|}||dS)zI Executes the drop table command using the given cursor. rrjz1 DROP TABLE IF EXISTS {} CASCADE N)r5rkrprnrorm)r"rr2r+rk table_nameZdrop_table_queryrrrr|s  zYellowbrick._drop_tabler[cCsl|j*}d}|||d}Wdn1s:0Y|dks\|dkr`dStddS) zE Helper function: Test the database is UTF-8 encoded z SELECT pg_encoding_to_char(encoding) FROM pg_database WHERE datname = current_database(); rNutf8zutf-8TzDatabase encoding is not UTF-8F)rrLrmfetchonelower Exception)r"rqueryencodingrrrrKs  *z Yellowbrick._check_database_utf8z Iterable[str]zOptional[List[dict]]rz List[str])texts metadataskwargsr'c Ksd}t|}|jt|}g}|s2dd|D}|dpBt}|j}t} t} t j | ddt j d} t j | ddt j d} d} t |D]\}}t t}||| ||t||gt ||D]\}}| |||gq| d 7} | |kr||| | | d| d| d| dd} q| dkrT||| | Wdn1sj0Y|jtjjkr||t||S) Ni'cSsg|]}iqSrr).0_rrr ?z)Yellowbrick.add_texts.. index_params ") delimiter quotecharquotingr)listrJZembed_documentsr(rr*rrLrcsvwriter QUOTE_MINIMAL enumerater$uuiduuid4appendwriterowjsondumps _copy_to_dbseektruncaterrr _update_indexUUID)r"rrrZ batch_size embeddingsresultsrr content_io embeddings_ioZcontent_writerZembeddings_writerZcurrent_batch_sizeitextZdoc_uuid embedding_idr0rrr add_texts3sH         .zYellowbrick.add_textsr)rrrr'c Cs|d|dddlm}|jr.|jfnd}|jg||j|jR}|dj|d}| |||jrz|jfnd}|jg||jR}|dj|d}| ||dS)Nrrjrz COPY {table} (doc_id, text, metadata) FROM STDIN WITH (FORMAT CSV, DELIMITER E'\t', QUOTE '"') )r2z COPY {table} (doc_id, embedding_id, embedding) FROM STDIN WITH (FORMAT CSV, DELIMITER E'\t', QUOTE '"') ) rr5rkrHrprIrCrnroZ copy_expert) r"rrrrkr{r2Zcontent_copy_queryZembeddings_copy_queryrrrrjs&    zYellowbrick._copy_to_db langchainpubliczType[Yellowbrick]) rXrr0rr1r2r+r-rr'c Ks,||||||d} | jf||d|| S)aAdd texts to the vectorstore index. Args: texts: Iterable of strings to add to the vectorstore. metadatas: Optional list of metadatas associated with the texts. connection_string: URI to Yellowbrick instance embedding: Embedding function table: table to store embeddings kwargs: vectorstore specific parameters )r0r1r2r+r-)rr)r) rXrr0rr1r2r+r-rZvssrrr from_textsszYellowbrick.from_textszOptional[List[str]]zOptional[bool])ids delete_allrr'c  snddlm|rd}nJ|dur^tfdd|D}d|}dj|d }ntd |jrt|jfnd }|j }j g||j |j R} d j| |d } | | j g||j R} d j| |d } | | |j||j |jg|RrJj g||j |jR} d j| |d } | | Wdn1s`0YdS)zxDelete vectors by uuids. Args: ids: List of ids to delete, where each id is a uuid string. rrjz' WHERE 1=1 Nc3s|]}|VqdSr!)Literal)ridrjrr rz%Yellowbrick.delete..z, z5 WHERE doc_id IN ({ids}) )rz*Either ids or delete_all must be provided.rzDELETE FROM {table} {where_sql})r2 where_sql)r5rkrntuplejoinro ValueErrorrHrrLrprIrCrm _table_existsrA) r"rrrrZuuidsZ ids_formattedr{rZtable_identifierrrrjrdeletesZ           *zYellowbrick.delete)rr}r+r'cCsJddlm}||}||}||dj||d|ddkS)z> Checks if a table exists in the given schema rrjz SELECT COUNT(*) FROM sys.table t INNER JOIN sys.schema s ON t.schema_id = s.schema_id WHERE s.name = {schema} AND t.name = {table_name} )r+r})r5rkrrmrnror)r"rr}r+rkrrrrs    zYellowbrick._table_existsz List[float]z uuid.UUID)vectorr'cCsFddl}dtt|}||}|}tj|ddd}|S)Nr,)bytes) hashlibrmapr$sha1encodedigestrr)r"rrZ vector_strZ hash_objectZ hash_digestZ vector_uuidrrr_generate_vector_uuids z!Yellowbrick._generate_vector_uuidintzList[Tuple[Document, float]])r0krr'c s,ddlm}ddlm}|dp(t}|j}d|j }| || d | |} || fddt|D} | d  | |} ||| | | |} |jr|jfnd } |j g| |j R}|j g| |j |jR}|jtjjkr|j d }|||||jr*|jfnd } |j g| |j |jR}| |}| d j | ||||||d dd}|||f|}n,| dj | ||d}|||f|}Wdn1s0Yg}|D]<}t|dpi}t|d|d}|||dfq|S)aPerform a similarity search with Yellowbrick with vector Args: embedding (List[float]): query embedding k (int, optional): Top K neighbors to retrieve. Defaults to 4. NOTE: Please do not let end-user fill this and always be aware of SQL injection. Returns: List[Document, float]: List of Documents and scores rrj)execute_valuesrZtmp_z CREATE TEMPORARY TABLE {} ( doc_id UUID, embedding_id SMALLINT, embedding FLOAT) ON COMMIT DROP DISTRIBUTE REPLICATE csg|]\}}t||fqSr)r$)rrZembedding_valueZ tmp_doc_idrrr%szFYellowbrick.similarity_search_with_score_by_vector..z:INSERT INTO {} (doc_id, embedding_id, embedding) VALUES %srZ _tmp_hasha/ WITH index_docs AS ( SELECT t1.doc_id, SUM(ABS(t1.hash-t2.hash)) as hamming_distance FROM {lsh_index} t1 INNER JOIN {input_hash_table} t2 ON t1.hash_index = t2.hash_index GROUP BY t1.doc_id HAVING hamming_distance <= {hamming_distance} ) SELECT text, metadata, SUM(v1.embedding * v2.embedding) / (SQRT(SUM(v1.embedding * v1.embedding)) * SQRT(SUM(v2.embedding * v2.embedding))) AS score FROM {v1} v1 INNER JOIN {v2} v2 ON v1.embedding_id = v2.embedding_id INNER JOIN {v3} v3 ON v2.doc_id = v3.doc_id INNER JOIN index_docs v4 ON v2.doc_id = v4.doc_id GROUP BY v3.doc_id, v3.text, v3.metadata ORDER BY score DESC LIMIT %s hamming_distance)v1v2v3 lsh_indexinput_hash_tablera SELECT text, metadata, score FROM (SELECT v2.doc_id doc_id, SUM(v1.embedding * v2.embedding) / (SQRT(SUM(v1.embedding * v1.embedding)) * SQRT(SUM(v2.embedding * v2.embedding))) AS score FROM {v1} v1 INNER JOIN {v2} v2 ON v1.embedding_id = v2.embedding_id GROUP BY v2.doc_id ORDER BY score DESC LIMIT %s ) v4 INNER JOIN {v3} v3 ON v4.doc_id = v3.doc_id ORDER BY score DESC )rrrNr)Z page_contentmetadata)r5rkZpsycopg2.extrasrr(rr*rrLrIrrnrorprmrrHrCrrr_generate_tmp_lsh_hashesrArr)Zfetchallrloadsrr)r"r0rrrkrrrZtmp_embeddings_tableZcreate_table_queryZ data_input insert_queryrr{rrtmp_hash_tablerrZ sql_queryr documentsresultrdocrrr&similarity_search_with_score_by_vectors            $ - (z2Yellowbrick.similarity_search_with_score_by_vectorzList[Document])rrrr'cKs0|j|}|jf||d|}dd|DS)aePerform a similarity search with Yellowbrick Args: query (str): query string k (int, optional): Top K neighbors to retrieve. Defaults to 4. NOTE: Please do not let end-user fill this and always be aware of SQL injection. Returns: List[Document]: List of Documents r0rcSsg|] \}}|qSrrrrrrrrrrz1Yellowbrick.similarity_search..rJZ embed_queryrr"rrrr0rrrrsimilarity_searchs zYellowbrick.similarity_searchcKs&|j|}|jf||d|}|S)arPerform a similarity search with Yellowbrick Args: query (str): query string k (int, optional): Top K neighbors to retrieve. Defaults to 4. NOTE: Please do not let end-user fill this and always be aware of SQL injection. Returns: List[Document]: List of (Document, similarity) rrrrrrsimilarity_search_with_scores z(Yellowbrick.similarity_search_with_scorecKs$|jf||d|}dd|DS)aPerform a similarity search with Yellowbrick by vectors Args: embedding (List[float]): query embedding k (int, optional): Top K neighbors to retrieve. Defaults to 4. NOTE: Please do not let end-user fill this and always be aware of SQL injection. Returns: List[Document]: List of documents rcSsg|] \}}|qSrrrrrrrrz;Yellowbrick.similarity_search_by_vector..)r)r"r0rrrrrrsimilarity_search_by_vectors z'Yellowbrick.similarity_search_by_vectorzOptional[uuid.UUID])rdoc_idr'c Csddlm}|jr|jfnd}|jg||j|jR}|jg||j|jR}|jg||jR}|d|}|r|dj| t |dn|d} |d} |d j|||| | d } | | d S) zAdd hashes to LSH indexrrjrzINSERT INTO {}zWHERE e.doc_id = {doc_id})rrz GROUP BY 1, 2av {query_prefix} SELECT e.doc_id as doc_id, h.id as hash_index, CASE WHEN SUM(e.embedding * h.hyperplane) > 0 THEN 1 ELSE 0 END as hash FROM {embedding_table} e INNER JOIN {hyperplanes} h ON e.embedding_id = h.hyperplane_id {condition} {group_by} ) query_prefixembedding_table hyperplanes conditiongroup_byN) r5rkrHrprIrBrArnrorr$rm) r"rrrkr{lsh_hyperplane_tableZlsh_index_table_idZembedding_table_idZquery_prefix_idrr input_queryrrr_update_lsh_hashess:     zYellowbrick._update_lsh_hashes)rtmp_embedding_tablerr'c Csddlm}|jr|jfnd}|jg||j|jR}||}||}|d|} |d} |dj| ||| d} || dS) zGenerate temp LSHrrjrz+CREATE TEMPORARY TABLE {} ON COMMIT DROP ASz GROUP BY 1a[ {query_prefix} SELECT h.id as hash_index, CASE WHEN SUM(e.embedding * h.hyperplane) > 0 THEN 1 ELSE 0 END as hash FROM {embedding_table} e INNER JOIN {hyperplanes} h ON e.embedding_id = h.hyperplane_id {group_by} DISTRIBUTE REPLICATE )rrrrN) r5rkrHrprIrBrnrorm) r"rrrrkr{rZtmp_embedding_table_idZtmp_hash_table_idrrrrrrrs,       z$Yellowbrick._generate_tmp_lsh_hashes)rnum_hyperplanesr'c Csddlm}|jr|jfnd}|jg||j|jR}||dj|d| ddkrfdS|jg||jR}||dj|d| d}|d7}|d j| || ||d }||dS) z4Generate random hyperplanes and store in YellowbrickrrjrzSELECT COUNT(*) FROM {t})rsNz!SELECT MAX(embedding_id) FROM {t}ra2 WITH parameters AS ( SELECT {num_hyperplanes} AS num_hyperplanes, {dims_per_hyperplane} AS dims_per_hyperplane ) INSERT INTO {hyperplanes_table} (id, hyperplane_id, hyperplane) SELECT id, hyperplane_id, (random() * 2 - 1) AS hyperplane FROM (SELECT range-1 id FROM sys.rowgenerator WHERE range BETWEEN 1 AND (SELECT num_hyperplanes FROM parameters) AND worker_lid = 0 AND thread_id = 0) a, (SELECT range-1 hyperplane_id FROM sys.rowgenerator WHERE range BETWEEN 1 AND (SELECT dims_per_hyperplane FROM parameters) AND worker_lid = 0 AND thread_id = 0) b )rZdims_per_hyperplanehyperplanes_table) r5rkrHrprIrBrmrnrorr) r"rrrkr{rrsZnum_dimensionsrrrr_populate_hyperplanes*s,   z!Yellowbrick._populate_hyperplanesc Csddlm}|jr|jfnd}|jg||j|jR}|jg||j|jR}||j|jd}||j|jd}||dj ||||d|jr|jfnd}|jg||j|j R}||j|j d} ||d j || d d S) z&Create LSH index and hyperplane tablesrrjrrqrua CREATE TABLE IF NOT EXISTS {t1} ( doc_id UUID NOT NULL, hash_index SMALLINT NOT NULL, hash SMALLINT NOT NULL, CONSTRAINT {c1} PRIMARY KEY (doc_id, hash_index), CONSTRAINT {c2} FOREIGN KEY (doc_id) REFERENCES {t2}(doc_id)) DISTRIBUTE ON (doc_id) SORT ON (doc_id) rvZ _pk_id_hp_ida2 CREATE TABLE IF NOT EXISTS {t} ( id SMALLINT NOT NULL, hyperplane_id SMALLINT NOT NULL, hyperplane FLOAT NOT NULL, CONSTRAINT {c} PRIMARY KEY (id, hyperplane_id)) DISTRIBUTE REPLICATE SORT ON (id) rrN) r5rkrHrprIrArCrmrnrorB) r"rrkr{rwrxryrzrsrtrrr_create_lsh_index_tablesTs6   z$Yellowbrick._create_lsh_index_tablescCs8|j|j|j|j|d|j|j|j|j|ddS)zDrop LSH index tables)r+r2rN)r-rHrIrArB)r"rrrrrMs z"Yellowbrick._drop_lsh_index_tablesr)rr'cCsn|jtjjkrj|jB}||||||| dd| |Wdn1s`0YdS)z"Create index from existing vectorsrN) rrrrrrLrMrrr)rr"rrrrr create_indexs    zYellowbrick.create_indexcCsF|jtjjkrB|j}||Wdn1s80YdS)z Drop an indexN)rrrrrrLrMrrrr drop_indexs zYellowbrick.drop_index)rrr'cCsH|jtjjkrD|j}|||Wdn1s:0YdS)zHUpdate an index with a new or modified embedding in the embeddings tableN)rrrrrrLr)r"rrrrrrrs zYellowbrick._update_indexc Cs8ddlm}z|j}|jr(|jfnd}|jg||jR}|jg||jdR}|jg||j|jR}|dj ||d}| || ||dj ||d}| ||dj ||d} | | Wdn1s0YWn6t y2} zt d | | WYd} ~ n d} ~ 00dS) NrrjrZ_v1zALTER TABLE {t1} RENAME TO {t2})rwrxz INSERT INTO {t1} (doc_id, embedding_id, embedding) SELECT id, embedding_id, embedding FROM {t2} z INSERT INTO {t1} (doc_id, text, metadata) SELECT DISTINCT id, text, metadata FROM {t2} zFailed to migrate schema: )r5rkrrLrHrprIrCrnrormrOrrc) r"rkrr{rZold_embeddingscontentZalter_table_queryrZinsert_content_queryrfrrrmigrate_schema_v1_to_v2s@       ,z#Yellowbrick.migrate_schema_v1_to_v2)NN)N)N)NrrrF)NN)r)r)r)r)r)N)$rrrrr$enumEnumrr*r#rDrNrOr-r|rKrr classmethodrrrrrrrrrrrrrMrrrrrrrrr!sf>667$ > -$*/  r)" __future__rrErrrr6r contextlibriortypingrrrrr r r r r Zlangchain_core.embeddingsrZlangchain_core.vectorstoresrZ%langchain_community.docstore.documentrZpsycopg2.extensionsrZ PgConnectionrZPgCursorrrrrrs   ,