a bg@s"ddlmZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z m Z m Z mZmZmZmZmZmZmZddlZddlmZddlmZddlmZddlmZdd lm Z m!Z!dd l"m#Z#dd l$m%Z%m&Z&e'e(Z)dd d dddZ*d d ddddddZ+GdddeZ,dS)) annotationsN)Path) AnyCallableDictIterableListOptionalSequenceSizedTupleUnionDocument) Embeddings)run_in_executor) VectorStore) AddableMixinDocstore)InMemoryDocstore)DistanceStrategymaximal_marginal_relevanceOptional[bool]r)no_avx2returncCs^|dur dtjvr ttd}z|r4ddlm}nddl}WntyXtdYn0|S)aM Import faiss if available, otherwise raise error. If FAISS_NO_AVX2 environment variable is set, it will be considered to load FAISS with no AVX2 optimization. Args: no_avx2: Load FAISS strictly with no AVX2 optimization so that the vectorstore is portable and compatible with other devices. NZ FAISS_NO_AVX2r) swigfaisszCould not import faiss python package. Please install it with `pip install faiss-gpu` (for CUDA supported GPU) or `pip install faiss-cpu` (depending on Python version).)osenvironboolgetenvfaissr ImportError)rr r"t/var/www/html/cobodadashboardai.evdpl.com/venv/lib/python3.9/site-packages/langchain_community/vectorstores/faiss.pydependable_faiss_import's    r$strNone)xyx_namey_namerc CsZt|trVt|trVt|t|krVt|d|d|dt|d|dt| dS)Nz and z% expected to be equal length but len(z)=z and len() isinstancer len ValueError)r'r(r)r*r"r"r#_len_check_if_sizedBs$ r.c @seZdZdZddejfdddddd d d d d ZeddddZdddddZ dddddZ dddddZ dddddZ dd d!d"d#dd$d%d&Z dd d'd#ddd(d)d*Zdd d'd#ddd(d+d,Zdd-d'd#ddd.d/d0Zddd3d4d3dd5d6d7d8Zddd3d4d3dd5d6d9d:Zddd3d4d3dd5d;dd?Zddd3d@d3ddAd6dBdCZddd3d4d3ddAd6dDdEZddd3d4d3ddAd;dFdGZddd3d4d3ddAd;dHdIZd1d2dJddKdd3d3dLd4d5dMdNdOZd1d2dJddKdd3d3dLd4d5dMdPdQZddd3d3dLd4ddAdRdSdTZddd3d3dLd4ddAdRdUdVZddd3d3dLd4ddAdWdXdYZddd3d3dLd4ddAdWdZd[Zdd#dd\d]d^d_Zdd`dadbdcZ e!dddejfd dddd"d#d d ddde dfdgZ"e!ddddd'd#dddhdidjZ#e!ddkddd'd#dddhdldmZ$e!dd-ddd"d#dddndodpZ%e!dd-ddd"d#dddndqdrZ&dddd`dtdudvZ'e!dddwddddd dddxdydzZ(d{dd|d}Z)e!ddwd{ddd ddd~ddZ*ddddZ+ddd3d4d3dd5d;ddZ,ddd3d4d3dd5d;ddZ-e.d4ddddZ/dddddZ0dS)FAISSuFAISS vector store integration. See [The FAISS Library](https://arxiv.org/pdf/2401.08281) paper. Setup: Install ``langchain_community`` and ``faiss-cpu`` python packages. .. code-block:: bash pip install -qU langchain_community faiss-cpu Key init args — indexing params: embedding_function: Embeddings Embedding function to use. Key init args — client params: index: Any FAISS index to use. docstore: Docstore Docstore to use. index_to_docstore_id: Dict[int, str] Mapping of index to docstore id. Instantiate: .. code-block:: python import faiss from langchain_community.vectorstores import FAISS from langchain_community.docstore.in_memory import InMemoryDocstore from langchain_openai import OpenAIEmbeddings index = faiss.IndexFlatL2(len(OpenAIEmbeddings().embed_query("hello world"))) vector_store = FAISS( embedding_function=OpenAIEmbeddings(), index=index, docstore= InMemoryDocstore(), index_to_docstore_id={} ) Add Documents: .. code-block:: python from langchain_core.documents import Document document_1 = Document(page_content="foo", metadata={"baz": "bar"}) document_2 = Document(page_content="thud", metadata={"bar": "baz"}) document_3 = Document(page_content="i will be deleted :(") documents = [document_1, document_2, document_3] ids = ["1", "2", "3"] vector_store.add_documents(documents=documents, ids=ids) Delete Documents: .. code-block:: python vector_store.delete(ids=["3"]) Search: .. code-block:: python results = vector_store.similarity_search(query="thud",k=1) for doc in results: print(f"* {doc.page_content} [{doc.metadata}]") .. code-block:: python * thud [{'bar': 'baz'}] Search with filter: .. code-block:: python results = vector_store.similarity_search(query="thud",k=1,filter={"bar": "baz"}) for doc in results: print(f"* {doc.page_content} [{doc.metadata}]") .. code-block:: python * thud [{'bar': 'baz'}] Search with score: .. code-block:: python results = vector_store.similarity_search_with_score(query="qux",k=1) for doc, score in results: print(f"* [SIM={score:3f}] {doc.page_content} [{doc.metadata}]") .. code-block:: python * [SIM=0.335304] foo [{'baz': 'bar'}] Async: .. code-block:: python # add documents # await vector_store.aadd_documents(documents=documents, ids=ids) # delete documents # await vector_store.adelete(ids=["3"]) # search # results = vector_store.asimilarity_search(query="thud",k=1) # search with score results = await vector_store.asimilarity_search_with_score(query="qux",k=1) for doc,score in results: print(f"* [SIM={score:3f}] {doc.page_content} [{doc.metadata}]") .. code-block:: python * [SIM=0.335304] foo [{'baz': 'bar'}] Use as Retriever: .. code-block:: python retriever = vector_store.as_retriever( search_type="mmr", search_kwargs={"k": 1, "fetch_k": 2, "lambda_mult": 0.5}, ) retriever.invoke("thud") .. code-block:: python [Document(metadata={'bar': 'baz'}, page_content='thud')] NFz/Union[Callable[[str], List[float]], Embeddings]rrzDict[int, str]z"Optional[Callable[[float], float]]rr)embedding_functionindexdocstoreindex_to_docstore_idrelevance_score_fn normalize_L2distance_strategycCsft|tstd||_||_||_||_||_||_ ||_ |jt j krb|j rbt d|jdS)z%Initialize with necessary components.t`embedding_function` is expected to be an Embeddings object, support for passing in a function will soon be removed.z2Normalizing L2 is not applicable for metric type: N)r+rloggerwarningr0r1r2r3r6override_relevance_score_fn _normalize_L2rEUCLIDEAN_DISTANCEwarningswarn)selfr0r1r2r3r4r5r6r"r"r#__init__s(  zFAISS.__init__zOptional[Embeddings])rcCst|jtr|jSdSN)r+r0rr?r"r"r# embeddingss zFAISS.embeddingsz List[str]zList[List[float]])textsrcs.tjtrj|Sfdd|DSdS)Ncsg|]}|qSr")r0).0textrBr"r# z*FAISS._embed_documents..)r+r0rembed_documentsr?rDr"rBr#_embed_documentss  zFAISS._embed_documentscs*t|jtr|j|IdHStddSNr7)r+r0raembed_documents ExceptionrJr"r"r#_aembed_documentss  zFAISS._aembed_documentsr%z List[float])rFrcCs&t|jtr|j|S||SdSrA)r+r0rZ embed_queryr?rFr"r"r# _embed_querys  zFAISS._embed_querycs*t|jtr|j|IdHStddSrL)r+r0rZ aembed_queryrNrPr"r"r# _aembed_querys  zFAISS._aembed_queryz Iterable[str]zIterable[List[float]]zOptional[Iterable[dict]]zOptional[List[str]])rDrC metadatasidsrc s t}t|jts$td|jdt||dd|pBdd|D}t||dd|pbdd |D}d dt|||D}t||d d |rt|tt|krtd t j |t j d}|j r| ||j||jddt||Dt|jfddt|D} |j| |S)NzSIf trying to add texts, the underlying docstore should support adding items, which z does notrDrScSsg|]}ttqSr")r%uuiduuid4rE_r"r"r#rG(rHzFAISS.__add..rTcss|] }iVqdSrAr"rWr"r"r# +rHzFAISS.__add..cSs g|]\}}}t|||dqS))idZ page_contentmetadatar)rEid_tmr"r"r#rG,s documentsrCz$Duplicate ids found in the ids list.ZdtypecSsi|]\}}||qSr"r")rEr\docr"r"r# <rHzFAISS.__add..csi|]\}}||qSr"r")rEjr\ starting_lenr"r#rb>rH)r$r+r2rr-r.zipr,setnparrayfloat32r;r5r1addr3 enumerateupdate) r?rDrCrSrTr Z _metadatasr_vector index_to_idr"rdr#Z__adds4      z FAISS.__addzOptional[List[dict]])rDrSrTkwargsrcKs$t|}||}|j||||dS)alRun more texts through the embeddings and add to the vectorstore. Args: texts: Iterable of strings to add to the vectorstore. metadatas: Optional list of metadatas associated with the texts. ids: Optional list of unique IDs. Returns: List of ids from adding the texts into the vectorstore. rSrT)listrK _FAISS__addr?rDrSrTrprCr"r"r# add_textsBs zFAISS.add_textscs*t|}||IdH}|j||||dS)aRun more texts through the embeddings and add to the vectorstore asynchronously. Args: texts: Iterable of strings to add to the vectorstore. metadatas: Optional list of metadatas associated with the texts. ids: Optional list of unique IDs. Returns: List of ids from adding the texts into the vectorstore. Nrq)rrrOrsrtr"r"r# aadd_textsWszFAISS.aadd_textsz!Iterable[Tuple[str, List[float]]])text_embeddingsrSrTrprcKst|\}}|j||||dS)aAdd the given texts and embeddings to the vectorstore. Args: text_embeddings: Iterable pairs of string and embedding to add to the vectorstore. metadatas: Optional list of metadatas associated with the texts. ids: Optional list of unique IDs. Returns: List of ids from adding the texts into the vectorstore. rq)rfrs)r?rwrSrTrprDrCr"r"r#add_embeddingsms zFAISS.add_embeddingsintz)Optional[Union[Callable, Dict[str, Any]]]zList[Tuple[Document, float]]) embeddingkfilterfetch_krprc sDt}tj|gtjd}|jr(|||j||dur<|n|\}} g} |dur\||} t | dD]\} } | dkrzqh|j | }|j |}t |t std|d||dur| |jr| ||d| fqh| ||d| fqh|ddur8|jtjtjfvrtjntjfdd | D} | d|S) aReturn docs most similar to query. Args: embedding: Embedding vector to look up documents similar to. k: Number of Documents to return. Defaults to 4. filter (Optional[Union[Callable, Dict[str, Any]]]): Filter by metadata. Defaults to None. If a callable, it must take as input the metadata dict of Document and return a bool. fetch_k: (Optional[int]) Number of Documents to fetch before filtering. Defaults to 20. **kwargs: kwargs to be passed to similarity search. Can include: score_threshold: Optional, a floating point value between 0 to 1 to filter the resulting set of retrieved docs Returns: List of documents most similar to the query text and L2 distance in float for each. Lower score represents more similarity. r`NrCould not find document for id , got score_thresholdcs"g|]\}}|r||fqSr"r")rEraZ similaritycmprr"r#rGs z@FAISS.similarity_search_with_score_by_vector..)r$rhrirjr;r5r1search_create_filter_funcrlr3r2r+rr-r[appendgetr6rMAX_INNER_PRODUCTZJACCARDoperatorgele)r?r|r}r~rrpr rnscoresindicesdocs filter_funcrci_idrar"rr#&similarity_search_with_score_by_vectors>           z,FAISS.similarity_search_with_score_by_vectorcs$td|j|f|||d|IdHS)aReturn docs most similar to query asynchronously. Args: embedding: Embedding vector to look up documents similar to. k: Number of Documents to return. Defaults to 4. filter (Optional[Dict[str, Any]]): Filter by metadata. Defaults to None. If a callable, it must take as input the metadata dict of Document and return a bool. fetch_k: (Optional[int]) Number of Documents to fetch before filtering. Defaults to 20. **kwargs: kwargs to be passed to similarity search. Can include: score_threshold: Optional, a floating point value between 0 to 1 to filter the resulting set of retrieved docs Returns: List of documents most similar to the query text and L2 distance in float for each. Lower score represents more similarity. Nr}r~r)rr)r?r|r}r~rrpr"r"r#'asimilarity_search_with_score_by_vectorsz-FAISS.asimilarity_search_with_score_by_vector)queryr}r~rrprcKs(||}|j||f||d|}|S)aReturn docs most similar to query. Args: query: Text to look up documents similar to. k: Number of Documents to return. Defaults to 4. filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. If a callable, it must take as input the metadata dict of Document and return a bool. fetch_k: (Optional[int]) Number of Documents to fetch before filtering. Defaults to 20. Returns: List of documents most similar to the query text with L2 distance in float. Lower score represents more similarity. r~r)rQrr?rr}r~rrpr|rr"r"r#similarity_search_with_scores z"FAISS.similarity_search_with_scorecs4||IdH}|j||f||d|IdH}|S)aReturn docs most similar to query asynchronously. Args: query: Text to look up documents similar to. k: Number of Documents to return. Defaults to 4. filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. If a callable, it must take as input the metadata dict of Document and return a bool. fetch_k: (Optional[int]) Number of Documents to fetch before filtering. Defaults to 20. Returns: List of documents most similar to the query text with L2 distance in float. Lower score represents more similarity. Nr)rRrrr"r"r#asimilarity_search_with_score s z#FAISS.asimilarity_search_with_scorezOptional[Dict[str, Any]]zList[Document]cKs(|j||f||d|}dd|DS)aYReturn docs most similar to embedding vector. Args: embedding: Embedding to look up documents similar to. k: Number of Documents to return. Defaults to 4. filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. If a callable, it must take as input the metadata dict of Document and return a bool. fetch_k: (Optional[int]) Number of Documents to fetch before filtering. Defaults to 20. Returns: List of Documents most similar to the embedding. rcSsg|] \}}|qSr"r"rErarXr"r"r#rGMrHz5FAISS.similarity_search_by_vector..)rr?r|r}r~rrpdocs_and_scoresr"r"r#similarity_search_by_vector/sz!FAISS.similarity_search_by_vectorcs.|j||f||d|IdH}dd|DS)ahReturn docs most similar to embedding vector asynchronously. Args: embedding: Embedding to look up documents similar to. k: Number of Documents to return. Defaults to 4. filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. If a callable, it must take as input the metadata dict of Document and return a bool. fetch_k: (Optional[int]) Number of Documents to fetch before filtering. Defaults to 20. Returns: List of Documents most similar to the embedding. rNcSsg|] \}}|qSr"r"rr"r"r#rGmrHz6FAISS.asimilarity_search_by_vector..)rrr"r"r#asimilarity_search_by_vectorOs z"FAISS.asimilarity_search_by_vectorcKs(|j||f||d|}dd|DS)aReturn docs most similar to query. Args: query: Text to look up documents similar to. k: Number of Documents to return. Defaults to 4. filter: (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. fetch_k: (Optional[int]) Number of Documents to fetch before filtering. Defaults to 20. Returns: List of Documents most similar to the query. rcSsg|] \}}|qSr"r"rr"r"r#rGrHz+FAISS.similarity_search..)rr?rr}r~rrprr"r"r#similarity_searchoszFAISS.similarity_searchcs.|j||f||d|IdH}dd|DS)aReturn docs most similar to query asynchronously. Args: query: Text to look up documents similar to. k: Number of Documents to return. Defaults to 4. filter: (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. fetch_k: (Optional[int]) Number of Documents to fetch before filtering. Defaults to 20. Returns: List of Documents most similar to the query. rNcSsg|] \}}|qSr"r"rr"r"r#rGrHz,FAISS.asimilarity_search..)rrr"r"r#asimilarity_searchs zFAISS.asimilarity_search?r}r lambda_multr~float)r|r}rrr~rcs`jtj|gtjd|dur"|n|d\}}|dur|}g} |dD]V} | dkr\qNj| } j| } t| t st d| d| || j rN| | qNt| g}fdd |dD} t tj|gtjd| ||d }g}|D]l} |d| dkrqj|d| } j| } t| t sDt d| d| | | |d| fq|S) azReturn docs and their similarity scores selected using the maximal marginal relevance. Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents. Args: embedding: Embedding to look up documents similar to. k: Number of Documents to return. Defaults to 4. fetch_k: Number of Documents to fetch before filtering to pass to MMR algorithm. lambda_mult: Number between 0 and 1 that determines the degree of diversity among the results with 0 corresponding to maximum diversity and 1 to minimum diversity. Defaults to 0.5. Returns: List of Documents and similarity scores selected by maximal marginal relevance and score for each. r`Nrrrrcs$g|]}|dkrjt|qS)r)r1Z reconstructr{)rErrBr"r#rGrHzLFAISS.max_marginal_relevance_search_with_score_by_vector..)r}r)r1rrhrirjrr3r2r+rr-r[rr)r?r|r}rrr~rrrZfiltered_indicesrrrarCZ mmr_selectedrr"rBr#2max_marginal_relevance_search_with_score_by_vectorsD          z8FAISS.max_marginal_relevance_search_with_score_by_vectorc std|j|||||dIdHS)aReturn docs and their similarity scores selected using the maximal marginal relevance asynchronously. Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents. Args: embedding: Embedding to look up documents similar to. k: Number of Documents to return. Defaults to 4. fetch_k: Number of Documents to fetch before filtering to pass to MMR algorithm. lambda_mult: Number between 0 and 1 that determines the degree of diversity among the results with 0 corresponding to maximum diversity and 1 to minimum diversity. Defaults to 0.5. Returns: List of Documents and similarity scores selected by maximal marginal relevance and score for each. Nr)rr)r?r|r}rrr~r"r"r#3amax_marginal_relevance_search_with_score_by_vectorsz9FAISS.amax_marginal_relevance_search_with_score_by_vector)r|r}rrr~rprcKs"|j|||||d}dd|DS)aReturn docs selected using the maximal marginal relevance. Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents. Args: embedding: Embedding to look up documents similar to. k: Number of Documents to return. Defaults to 4. fetch_k: Number of Documents to fetch before filtering to pass to MMR algorithm. lambda_mult: Number between 0 and 1 that determines the degree of diversity among the results with 0 corresponding to maximum diversity and 1 to minimum diversity. Defaults to 0.5. Returns: List of Documents selected by maximal marginal relevance. rcSsg|] \}}|qSr"r"rr"r"r#rG)rHzAFAISS.max_marginal_relevance_search_by_vector..)rr?r|r}rrr~rprr"r"r#'max_marginal_relevance_search_by_vector s z-FAISS.max_marginal_relevance_search_by_vectorcs(|j|||||dIdH}dd|DS)a(Return docs selected using the maximal marginal relevance asynchronously. Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents. Args: embedding: Embedding to look up documents similar to. k: Number of Documents to return. Defaults to 4. fetch_k: Number of Documents to fetch before filtering to pass to MMR algorithm. lambda_mult: Number between 0 and 1 that determines the degree of diversity among the results with 0 corresponding to maximum diversity and 1 to minimum diversity. Defaults to 0.5. Returns: List of Documents selected by maximal marginal relevance. rNcSsg|] \}}|qSr"r"rr"r"r#rGJrHzBFAISS.amax_marginal_relevance_search_by_vector..)rrr"r"r#(amax_marginal_relevance_search_by_vector+s  z.FAISS.amax_marginal_relevance_search_by_vector)rr}rrr~rprc Ks*||}|j|f||||d|}|S)aReturn docs selected using the maximal marginal relevance. Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents. Args: query: Text to look up documents similar to. k: Number of Documents to return. Defaults to 4. fetch_k: Number of Documents to fetch before filtering (if needed) to pass to MMR algorithm. lambda_mult: Number between 0 and 1 that determines the degree of diversity among the results with 0 corresponding to maximum diversity and 1 to minimum diversity. Defaults to 0.5. Returns: List of Documents selected by maximal marginal relevance. r)rQr r?rr}rrr~rpr|rr"r"r#max_marginal_relevance_searchLs z#FAISS.max_marginal_relevance_searchc s6||IdH}|j|f||||d|IdH}|S)a+Return docs selected using the maximal marginal relevance asynchronously. Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents. Args: query: Text to look up documents similar to. k: Number of Documents to return. Defaults to 4. fetch_k: Number of Documents to fetch before filtering (if needed) to pass to MMR algorithm. lambda_mult: Number between 0 and 1 that determines the degree of diversity among the results with 0 corresponding to maximum diversity and 1 to minimum diversity. Defaults to 0.5. Returns: List of Documents selected by maximal marginal relevance. Nr)rRrrr"r"r#amax_marginal_relevance_searchqs z$FAISS.amax_marginal_relevance_searchr)rTrprc s|durtdt||j}|r6td|dd|jDfdd|D|jtj tj d|j |fd d t |jD}d dt|D|_d S) zDelete by ID. These are the IDs in the vectorstore. Args: ids: List of ids to delete. Returns: Optional[bool]: True if deletion is successful, False otherwise, None if not implemented. NzNo ids provided to delete.zESome specified ids do not exist in the current store. Ids not found: cSsi|]\}}||qSr"r")rEidxr\r"r"r#rbrHz FAISS.delete..csh|] }|qSr"r"rEr\)reversed_indexr"r# rHzFAISS.delete..r`csg|]\}}|vr|qSr"r"rErr\)index_to_deleter"r#rGsz FAISS.delete..cSsi|]\}}||qSr"r"rr"r"r#rbrHT)r-rg differencer3valuesitemsr1Z remove_idsrhZfromiterZint64r2deletesortedrl)r?rTrpZ missing_idsZ remaining_idsr")rrr#rs$    z FAISS.deleter&)targetrcCst|jtstdt|j}|j|jg}|jD]:\}}|j |}t|t s`td| ||||fq:|j dd|Ddd|D}|j |dS)zMerge another FAISS object with the current one. Add the target FAISS to the current one. Args: target: FAISS object you wish to merge into the current one Returns: None. z'Cannot merge with this type of docstorezDocument should be returnedcSsi|]\}}}||qSr"r")rErXrrar"r"r#rbrHz$FAISS.merge_from..cSsi|]\}}}||qSr"r")rEr1rrXr"r"r#rbrHN)r+r2rr-r,r3r1 merge_fromrrrrrkrm)r?rreZ full_inforZ target_idraror"r"r#rs    zFAISS.merge_fromr) rDrCr|rSrTr5r6rprcKst} |tjkr$| t|d} n| t|d} |dt} |di} ||| | | f||d|} | j||||d| S)Nrr2r3)r5r6rq) r$rrZ IndexFlatIPr,Z IndexFlatL2poprrs)clsrDrCr|rSrTr5r6rpr r1r2r3Zvecstorer"r"r#Z__froms&    z FAISS.__from)rDr|rSrTrprcKs&||}|j|||f||d|S)aOConstruct FAISS wrapper from raw documents. This is a user friendly interface that: 1. Embeds documents. 2. Creates an in memory docstore 3. Initializes the FAISS database This is intended to be a quick way to get started. Example: .. code-block:: python from langchain_community.vectorstores import FAISS from langchain_community.embeddings import OpenAIEmbeddings embeddings = OpenAIEmbeddings() faiss = FAISS.from_texts(texts, embeddings) rq)rI _FAISS__fromrrDr|rSrTrprCr"r"r# from_textss zFAISS.from_textsz list[str]cs,||IdH}|j|||f||d|S)aeConstruct FAISS wrapper from raw documents asynchronously. This is a user friendly interface that: 1. Embeds documents. 2. Creates an in memory docstore 3. Initializes the FAISS database This is intended to be a quick way to get started. Example: .. code-block:: python from langchain_community.vectorstores import FAISS from langchain_community.embeddings import OpenAIEmbeddings embeddings = OpenAIEmbeddings() faiss = await FAISS.afrom_texts(texts, embeddings) Nrq)rMrrr"r"r# afrom_textsszFAISS.afrom_texts)rwr|rSrTrprcKs0t|\}}|jt|t||f||d|S)aConstruct FAISS wrapper from raw documents. This is a user friendly interface that: 1. Embeds documents. 2. Creates an in memory docstore 3. Initializes the FAISS database This is intended to be a quick way to get started. Example: .. code-block:: python from langchain_community.vectorstores import FAISS from langchain_community.embeddings import OpenAIEmbeddings embeddings = OpenAIEmbeddings() text_embeddings = embeddings.embed_documents(texts) text_embedding_pairs = zip(texts, text_embeddings) faiss = FAISS.from_embeddings(text_embedding_pairs, embeddings) rq)rfrrr)rrwr|rSrTrprDrCr"r"r#from_embeddingsBs zFAISS.from_embeddingscs|j||f||d|S)z:Construct FAISS wrapper from raw documents asynchronously.rq)r)rrwr|rSrTrpr"r"r#afrom_embeddingsis zFAISS.afrom_embeddingsr1) folder_path index_namercCst|}|jdddt}||jt||dt||dd$}t|j |j f|Wdn1sv0YdS)aSave FAISS index, docstore, and index_to_docstore_id to disk. Args: folder_path: folder path to save index, docstore, and index_to_docstore_id to. index_name: for saving with a specific index file name T)exist_okparents.faiss.pklwbN) rmkdirr$Z write_indexr1r%openpickledumpr2r3)r?rrpathr fr"r"r# save_local{s zFAISS.save_local)allow_dangerous_deserialization)rrCrrrprc Ks|s tdt|}t}|t||d}t||dd} t| \} } Wdn1sj0Y|||| | fi|S)aLoad FAISS index, docstore, and index_to_docstore_id from disk. Args: folder_path: folder path to load index, docstore, and index_to_docstore_id from. embeddings: Embeddings to use when generating queries index_name: for saving with a specific index file name allow_dangerous_deserialization: whether to allow deserialization of the data which involves loading a pickle file. Pickle files can be modified by malicious actors to deliver a malicious payload that results in execution of arbitrary code on your machine. BThe de-serialization relies loading a pickle file. Pickle files can be modified to deliver a malicious payload that results in execution of arbitrary code on your machine.You will need to set `allow_dangerous_deserialization` to `True` to enable deserialization. If you do this, make sure that you trust the source of the data. For example, if you are loading a file that you created, and know that no one else has modified the file, then this is safe to do. Do not set this to `True` if you are loading a file from an untrusted source (e.g., some random site on the internet.).rrrbN)r-rr$Z read_indexr%rrload) rrrCrrrprr r1rr2r3r"r"r# load_locals  zFAISS.load_localbytescCst|j|j|jfS)zCSerialize FAISS index, docstore, and index_to_docstore_id to bytes.)rdumpsr1r2r3rBr"r"r#serialize_to_bytesszFAISS.serialize_to_bytes) serializedrCrrprcKs2|s tdt|\}}}|||||fi|S)zGDeserialize FAISS index, docstore, and index_to_docstore_id from bytes.r)r-rloads)rrrCrrpr1r2r3r"r"r#deserialize_from_bytess zFAISS.deserialize_from_byteszCallable[[float], float]cCsR|jdur|jS|jtjkr"|jS|jtjkr4|jS|jtjkrF|jSt ddS)a8 The 'correct' relevance function may differ depending on a few things, including: - the distance / similarity metric used by the VectorStore - the scale of your embeddings (OpenAI's are unit normed. Many others are not!) - embedding dimensionality - etc. NzJUnknown distance strategy, must be cosine, max_inner_product, or euclidean) r:r6rrZ%_max_inner_product_relevance_score_fnr<Z_euclidean_relevance_score_fnZCOSINEZ_cosine_relevance_score_fnr-rBr"r"r#_select_relevance_score_fns    z FAISS._select_relevance_score_fnc sH|durtd|j|f|||d|}fdd|D}|S)?Return docs and their similarity scores on a scale from 0 to 1.NLrelevance_score_fn must be provided to FAISS constructor to normalize scoresrcsg|]\}}||fqSr"r"rEraZscorer4r"r#rGszBFAISS._similarity_search_with_relevance_scores..)rr-rr?rr}r~rrprZdocs_and_rel_scoresr"rr#(_similarity_search_with_relevance_scoress$  z.FAISS._similarity_search_with_relevance_scorescsN|durtd|j|f|||d|IdH}fdd|D}|S)rNrrcsg|]\}}||fqSr"r"rrr"r#rG6szCFAISS._asimilarity_search_with_relevance_scores..)rr-rrr"rr#)_asimilarity_search_with_relevance_scoress$   z/FAISS._asimilarity_search_with_relevance_scores Callable[[Dict[str, Any]], bool]r~rc st|r |St|ts(tdt|ddlm}m}m}m }m }m }||||||d}ddddd}||Bt t gd } d |D](} | r| d r| | vrtd | qd dddfdd dddfdd |S)a Create a filter function based on the provided filter. Args: filter: A callable or a dictionary representing the filter conditions for documents. Returns: A function that takes Document's metadata and returns True if it satisfies the filter conditions, otherwise False. Raises: ValueError: If the filter is invalid or contains unsuported operators. z5filter must be a dict of metadata or a callable, not r)eqrgtrltne)z$eqz$neqz$gtz$ltz$gtez$ltecSs||vSrAr"abr"r"r#`rHz+FAISS._create_filter_func..cSs||vSrAr"rr"r"r#rarH)z$inz$nin)$and$or$not $&filter contains unsupported operator: r%z%Union[Dict[str, Any], List[Any], Any]r)field conditionrcsttrbgD]0\}}|vr4td|||fqdddfdd }|SttrtkrtfddSfd dSfd dS) a Creates a filter function based on field and condition. Args: field: The document field to filter on condition: Filter condition (dict for operators, list for in, or direct value for equality) Returns: A filter function that takes a document and returns boolean rDict[str, Any]r)rarcs |tfddDS)aW Evaluates a document against a set of predefined operators and their values. This function applies multiple comparison/sequence operators to a specific field value from the document. All conditions must be satisfied for the function to return True. Args: doc (Dict[str, Any]): The document to evaluate, containing key-value pairs where keys are field names and values are the field values. The document must contain the field being filtered. Returns: bool: True if the document's field value satisfies all operator conditions, False otherwise. c3s|]\}}||VqdSrAr")rEopvalueZ doc_valuer"r#rYrHzYFAISS._create_filter_func..filter_func_cond..filter_fn..)rallra)r operatorsrr# filter_fns zFFAISS._create_filter_func..filter_func_cond..filter_fncs|vSrArr) condition_setrr"r#rrHzEFAISS._create_filter_func..filter_func_cond..cs|vSrArrrrr"r#rrHcs|kSrArrrr"r#rrH)r+dictrr-rrrr, frozenset)rrrrr) OPERATIONSSET_CONVERT_THRESHOLD)rrrrr#filter_func_condls   z3FAISS._create_filter_func..filter_func_condrrcsd|vr*fdd|dDfddSd|vrTfdd|dDfddSd |vrt|d fd dSfd d|Dfd dS) a Creates a filter function that evaluates documents against specified filter conditions. This function processes a dictionary of filter conditions and returns a callable that can evaluate documents against these conditions. It supports logical operators ($and, $or, $not) and field-level filtering. Args: filter (Dict[str, Any]): A dictionary containing filter conditions. Can include: - Logical operators ($and, $or, $not) with lists of sub-filters - Field-level conditions with comparison or sequence operators - Direct field-value mappings for equality comparison Returns: Callable[[Dict[str, Any]], bool]: A function that takes a document (as a dictionary) and returns True if the document matches all filter conditions, False otherwise. rcsg|] }|qSr"r"rEZ sub_filterrr"r#rGrHzBFAISS._create_filter_func..filter_func..cstfddDS)Nc3s|]}|VqdSrAr"rErrr"r#rYrHSFAISS._create_filter_func..filter_func....rrfiltersrr#rrHz@FAISS._create_filter_func..filter_func..rcsg|] }|qSr"r"r r r"r#rGrHcstfddDS)Nc3s|]}|VqdSrAr"r rr"r#rYrHr )anyrrrr#rrHrcs | SrAr"r)condr"r#rrHcsg|]\}}||qSr"r")rErr)rr"r#rGscstfddDS)Nc3s|]}|VqdSrAr")rErrr"r#rYrHr r r) conditionsrr#rrH)r)r~)rr)rrrr#rs     z.FAISS._create_filter_func..filter_func)callabler+rr-typerrrrrrrrrr startswith) r~rrrrrrZCOMPARISON_OPERATORSZSEQUENCE_OPERATORSZVALID_OPERATORSrr")rrrrr#r;s4    4'zFAISS._create_filter_funcz Sequence[str]zlist[Document])rTrcs fdd|D}dd|DS)Ncsg|]}j|qSr")r2rrrBr"r#rGrHz$FAISS.get_by_ids..cSsg|]}t|tr|qSr")r+r)rErar"r"r#rGrHr")r?rTrr"rBr# get_by_idsszFAISS.get_by_ids)NN)NN)NN)NN)ryNrz)ryNrz)ryNrz)ryNrz)ryNrz)ryNrz)ryNrz)ryNrz)ryrzrN)ryrzrN)ryrzrN)ryrzrN)N)NN)NN)NN)NN)r1)r1)ryNrz)ryNrz)1__name__ __module__ __qualname____doc__rr<r@propertyrCrKrOrQrRrsrurvrxrrrrrrrrrrrrrrrr classmethodrrrrrrrrrrrr staticmethodrrr"r"r"r#r/Ks  # -D*%%##H*"$(%" "$$&3 r/)N)- __future__rloggingrrrrUr=pathlibrtypingrrrrrr r r r r numpyrhZlangchain_core.documentsrZlangchain_core.embeddingsrZlangchain_core.runnables.configrZlangchain_core.vectorstoresrZ!langchain_community.docstore.baserrZ&langchain_community.docstore.in_memoryrZ&langchain_community.vectorstores.utilsrr getLoggerrr8r$r.r/r"r"r"r#s(  0