tj0ddlmZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z m Z m Z mZmZmZmZmZmZmZddlZddlmZddlmZddlmZddlmZdd lm Z m!Z!dd l"m#Z#dd l$m%Z%m&Z&ej'e(Z)dddZ*ddZ+GddeZ,dS)) annotationsN)Path) AnyCallableDictIterableListOptionalSequenceSizedTupleUnionDocument) Embeddings)run_in_executor) VectorStore) AddableMixinDocstore)InMemoryDocstore)DistanceStrategymaximal_marginal_relevanceno_avx2Optional[bool]returnrc|/dtjvr!ttjd} |rddlm}nddl}n#t $rt dwxYw|S)aM Import faiss if available, otherwise raise error. If FAISS_NO_AVX2 environment variable is set, it will be considered to load FAISS with no AVX2 optimization. Args: no_avx2: Load FAISS strictly with no AVX2 optimization so that the vectorstore is portable and compatible with other devices. N FAISS_NO_AVX2r) swigfaisszCould not import faiss python package. Please install it with `pip install faiss-gpu` (for CUDA supported GPU) or `pip install faiss-cpu` (depending on Python version).)osenvironboolgetenvfaissr ImportError)rr#s d/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/langchain_community/vectorstores/faiss.pydependable_faiss_importr&'s?bj88ry1122    0 0 0 0 0 0 0 LLL     H    Ls AAxyx_namestry_nameNonect|trot|trZt|t|kr:t|d|d|dt|d|dt| dS)Nz and z% expected to be equal length but len(z)=z and len() isinstancer len ValueError)r'r(r)r+s r%_len_check_if_sizedr1Bs!U 1e 4 4 Q3q669I9I A AF A A A A VV A A.4 A A8;A A A    FceZdZdZddejfdodZedpdZdqdZ dqdZ drdZ drdZ dsdtd'Z dsdud*Z dsdud+Z dsdvd.Z dwdxd8Z dwdxd9Z dwdyd;Z dwdyd<Z dwdzd?Z dwd{d@Z dwd|dAZ dwd|dBZd/d0dCddDd}dGZd/d0dCddDd}dHZ d~ddIZ d~ddJZ d~ddKZ d~ddLZdddNZddQZ e!dddejfddSZ"e! dsddTZ#e! dsddVZ$e! dsddWZ%e! dsddXZ&ddd[Z'e! ddd\d]dd`Z(ddbZ)e!ddcddeZ*ddgZ+ dwdydhZ, dwdydiZ-e.ddkZ/ddnZ0dS)FAISSuFAISS vector store integration. See [The FAISS Library](https://arxiv.org/pdf/2401.08281) paper. Setup: Install ``langchain_community`` and ``faiss-cpu`` python packages. .. code-block:: bash pip install -qU langchain_community faiss-cpu Key init args — indexing params: embedding_function: Embeddings Embedding function to use. Key init args — client params: index: Any FAISS index to use. docstore: Docstore Docstore to use. index_to_docstore_id: Dict[int, str] Mapping of index to docstore id. Instantiate: .. code-block:: python import faiss from langchain_community.vectorstores import FAISS from langchain_community.docstore.in_memory import InMemoryDocstore from langchain_openai import OpenAIEmbeddings index = faiss.IndexFlatL2(len(OpenAIEmbeddings().embed_query("hello world"))) vector_store = FAISS( embedding_function=OpenAIEmbeddings(), index=index, docstore= InMemoryDocstore(), index_to_docstore_id={} ) Add Documents: .. code-block:: python from langchain_core.documents import Document document_1 = Document(page_content="foo", metadata={"baz": "bar"}) document_2 = Document(page_content="thud", metadata={"bar": "baz"}) document_3 = Document(page_content="i will be deleted :(") documents = [document_1, document_2, document_3] ids = ["1", "2", "3"] vector_store.add_documents(documents=documents, ids=ids) Delete Documents: .. code-block:: python vector_store.delete(ids=["3"]) Search: .. code-block:: python results = vector_store.similarity_search(query="thud",k=1) for doc in results: print(f"* {doc.page_content} [{doc.metadata}]") .. code-block:: python * thud [{'bar': 'baz'}] Search with filter: .. code-block:: python results = vector_store.similarity_search(query="thud",k=1,filter={"bar": "baz"}) for doc in results: print(f"* {doc.page_content} [{doc.metadata}]") .. code-block:: python * thud [{'bar': 'baz'}] Search with score: .. code-block:: python results = vector_store.similarity_search_with_score(query="qux",k=1) for doc, score in results: print(f"* [SIM={score:3f}] {doc.page_content} [{doc.metadata}]") .. code-block:: python * [SIM=0.335304] foo [{'baz': 'bar'}] Async: .. code-block:: python # add documents # await vector_store.aadd_documents(documents=documents, ids=ids) # delete documents # await vector_store.adelete(ids=["3"]) # search # results = vector_store.asimilarity_search(query="thud",k=1) # search with score results = await vector_store.asimilarity_search_with_score(query="qux",k=1) for doc,score in results: print(f"* [SIM={score:3f}] {doc.page_content} [{doc.metadata}]") .. code-block:: python * [SIM=0.335304] foo [{'baz': 'bar'}] Use as Retriever: .. code-block:: python retriever = vector_store.as_retriever( search_type="mmr", search_kwargs={"k": 1, "fetch_k": 2, "lambda_mult": 0.5}, ) retriever.invoke("thud") .. code-block:: python [Document(metadata={'bar': 'baz'}, page_content='thud')] NFembedding_function/Union[Callable[[str], List[float]], Embeddings]indexrdocstorerindex_to_docstore_idDict[int, str]relevance_score_fn"Optional[Callable[[float], float]] normalize_L2r!distance_strategyrc>t|tstd||_||_||_||_||_||_ ||_ |jtj kr%|j r tjd|jdSdSdS)z%Initialize with necessary components.t`embedding_function` is expected to be an Embeddings object, support for passing in a function will soon be removed.z2Normalizing L2 is not applicable for metric type: N)r.rloggerwarningr5r7r8r9r>override_relevance_score_fn _normalize_L2rEUCLIDEAN_DISTANCEwarningswarn)selfr5r7r8r9r;r=r>s r%__init__zFAISS.__init__s,j99  NNB   #5   $8!!2+=()  "&6&I I I" J M9 $ 699      J I I Ir2rOptional[Embeddings]cHt|jtr|jndSN)r.r5rrHs r% embeddingszFAISS.embeddingss)$1:>> D # # r2texts List[str]List[List[float]]ctjtrj|Sfd|DS)Nc:g|]}|S)r5).0textrHs r% z*FAISS._embed_documents..s'DDDdD++D11DDDr2)r.r5rembed_documentsrHrOs` r%_embed_documentszFAISS._embed_documentssJ d-z : : E*::5AA ADDDDeDDD Dr2cKt|jtr |j|d{VSt dNr@)r.r5raembed_documents ExceptionrYs r%_aembed_documentszFAISS._aembed_documentss^ d-z : : 0AA%HHHHHHHH H B r2rVr* List[float]ct|jtr|j|S||SrL)r.r5r embed_queryrHrVs r% _embed_queryzFAISS._embed_querysB d-z : : 1*66t<< <**400 0r2cKt|jtr |j|d{VSt dr\)r.r5r aembed_queryr^rcs r% _aembed_queryzFAISS._aembed_querys^ d-z : : 0==dCCCCCCCC CB r2 Iterable[str]rNIterable[List[float]] metadatasOptional[Iterable[dict]]idsOptional[List[str]]c t}t|jtst d|jdt ||dd|p d|D}t ||dd|p d|D}dt |||D}t ||d d |r.(s&777Ac$*,,''777r2rlc3K|]}iVdSrLrTrrs r% zFAISS.__add..+s""5"5!2"5"5"5"5"5"5r2c:g|]\}}}t|||S))id page_contentmetadatar)rUid_tms r%rWzFAISS.__add..,s=   Q !a 8 8 8   r2 documentsrNz$Duplicate ids found in the ids list.dtypeci|]\}}|| SrTrT)rUrzdocs r% zFAISS.__add..<sHHHS3HHHr2c"i|] \}}|z| SrTrT)rUjrz starting_lens r%rzFAISS.__add..>s$JJJC|a'JJJr2)r&r.r8rr0r1zipr/setnparrayfloat32rDr=r7addr9 enumerateupdate) rHrOrNrjrlr# _metadatasr}vector index_to_idrs @r%__addz FAISS.__adds())$-66 @'+}@@@  E9g{CCC777777E37775"5"5u"5"5"5    eZ88   Iz; MMM  E3s88s3s88}},,CDD D*BJ777   '   v & & & v HHCY4G4GHHHIII4455 JJJJ9S>>JJJ  !((555 r2Optional[List[dict]]kwargsc |t|}||}|||||S)alRun more texts through the embeddings and add to the vectorstore. Args: texts: Iterable of strings to add to the vectorstore. metadatas: Optional list of metadatas associated with the texts. ids: Optional list of unique IDs. Returns: List of ids from adding the texts into the vectorstore. rjrl)listrZ _FAISS__addrHrOrjrlrrNs r% add_textszFAISS.add_textsBs<"U **511 zz%yczJJJr2cKt|}||d{V}|||||S)aRun more texts through the embeddings and add to the vectorstore asynchronously. Args: texts: Iterable of strings to add to the vectorstore. metadatas: Optional list of metadatas associated with the texts. ids: Optional list of unique IDs. Returns: List of ids from adding the texts into the vectorstore. Nr)rr_rrs r% aadd_textszFAISS.aadd_textsWsR$U 11%88888888 zz%yczJJJr2text_embeddings!Iterable[Tuple[str, List[float]]]c Lt|\}}|||||S)aAdd the given texts and embeddings to the vectorstore. Args: text_embeddings: Iterable pairs of string and embedding to add to the vectorstore. metadatas: Optional list of metadatas associated with the texts. ids: Optional list of unique IDs. Returns: List of ids from adding the texts into the vectorstore. r)rr)rHrrjrlrrOrNs r%add_embeddingszFAISS.add_embeddingsms-& 1zzz%yczJJJr2 embeddingkintfilter)Optional[Union[Callable, Dict[str, Any]]]fetch_kList[Tuple[Document, float]]c xt}tj|gtj}|jr|||j|||n|\}} g} |||} t| dD]\} } | dkr |j | }|j |}t|tstd|d||4| |jr#| ||d| f| ||d| f|dF|jt&jt&jfvr t,jn t,jfd| D} | d|S) aReturn docs most similar to query. Args: embedding: Embedding vector to look up documents similar to. k: Number of Documents to return. Defaults to 4. filter (Optional[Union[Callable, Dict[str, Any]]]): Filter by metadata. Defaults to None. If a callable, it must take as input the metadata dict of Document and return a bool. fetch_k: (Optional[int]) Number of Documents to fetch before filtering. Defaults to 20. **kwargs: kwargs to be passed to similarity search. Can include: score_threshold: Optional, a floating point value between 0 to 1 to filter the resulting set of retrieved docs Returns: List of documents most similar to the query text and L2 distance in float for each. Lower score represents more similarity. r~NrCould not find document for id , got score_thresholdc6g|]\}}|||fSrTrT)rUr similaritycmprs r%rWz@FAISS.similarity_search_with_score_by_vector..sD#C3z?33j!r2)r&rrrrDr=r7search_create_filter_funcrr9r8r.rr0ryappendgetr>rMAX_INNER_PRODUCTJACCARDoperatorgele)rHrrrrrr#rscoresindicesdocs filter_funcri_idrrrs @@r%&similarity_search_with_score_by_vectorz,FAISS.similarity_search_with_score_by_vectors4())9+RZ888   '   v & & &*++FAAWUU  226::Kgaj)) 1 1DAqBww+A.C-&&s++Cc8,, U !S3!S!Sc!S!STTT!;s|,,5KKfQil 3444 S&)A,/0000 **%677  &)$68H8PQRR [  '+D BQBxr2cBKtd|j|f|||d|d{VS)aReturn docs most similar to query asynchronously. Args: embedding: Embedding vector to look up documents similar to. k: Number of Documents to return. Defaults to 4. filter (Optional[Dict[str, Any]]): Filter by metadata. Defaults to None. If a callable, it must take as input the metadata dict of Document and return a bool. fetch_k: (Optional[int]) Number of Documents to fetch before filtering. Defaults to 20. **kwargs: kwargs to be passed to similarity search. Can include: score_threshold: Optional, a floating point value between 0 to 1 to filter the resulting set of retrieved docs Returns: List of documents most similar to the query text and L2 distance in float for each. Lower score represents more similarity. Nrrr)rr)rHrrrrrs r%'asimilarity_search_with_score_by_vectorz-FAISS.asimilarity_search_with_score_by_vectorsc:%   7              r2queryc T||}|j||f||d|}|S)aReturn docs most similar to query. Args: query: Text to look up documents similar to. k: Number of Documents to return. Defaults to 4. filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. If a callable, it must take as input the metadata dict of Document and return a bool. fetch_k: (Optional[int]) Number of Documents to fetch before filtering. Defaults to 20. Returns: List of documents most similar to the query text with L2 distance in float. Lower score represents more similarity. rr)rdrrHrrrrrrrs r%similarity_search_with_scorez"FAISS.similarity_search_with_scoresT0%%e,, :t:          r2cpK||d{V}|j||f||d|d{V}|S)aReturn docs most similar to query asynchronously. Args: query: Text to look up documents similar to. k: Number of Documents to return. Defaults to 4. filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. If a callable, it must take as input the metadata dict of Document and return a bool. fetch_k: (Optional[int]) Number of Documents to fetch before filtering. Defaults to 20. Returns: List of documents most similar to the query text with L2 distance in float. Lower score represents more similarity. Nr)rgrrs r%asimilarity_search_with_scorez#FAISS.asimilarity_search_with_score s0,,U33333333 ATA                r2Optional[Dict[str, Any]]List[Document]c >|j||f||d|}d|DS)aYReturn docs most similar to embedding vector. Args: embedding: Embedding to look up documents similar to. k: Number of Documents to return. Defaults to 4. filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. If a callable, it must take as input the metadata dict of Document and return a bool. fetch_k: (Optional[int]) Number of Documents to fetch before filtering. Defaults to 20. Returns: List of Documents most similar to the embedding. rcg|]\}}|SrTrTrUrrss r%rWz5FAISS.similarity_search_by_vector..M222Q222r2)rrHrrrrrdocs_and_scoress r%similarity_search_by_vectorz!FAISS.similarity_search_by_vector/sQ.F$E         32/2222r2cNK|j||f||d|d{V}d|DS)ahReturn docs most similar to embedding vector asynchronously. Args: embedding: Embedding to look up documents similar to. k: Number of Documents to return. Defaults to 4. filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. If a callable, it must take as input the metadata dict of Document and return a bool. fetch_k: (Optional[int]) Number of Documents to fetch before filtering. Defaults to 20. Returns: List of Documents most similar to the embedding. rNcg|]\}}|SrTrTrs r%rWz6FAISS.asimilarity_search_by_vector..mrr2)rrs r%asimilarity_search_by_vectorz"FAISS.asimilarity_search_by_vectorOss.!M L  !  ! !  ! !       32/2222r2c >|j||f||d|}d|DS)aReturn docs most similar to query. Args: query: Text to look up documents similar to. k: Number of Documents to return. Defaults to 4. filter: (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. fetch_k: (Optional[int]) Number of Documents to fetch before filtering. Defaults to 20. Returns: List of Documents most similar to the query. rcg|]\}}|SrTrTrs r%rWz+FAISS.similarity_search..rr2)rrHrrrrrrs r%similarity_searchzFAISS.similarity_searchosJ(<$; 1 #W  8>  32/2222r2cNK|j||f||d|d{V}d|DS)aReturn docs most similar to query asynchronously. Args: query: Text to look up documents similar to. k: Number of Documents to return. Defaults to 4. filter: (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. fetch_k: (Optional[int]) Number of Documents to fetch before filtering. Defaults to 20. Returns: List of Documents most similar to the query. rNcg|]\}}|SrTrTrs r%rWz,FAISS.asimilarity_search..rr2)rrs r%asimilarity_searchzFAISS.asimilarity_searchsm(!C B 1! #W! ! 8>! !       32/2222r2?rr lambda_multrrfloatcjtj|gtj||n|dz\}}||}g} |dD]} | dkr j| } j| } t| tstd| d| || j r| | tj| g}fd|dD} ttj|gtj| || }g}|D]} |d| dkrj|d| } j| } t| tstd| d| | | |d| f|S) azReturn docs and their similarity scores selected using the maximal marginal relevance. Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents. Args: embedding: Embedding to look up documents similar to. k: Number of Documents to return. Defaults to 4. fetch_k: Number of Documents to fetch before filtering to pass to MMR algorithm. lambda_mult: Number between 0 and 1 that determines the degree of diversity among the results with 0 corresponding to maximum diversity and 1 to minimum diversity. Defaults to 0.5. Returns: List of Documents and similarity scores selected by maximal marginal relevance and score for each. r~Nrrrrcjg|]/}|dkjt|0S)r)r7 reconstructr)rUrrHs r%rWzLFAISS.max_marginal_relevance_search_with_score_by_vector..s5TTTAQSGGdj,,SVV44GGGr2)rr)r7rrrrrr9r8r.rr0ryrr)rHrrrrrrrrfiltered_indicesrrrrN mmr_selectedrs` r%2max_marginal_relevance_search_with_score_by_vectorz8FAISS.max_marginal_relevance_search_with_score_by_vectors8*++ Hi[ 3 3 3~GG7Q;    226::K! QZ / /77/2m**3//!#x00Y$%Ws%W%WRU%W%WXXX;s|,,/$++A...h 0122GTTTTgajTTT 1 Hi[ 3 3 3 #     8 8Aqz!}""+GAJqM:C-&&s++Cc8,, U !S3!S!Sc!S!STTT  " "C1#6 7 7 7 7r2c HKtd|j|||||d{VS)aReturn docs and their similarity scores selected using the maximal marginal relevance asynchronously. Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents. Args: embedding: Embedding to look up documents similar to. k: Number of Documents to return. Defaults to 4. fetch_k: Number of Documents to fetch before filtering to pass to MMR algorithm. lambda_mult: Number between 0 and 1 that determines the degree of diversity among the results with 0 corresponding to maximum diversity and 1 to minimum diversity. Defaults to 0.5. Returns: List of Documents and similarity scores selected by maximal marginal relevance and score for each. Nr)rr)rHrrrrrs r%3amax_marginal_relevance_search_with_score_by_vectorz9FAISS.amax_marginal_relevance_search_with_score_by_vectorsU:%   C #          r2c N||||||}d|DS)aReturn docs selected using the maximal marginal relevance. Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents. Args: embedding: Embedding to look up documents similar to. k: Number of Documents to return. Defaults to 4. fetch_k: Number of Documents to fetch before filtering to pass to MMR algorithm. lambda_mult: Number between 0 and 1 that determines the degree of diversity among the results with 0 corresponding to maximum diversity and 1 to minimum diversity. Defaults to 0.5. Returns: List of Documents selected by maximal marginal relevance. rcg|]\}}|SrTrTrs r%rWzAFAISS.max_marginal_relevance_search_by_vector..)rr2)rrHrrrrrrrs r%'max_marginal_relevance_search_by_vectorz-FAISS.max_marginal_relevance_search_by_vector s@4QQ GVR  32/2222r2c^K||||||d{V}d|DS)a(Return docs selected using the maximal marginal relevance asynchronously. Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents. Args: embedding: Embedding to look up documents similar to. k: Number of Documents to return. Defaults to 4. fetch_k: Number of Documents to fetch before filtering to pass to MMR algorithm. lambda_mult: Number between 0 and 1 that determines the degree of diversity among the results with 0 corresponding to maximum diversity and 1 to minimum diversity. Defaults to 0.5. Returns: List of Documents selected by maximal marginal relevance. rNcg|]\}}|SrTrTrs r%rWzBFAISS.amax_marginal_relevance_search_by_vector..Jrr2)rrs r%(amax_marginal_relevance_search_by_vectorz.FAISS.amax_marginal_relevance_search_by_vector+sf6JJQ[QWK        32/2222r2c V||}|j|f||||d|}|S)aReturn docs selected using the maximal marginal relevance. Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents. Args: query: Text to look up documents similar to. k: Number of Documents to return. Defaults to 4. fetch_k: Number of Documents to fetch before filtering (if needed) to pass to MMR algorithm. lambda_mult: Number between 0 and 1 that determines the degree of diversity among the results with 0 corresponding to maximum diversity and 1 to minimum diversity. Defaults to 0.5. Returns: List of Documents selected by maximal marginal relevance. r)rdr rHrrrrrrrrs r%max_marginal_relevance_searchz#FAISS.max_marginal_relevance_searchLsU4%%e,, ;t;  #       r2crK||d{V}|j|f||||d|d{V}|S)a+Return docs selected using the maximal marginal relevance asynchronously. Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents. Args: query: Text to look up documents similar to. k: Number of Documents to return. Defaults to 4. fetch_k: Number of Documents to fetch before filtering (if needed) to pass to MMR algorithm. lambda_mult: Number between 0 and 1 that determines the degree of diversity among the results with 0 corresponding to maximum diversity and 1 to minimum diversity. Defaults to 0.5. Returns: List of Documents selected by maximal marginal relevance. Nr)rgrrs r%amax_marginal_relevance_searchz$FAISS.amax_marginal_relevance_searchqs4,,U33333333 BTB  #             r2rc l|tdt||j}|rtd|d|jDfd|D|jtj tj |j |fdt|jD}dt|D|_d S) zDelete by ID. These are the IDs in the vectorstore. Args: ids: List of ids to delete. Returns: Optional[bool]: True if deletion is successful, False otherwise, None if not implemented. NzNo ids provided to delete.zESome specified ids do not exist in the current store. Ids not found: ci|]\}}|| SrTrT)rUidxrzs r%rz FAISS.delete..sUUUxsC#sUUUr2c h|] }| SrTrT)rUrzreversed_indexs r% zFAISS.delete..s>>>3>#.>>>r2r~c"g|] \}}|v | SrTrT)rUrrzindex_to_deletes r%rWz FAISS.delete..s2   3'' '''r2ci|]\}}|| SrTrT)rUrrzs r%rz FAISS.delete..s$S$S$S3Q$S$S$Sr2T)r0r differencer9valuesitemsr7 remove_idsrfromiterint64r8deletesortedr)rHrlr missing_ids remaining_idsrrs @@r%rz FAISS.deletesN ;9:: :#hh))$*C*J*J*L*LMM  !!!  VU43L3R3R3T3TUUU>>>>#>>> bk/JJJKKK S!!!     !:!@!@!B!BCC   %T$S)M:R:R$S$S$S!tr2targetr,cFt|jtstdt |j}|j|jg}|jD]^\}}|j |}t|tstd| ||z||f_|j d|Dd|D}|j |dS)zMerge another FAISS object with the current one. Add the target FAISS to the current one. Args: target: FAISS object you wish to merge into the current one Returns: None. z'Cannot merge with this type of docstorezDocument should be returnedci|] \}}}|| SrTrT)rUrsrrs r%rz$FAISS.merge_from..s AAA 33AAAr2ci|] \}}}|| SrTrT)rUr7rrss r%rz$FAISS.merge_from..s AAAmeS!ucAAAr2N)r.r8rr0r/r9r7 merge_fromrrrrrr)rHr r full_infor target_idrrs r%rzFAISS.merge_froms.$-66 HFGG G4455  fl+++ "7==?? A ALAy/((33Cc8,, @ !>???   lQ. 3? @ @ @ @ AAyAAABBBAAyAAA  !((55555r2rc t} |tjkr)| t |d} n(| t |d} |dt} |di} ||| | | f||d|} | ||||| S)Nrr8r9)r=r>r) r&rr IndexFlatIPr/ IndexFlatL2poprr)clsrOrNrrjrlr=r>rr#r7r8r9vecstores r%__fromz FAISS.__froms())  0 B B B%%c*Q-&8&899EE%%c*Q-&8&899E::j*:*<*<==%zz*@"EE3     &/      ujI3GGGr2c R||}|j|||f||d|S)aOConstruct FAISS wrapper from raw documents. This is a user friendly interface that: 1. Embeds documents. 2. Creates an in memory docstore 3. Initializes the FAISS database This is intended to be a quick way to get started. Example: .. code-block:: python from langchain_community.vectorstores import FAISS from langchain_community.embeddings import OpenAIEmbeddings embeddings = OpenAIEmbeddings() faiss = FAISS.from_texts(texts, embeddings) r)rX _FAISS__fromrrOrrjrlrrNs r% from_textszFAISS.from_textssR6..u55 sz            r2 list[str]cbK||d{V}|j|||f||d|S)aeConstruct FAISS wrapper from raw documents asynchronously. This is a user friendly interface that: 1. Embeds documents. 2. Creates an in memory docstore 3. Initializes the FAISS database This is intended to be a quick way to get started. Example: .. code-block:: python from langchain_community.vectorstores import FAISS from langchain_community.embeddings import OpenAIEmbeddings embeddings = OpenAIEmbeddings() faiss = await FAISS.afrom_texts(texts, embeddings) Nr)r]rrs r% afrom_textszFAISS.afrom_textssh6%55e<<<<<<<< sz            r2c tt|\}}|jt|t||f||d|S)aConstruct FAISS wrapper from raw documents. This is a user friendly interface that: 1. Embeds documents. 2. Creates an in memory docstore 3. Initializes the FAISS database This is intended to be a quick way to get started. Example: .. code-block:: python from langchain_community.vectorstores import FAISS from langchain_community.embeddings import OpenAIEmbeddings embeddings = OpenAIEmbeddings() text_embeddings = embeddings.embed_documents(texts) text_embedding_pairs = zip(texts, text_embeddings) faiss = FAISS.from_embeddings(text_embedding_pairs, embeddings) r)rrr)rrrrjrlrrOrNs r%from_embeddingszFAISS.from_embeddingsBs\: 1zsz KK             r2c*K|j||f||d|S)z:Construct FAISS wrapper from raw documents asynchronously.r)r )rrrrjrlrs r%afrom_embeddingszFAISS.afrom_embeddingsisA#s"           r2 folder_path index_namecjt|}|ddt}||jt ||dz t ||dz d5}tj|j |j f|ddddS#1swxYwYdS)aSave FAISS index, docstore, and index_to_docstore_id to disk. Args: folder_path: folder path to save index, docstore, and index_to_docstore_id to. index_name: for saving with a specific index file name T)exist_okparents.faiss.pklwbN) rmkdirr& write_indexr7r*openpickledumpr8r9)rHr#r$pathr#fs r% save_localzFAISS.save_local{sK   D$ ///()) $*c$J1F1F1F*F&G&GHHH$J,,,,d 3 3 Gq K(ABA F F F G G G G G G G G G G G G G G G G G Gs9"B((B,/B,r)allow_dangerous_deserializationio_flagsr3r4c R|stdt|}t}|t ||dz |} t ||dz d5} t j| \} } dddn #1swxYwY||| | | fi|S)aLoad FAISS index, docstore, and index_to_docstore_id from disk. Args: folder_path: folder path to load index, docstore, and index_to_docstore_id from. embeddings: Embeddings to use when generating queries index_name: for saving with a specific index file name allow_dangerous_deserialization: whether to allow deserialization of the data which involves loading a pickle file. Pickle files can be modified by malicious actors to deliver a malicious payload that results in execution of arbitrary code on your machine. io_flags: Flags to use when opening the pickle file. BThe de-serialization relies loading a pickle file. Pickle files can be modified to deliver a malicious payload that results in execution of arbitrary code on your machine.You will need to set `allow_dangerous_deserialization` to `True` to enable deserialization. If you do this, make sure that you trust the source of the data. For example, if you are loading a file that you created, and know that no one else has modified the file, then this is safe to do. Do not set this to `True` if you are loading a file from an untrusted source (e.g., some random site on the internet.).r(r)rbN)r0rr& read_indexr*r-r.load) rr#rNr$r3r4rr0r#r7r1r8r9s r% load_localzFAISS.load_locals2/  "   K  '))  Tz,A,A,A%A!B!BHMM$J,,,,d 3 3 q  $               s:uh0DOOOOOs.BBBbytescNtj|j|j|jfS)zCSerialize FAISS index, docstore, and index_to_docstore_id to bytes.)r.dumpsr7r8r9rMs r%serialize_to_byteszFAISS.serialize_to_bytess |TZ8QRSSSr2)r3 serializedc l|stdtj|\}}}|||||fi|S)zGDeserialize FAISS index, docstore, and index_to_docstore_id from bytes.r6)r0r.loads)rr?rNr3rr7r8r9s r%deserialize_from_byteszFAISS.deserialize_from_bytessf/  "    L      s:uh0DOOOOOr2Callable[[float], float]c|j|jS|jtjkr|jS|jtjkr|jS|jtjkr|jStd)a8 The 'correct' relevance function may differ depending on a few things, including: - the distance / similarity metric used by the VectorStore - the scale of your embeddings (OpenAI's are unit normed. Many others are not!) - embedding dimensionality - etc. NzJUnknown distance strategy, must be cosine, max_inner_product, or euclidean) rCr>rr%_max_inner_product_relevance_score_fnrE_euclidean_relevance_score_fnCOSINE_cosine_relevance_score_fnr0rMs r%_select_relevance_score_fnz FAISS._select_relevance_score_fns  + 73 3  !%5%G G G= =  #'7'J J J5 5  #'7'> > >2 2  r2c |td|j|f|||d|}fd|D}|S)?Return docs and their similarity scores on a scale from 0 to 1.NLrelevance_score_fn must be provided to FAISS constructor to normalize scoresrc0g|]\}}||fSrTrTrUrscorer;s r%rWzBFAISS._similarity_search_with_relevance_scores..;   1;eS$$U++ ,   r2)rIr0r rHrrrrrrdocs_and_rel_scoresr;s @r%(_similarity_search_with_relevance_scoresz.FAISS._similarity_search_with_relevance_scoress"<<>>  %9 <$;            ?N   #"r2cK|td|j|f|||d|d{V}fd|D}|S)rKNrLrc0g|]\}}||fSrTrTrNs r%rWzCFAISS._asimilarity_search_with_relevance_scores..8rPr2)rIr0rrQs @r%)_asimilarity_search_with_relevance_scoresz/FAISS._asimilarity_search_with_relevance_scores s"<<>>  %9 !C B !  ! !  ! !           ?N   #"r2 Callable[[Dict[str, Any]], bool]c t|r|St|tstdt |ddlm}m}m}m }m }m }||||||d}ddd}||z tt gdz} d |D]/} | r+| d r| | vrtd | 0d fd d fd  |S)a Create a filter function based on the provided filter. Args: filter: A callable or a dictionary representing the filter conditions for documents. Returns: A function that takes Document's metadata and returns True if it satisfies the filter conditions, otherwise False. Raises: ValueError: If the filter is invalid or contains unsupported operators. z5filter must be a dict of metadata or a callable, not r)eqrgtrltne)z$eqz$neqz$gtz$ltz$gtez$ltec ||vSrLrTabs r%z+FAISS._create_filter_func..bs Qr2c ||vSrLrTr^s r%raz+FAISS._create_filter_func..cs !r2)z$inz$nin)$and$or$not $&filter contains unsupported operator: fieldr* condition%Union[Dict[str, Any], List[Any], Any]rrWcxttrXgD]8\}}|vrtd|||f9d fd }|Stt r.t krtfdSfdSfd S) a Creates a filter function based on field and condition. Args: field: The document field to filter on condition: Filter condition (dict for operators, list for in, or direct value for equality) Returns: A filter function that takes a document and returns boolean rhrDict[str, Any]rr!cf|tfdDS)aW Evaluates a document against a set of predefined operators and their values. This function applies multiple comparison/sequence operators to a specific field value from the document. All conditions must be satisfied for the function to return True. Args: doc (Dict[str, Any]): The document to evaluate, containing key-value pairs where keys are field names and values are the field values. The document must contain the field being filtered. Returns: bool: True if the document's field value satisfies all operator conditions, False otherwise. c36K|]\}}||VdSrLrT)rUopvalue doc_values r%ruzYFAISS._create_filter_func..filter_func_cond..filter_fn..s3OO Err)U33OOOOOOr2)rall)rrrri operatorss @r% filter_fnzFFAISS._create_filter_func..filter_func_cond..filter_fns8$&)WWU^^IOOOOYOOOOOOr2c2|vSrLr)r condition_setris r%razEFAISS._create_filter_func..filter_func_cond..sswwu~~'Fr2c2|vSrLrwrrjris r%razEFAISS._create_filter_func..filter_func_cond..s3775>>Y#>r2c6|kSrLrwrzs r%razEFAISS._create_filter_func..filter_func_cond..sswwu~~:r2)rrmrr!)r.dictrr0rrr/ frozenset) rirjrprqrurxrt OPERATIONSSET_CONVERT_THRESHOLDs `` @@r%filter_func_condz3FAISS._create_filter_func..filter_func_condns)T** ! !*!2!2>>IB++()VRT)V)VWWW$$jne%<====PPPPPPP*! )T** ?y>>$999$-i$8$8MFFFFFF>>>>>>::::: :r2rrmcd|vrfd|dDfdSd|vrfd|dDfdSd|vr|dfdSfd |Dfd S) a Creates a filter function that evaluates documents against specified filter conditions. This function processes a dictionary of filter conditions and returns a callable that can evaluate documents against these conditions. It supports logical operators ($and, $or, $not) and field-level filtering. Args: `dict` containing filter conditions. Can include: - Logical operators ($and, $or, $not) with lists of sub-filters - Field-level conditions with comparison or sequence operators - Direct field-value mappings for equality comparison Returns: Callable[[Dict[str, Any]], bool]: A function that takes a document (as a dictionary) and returns True if the document matches all filter conditions, False otherwise. rcc&g|] }|SrTrTrU sub_filterrs r%rWzBFAISS._create_filter_func..filter_func..s#TTTz;;z22TTTr2c<tfdDS)Nc3.K|]}|VdSrLrTrUr1rs r%ruzSFAISS._create_filter_func..filter_func....+&?&?!qqvv&?&?&?&?&?&?r2rsrfilterss`r%raz@FAISS._create_filter_func..filter_func..%3&?&?&?&?w&?&?&?#?#?r2rdc&g|] }|SrTrTrs r%rWzBFAISS._create_filter_func..filter_func..s#SSSz;;z22SSSr2c<tfdDS)Nc3.K|]}|VdSrLrTrs r%ruzSFAISS._create_filter_func..filter_func....rr2)anyrs`r%raz@FAISS._create_filter_func..filter_func..rr2rec| SrLrT)rconds r%raz@FAISS._create_filter_func..filter_func..sttCyy=r2c.g|]\}}||SrTrT)rUrirjrs r%rWzBFAISS._create_filter_func..filter_func..s:$E9!  22r2c<tfdDS)Nc3.K|]}|VdSrLrT)rUrjrs r%ruzSFAISS._create_filter_func..filter_func....s+"N"Ni99S>>"N"N"N"N"N"Nr2r)r conditionss`r%raz@FAISS._create_filter_func..filter_func..s%s"N"N"N"N:"N"N"NNNr2)r)rrrrrrs @@@r%rz.FAISS._create_filter_func..filter_funcs*TTTTVF^TTT?????SSSSVE]SSS?????"{6&>2200000(. JONNN Nr2)rir*rjrkrrW)rrmrrW)callabler.r|r0typerrYrrZrr[r\r}r startswith)rrYrrZrr[r\COMPARISON_OPERATORSSEQUENCE_OPERATORSVALID_OPERATORSrpr~rrrs @@@@r%rzFAISS._create_filter_func=s$ F   M&$'' VV VV  4333333333333333   '&++  *,>> #D$4$47N7N7N$NOO " P PB PbmmC(( PR-F-F !N"!N!NOOO2 ;2 ;2 ;2 ;2 ;2 ;2 ;h% O% O% O% O% O% O% ON{6"""r2 Sequence[str]list[Document]c8fd|D}d|DS)NcDg|]}j|SrT)r8r)rUrzrHs r%rWz$FAISS.get_by_ids..s)999c $$S))999r2c<g|]}t|t|SrT)r.r)rUrs r%rWz$FAISS.get_by_ids..s'AAAz#x'@'@AAAAr2rT)rHrlrs` r% get_by_idszFAISS.get_by_idss09999S999AAtAAAAr2)r5r6r7rr8rr9r:r;r<r=r!r>r)rrJ)rOrPrrQ)rVr*rr`)NN) rOrhrNrirjrkrlrmrrP) rOrhrjrrlrmrrrrP) rrrjrrlrmrrrrP)rNr) rr`rrrrrrrrrr) rr*rrrrrrrrrr) rr`rrrrrrrrrr) rr`rrrrrrrrrr) rr*rrrrrrrrrr) rr`rrrrrrrrrr)rrrN)rr`rrrrrrrrrrrr)rr*rrrrrrrrrrrrrL)rlrmrrrr)r r4rr,)rOrhrNrQrrrjrkrlrmr=r!r>rrrrr4) rOrPrrrjrrlrmrrrr4) rOrrrrjrrlrmrrrr4) rrrrrjrkrlrmrrrr4)r7)r#r*r$r*rr,)r#r*rNrr$r*r3r!r4rrrrr4)rr;) r?r;rNrr3r!rrrr4)rrC)rrrrW)rlrrr)1__name__ __module__ __qualname____doc__rrErIpropertyrNrZr_rdrgrrrrrrrrrrrrrrrrrrrr classmethodrrrr r"r2r:r>rBrIrSrV staticmethodrrrTr2r%r4r4Ks}}PBF".>.Q!!!!!F   X EEEE    1111 /3#' (((((Z+/#' KKKKK0+/#' KKKKK2+/#' KKKKK2<@ ?????H<@ % % % % % T<@     J<@     J+/ 33333F<@ 33333F<@ 333338<@ 33333: <@BBBBBBP <@% % % % % % T <@ 33333D <@ 33333H <@ #####P <@ #####J     D6666@ /3#'".>.Q[> +/#' " " " " [" H +/#' " " " " [" H /3#' $ $ $ $ [$ L /3#'     [ "GGGGG& " 3P 163P3P3P3P3P[3PjTTTT 16 PPPPP[P><<@ #####@<@ #####:K#K#K#\K#ZBBBBBBr2r4rL)rrrr) r'rr(rr)r*r+r*rr,)- __future__rloggingrrr.rprFpathlibrtypingrrrrr r r r r rnumpyrlangchain_core.documentsrlangchain_core.embeddingsrlangchain_core.runnables.configrlangchain_core.vectorstoresr!langchain_community.docstore.baserr&langchain_community.docstore.in_memoryr&langchain_community.vectorstores.utilsrr getLoggerrrAr&r1r4rTr2r%rs""""""                         ------000000;;;;;;333333DDDDDDDDCCCCCC  8 $ $6    BBBBBBBBBBKBBBBBBBBBBr2