jddlmZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z ddl m Z mZmZmZmZmZmZmZmZddlmZddlmZddlmZe r dd lmZdd lmZ Gd d eZ!y) ) annotationsN)contextmanager)StringIO) TYPE_CHECKINGAnyDict GeneratorIterableListOptionalTupleType) Embeddings) VectorStore)Document) connection)cursorcleZdZdZGddeej ZGddZdddd d$d Z Gd d Z d%d Z d%d Z d& d'dZ d( d)dZd*dZ d( d+dZ d,dZe d- d.dZ d& d/dZ d0 d1dZd2dZ d3 d4dZ d3 d5dZ d3 d6dZ d3 d7dZ d( d8dZ d9dZd:dZd%dZd%dZd;d Z d;d!Z! d YellowbrickzYellowbrick as a vector database. Example: .. code-block:: python from langchain_community.vectorstores import Yellowbrick from langchain_community.embeddings.openai import OpenAIEmbeddings ... ceZdZdZdZdZy)Yellowbrick.IndexTypezrbrlconnectra autocommit)r*r>s r"get_connectionz-Yellowbrick.DatabaseConnection.get_connectionsO ##t'7'7'>'>#+#3#3D4K4K#L .3  +## #r!c#Kddlm}|j} ||jy#|$r>}|j |j j ddtd|d}~wwxYww)Nr) DatabaseErrorz2Database error occurred, rolling back transaction.T)exc_infozDatabase transaction failed.)r>rsrqcommitrollbackrdrJ RuntimeError)r*rsconnes r"get_managed_connectionz5Yellowbrick.DatabaseConnection.get_managed_connectionsu .&&(D   ! J  ""HSW###ABI  Js$A5/A5A29A--A22A5c#K|j5}|j} ||j dddy#|jwxYw#1swYyxYwwr')rzrrm)r*rxrs r"rXz)Yellowbrick.DatabaseConnection.get_cursorsO,,.$# LLLN /. LLN /.s1A$AAA A$AAA!A$)rNr2r6rcr3z 'Yellowbrick.DatabaseConnection'r3None)r3z'PgConnection')r3z%Generator['PgConnection', None, None])r3z!Generator['PgCursor', None, None]) rrrrf__annotations__rbrhrRrqrrzrX __classcell__)rjs@r"rOr`~sn 04 -4 !$' !1? ! - ! (  $      #  #r!rOcddlm}|jrJ|j|j dj |j |jyy)z> Helper function: create schema if not exists rsqlzE CREATE SCHEMA IF NOT EXISTS {s} )sN)r>rrSexecuteSQLformat Identifier)r*rrs r"rZzYellowbrick._create_schemasQ ! << NN&nnT\\2   r!c.ddlm}|jr |jfnd}|jg||j|j z}|j|j|j zdz}|j |jdj|||jr |jfnd}|jg||j}|jg||j|j z}|j|j|j zdz}|j|j|j zdz} |j |jd j|||| y ) z= Helper function: create table if not exists rrr _pk_doc_ida0 CREATE TABLE IF NOT EXISTS {t} ( doc_id UUID NOT NULL, text VARCHAR(60000) NOT NULL, metadata VARCHAR(1024) NOT NULL, CONSTRAINT {c} PRIMARY KEY (doc_id)) DISTRIBUTE ON (doc_id) SORT ON (doc_id) tc_pk_doc_id_embedding_id _fk_doc_ida CREATE TABLE IF NOT EXISTS {t1} ( doc_id UUID NOT NULL, embedding_id SMALLINT NOT NULL, embedding FLOAT NOT NULL, CONSTRAINT {c1} PRIMARY KEY (doc_id, embedding_id), CONSTRAINT {c2} FOREIGN KEY (doc_id) REFERENCES {t2}(doc_id)) DISTRIBUTE ON (doc_id) SORT ON (doc_id) t1t2c1c2N) r>rrSrrTrMrrr) r*rr schema_prefixrrrrrrs r"r[zYellowbrick._create_tabless !+/<<R CNN LM L4;;9K9K+K L NN4;;););;lJ K GG f ,0<<R S^^ 8] 8DKK 8 S^^ M] MDKK$:L:L,L M ^^ KK$,, ,/H H ^^DKK$*<*<<|K L GG f  r!c|8|jj5}|j|||dddy|j|||y#1swYyxYw)z Helper function: Drop data. If a cursor is provided, use it; otherwise, obtain a new cursor for the operation. N)r5)rrX _drop_table)r*r=r5rs r"r7zYellowbrick.dropsX >++-  v >.-   VU6  :.-s AAcddlm}|r|j||}n|j|}|jdj |}|j |y)zI Executes the drop table command using the given cursor. rrz1 DROP TABLE IF EXISTS {} CASCADE N)r>rrrrr)r*rr=r5r table_namedrop_table_querys r"rzYellowbrick._drop_tablesY ! 6J.J77   &   '(r!c|jj5}d}|j||jd}dddj dk(s|j dk(ryt d#1swY;xYw)zE Helper function: Test the database is UTF-8 encoded z SELECT pg_encoding_to_char(encoding) FROM pg_database WHERE datname = current_database(); rNutf8zutf-8TzDatabase encoding is not UTF-8)rrXrfetchonelower Exception)r*rqueryencodings r"rWz Yellowbrick._check_database_utf8sx__ ' ' )VE NN5 !(+H* >> v %)9W)D<= =* )s 'A<<Bc d}t|}|jjt|}g}|s|Dcgc]}i}}|jdxstj }|j j5} t} t} tj| ddtj} tj| ddtj} d}t|D]\}}ttj}|j!|| j#||t%j&||gt||D]\}}| j#|||g|dz }||k\s|j)| | | | j+d| j-d| j+d| j-dd}|dkDr|j)| | | ddd|j.tj0j2k(r%|j5|tj6|Scc}w#1swY\xYw)Ni' index_params ") delimiter quotecharquotingr)listrUembed_documentsr-rr4rrXrcsvwriter QUOTE_MINIMAL enumerater2uuiduuid4appendwriterowjsondumps _copy_to_dbseektruncater(r#r _update_indexUUID)r*texts metadataskwargs batch_size embeddingsresults_rr content_io embeddings_iocontent_writerembeddings_writercurrent_batch_sizeitextdoc_uuid embedding_idr\s r" add_textszYellowbrick.add_texts3s  U __44T%[A %*+UUI+zz.1N[5L5L5N __ ' ' )V!J$JM ZZdc3CTCTN!$ cFWFW! "# $U+4tzz|,x(''4IaL9Q(RS/8A/G+L)%.., /RS0H#a'"%3$$VZGOOA&''*!&&q)!**1-)*&%,("A%  ]CA*D  " "k&;&;&?&? ?   |TYYx-@ AS,* )s IDI A3II c|jd|jdddlm}|jr |jfnd}|jg||j |j z}|jdj|}|j|||jr |jfnd}|jg||j }|jdj|}|j||y)Nrrr z COPY {table} (doc_id, text, metadata) FROM STDIN WITH (FORMAT CSV, DELIMITER E'\t', QUOTE '"') )r=z COPY {table} (doc_id, embedding_id, embedding) FROM STDIN WITH (FORMAT CSV, DELIMITER E'\t', QUOTE '"') ) rr>rrSrrTrMrr copy_expert) r*rrrrrr=content_copy_queryembeddings_copy_querys r"rzYellowbrick._copy_to_dbjs 1 +/<<R P Pt{{T=O=O/OP WW  &u&   -z:+/<<R ; ;t{{; # ! &u&   0-@r!c J||||||} | jd||d|| S)aAdd texts to the vectorstore index. Args: texts: Iterable of strings to add to the vectorstore. metadatas: Optional list of metadatas associated with the texts. connection_string: URI to Yellowbrick instance embedding: Embedding function table: table to store embeddings kwargs: vectorstore specific parameters )r\rNr=r5r7)rrr )r) rirr\rrNr=r5r7rvsss r" from_textszYellowbrick.from_textss<*/    AEYA&A r!c  ddlm |r jd}nc|Vt fd|D} jdj |} jdj |}n t d |jr |jfnd }|jj5} jg||j|jz} jd j | | } |j|  jg||j} jd j | | } |j| |j||j|jzg|r^ jg||j|jz} jd j | | } |j| dddy#1swYyxYw) zxDelete vectors by uuids. Args: ids: List of ids to delete, where each id is a uuid string. rrz' WHERE 1=1 Nc3@K|]}j|ywr')Literal).0idrs r" z%Yellowbrick.delete..s8Cb#++b/Csz, z5 WHERE doc_id IN ({ids}) )idsz*Either ids or delete_all must be provided.r zDELETE FROM {table} {where_sql})r= where_sql)r>rrtuplejoinr ValueErrorrSrrXrrTrMr _table_existsrK) r*r delete_allrruuids ids_formattedrrtable_identifierrrs @r"deletezYellowbrick.deletes ! I _8C88EGGDM..u5Mf! IJ J+/<<R __ ' ' )V-s~~   $ d.@.@ @  GG=>EE&)FE NN5 !-s~~J}JdkkJ GG=>EE&)FE NN5 !!t!! d&:&::=J$23>>$"$$(KK$2F2F$F$  ABII*iJu%1*45*4s 4D9G66G?cddlm}|j|}|j|}|j|j dj |||j ddkDS)z> Checks if a table exists in the given schema rrz SELECT COUNT(*) FROM sys.table t INNER JOIN sys.schema s ON t.schema_id = s.schema_id WHERE s.name = {schema} AND t.name = {table_name} )r5r)r>rrrrrr)r*rrr5rs r"rzYellowbrick._table_existssp !V$[[,  GG  f%  #a''r!cddl}djtt|}|j |j }|j }tj|dd}|S)Nr,)bytes) hashlibrmapr2sha1encodedigestrr)r*vectorr vector_str hash_object hash_digest vector_uuids r"_generate_vector_uuidz!Yellowbrick._generate_vector_uuidsZXXc#v./ ll:#4#4#67 !((* iik#2&67 r!c xddlm}ddlm}|j dxst j }|jj5}d|jz}|j|} |jdj|j|} |j| t|D cgc]\} } t!| | | f} } } |jdj|j|}|||| |j|}|j"r |j"fnd}|jg||j}|jg||j|j$z}|j&t j(j*k(r|jd z}|j-||||j"r |j"fnd}|jg||j|j.z}|j|}|jd j||||||j1|j3d d }|j||f|j5}nF|jd j|||}|j||f|j5}dddg}D]D}t7j8|dxsi}t;|d|}|j=||dfF|Scc} } w#1swY\xYw)aPerform a similarity search with Yellowbrick with vector Args: embedding (List[float]): query embedding k (int, optional): Top K neighbors to retrieve. Defaults to 4. NOTE: Please do not let end-user fill this and always be aware of SQL injection. Returns: List[Document, float]: List of Documents and scores rr)execute_valuesrtmp_z CREATE TEMPORARY TABLE {} ( doc_id UUID, embedding_id SMALLINT, embedding FLOAT) ON COMMIT DROP DISTRIBUTE REPLICATE z:INSERT INTO {} (doc_id, embedding_id, embedding) VALUES %sr _tmp_hasha/ WITH index_docs AS ( SELECT t1.doc_id, SUM(ABS(t1.hash-t2.hash)) as hamming_distance FROM {lsh_index} t1 INNER JOIN {input_hash_table} t2 ON t1.hash_index = t2.hash_index GROUP BY t1.doc_id HAVING hamming_distance <= {hamming_distance} ) SELECT text, metadata, SUM(v1.embedding * v2.embedding) / (SQRT(SUM(v1.embedding * v1.embedding)) * SQRT(SUM(v2.embedding * v2.embedding))) AS score FROM {v1} v1 INNER JOIN {v2} v2 ON v1.embedding_id = v2.embedding_id INNER JOIN {v3} v3 ON v2.doc_id = v3.doc_id INNER JOIN index_docs v4 ON v2.doc_id = v4.doc_id GROUP BY v3.doc_id, v3.text, v3.metadata ORDER BY score DESC LIMIT %s hamming_distance)v1v2v3 lsh_indexinput_hash_tablera SELECT text, metadata, score FROM (SELECT v2.doc_id doc_id, SUM(v1.embedding * v2.embedding) / (SQRT(SUM(v1.embedding * v1.embedding)) * SQRT(SUM(v2.embedding * v2.embedding))) AS score FROM {v1} v1 INNER JOIN {v2} v2 ON v1.embedding_id = v2.embedding_id GROUP BY v2.doc_id ORDER BY score DESC LIMIT %s ) v4 INNER JOIN {v3} v3 ON v4.doc_id = v3.doc_id ORDER BY score DESC )rrrNr) page_contentmetadata)r>rpsycopg2.extrasrr-rr4rrXrTrrrrrrr2rSrMr(r#r_generate_tmp_lsh_hashesrKrr0fetchallrloadsrr)r*r\krrrrrtmp_embeddings_table tmp_doc_idcreate_table_queryrembedding_value data_input insert_queryrrrrtmp_hash_tablerr sql_queryr documentsresultrdocs r"&similarity_search_with_score_by_vectorz2Yellowbrick.similarity_search_with_score_by_vectors/ !2zz.1N[5L5L5N __ ' ' )V#)DKK#7 33I>J!$ "fS^^$89:  NN- .6?y5I5I1L/Z,@5I 77LfS^^$89:  6< < 45B/3||T\\OM<< P>P0PQB&&+*?*?*C*CC!%{!:--(" 48<<R *CNN"$(KK$2F2F$F $'>>.#A GG!#F&'%5%([[$../A1E&  GZD!//+GG2&3<y1$/ //+u*x35 Fzz&),2Hq HEC   c6!9- . i* )s A/L0=L*HL0*L00L9c |jj|}|jd||d|}|Dcgc]\}}| c}}Scc}}w)aePerform a similarity search with Yellowbrick Args: query (str): query string k (int, optional): Top K neighbors to retrieve. Defaults to 4. NOTE: Please do not let end-user fill this and always be aware of SQL injection. Returns: List[Document]: List of Documents r\rr rU embed_queryr)r*rrrr\r rrs r"similarity_searchzYellowbrick.similarity_searchsZOO//6 ?D?? 1 (. #,,)Q),,,s Ac f|jj|}|jd||d|}|S)arPerform a similarity search with Yellowbrick Args: query (str): query string k (int, optional): Top K neighbors to retrieve. Defaults to 4. NOTE: Please do not let end-user fill this and always be aware of SQL injection. Returns: List[Document]: List of (Document, similarity) rr r)r*rrrr\r s r"similarity_search_with_scorez(Yellowbrick.similarity_search_with_scoresDOO//6 ?D?? 1 (. r!c b|jd||d|}|Dcgc]\}}| c}}Scc}}w)aPerform a similarity search with Yellowbrick by vectors Args: embedding (List[float]): query embedding k (int, optional): Top K neighbors to retrieve. Defaults to 4. NOTE: Please do not let end-user fill this and always be aware of SQL injection. Returns: List[Document]: List of documents rr )r)r*r\rrr rrs r"similarity_search_by_vectorz'Yellowbrick.similarity_search_by_vectorsG@D?? 1 (. #,,)Q),,,s +cddlm}|jr |jfnd}|jg||j|j z}|jg||j|j z}|jg||j}|jdj|}|r9|jdj|jt|n|jd} |jd} |jd j|||| | } |j| y ) zAdd hashes to LSH indexrrr zINSERT INTO {}zWHERE e.doc_id = {doc_id})doc_idz GROUP BY 1, 2av {query_prefix} SELECT e.doc_id as doc_id, h.id as hash_index, CASE WHEN SUM(e.embedding * h.hyperplane) > 0 THEN 1 ELSE 0 END as hash FROM {embedding_table} e INNER JOIN {hyperplanes} h ON e.embedding_id = h.hyperplane_id {condition} {group_by} ) query_prefixembedding_table hyperplanes conditiongroup_byN) r>rrSrrTrLrKrrrr2r) r*rrrrlsh_hyperplane_tablelsh_index_table_idembedding_table_idquery_prefix_idr r! input_querys r"_update_lsh_hasheszYellowbrick._update_lsh_hashessI !+/<<R -s~~   KK$*C*CC ,S^^   KK$*>*>> ,S^^H]HDKKH''"23::;MN GG/ 0 7 7s{{3v;?W 7 X  77?+gg    &(.,   & {#r!cddlm}|jr |jfnd}|jg||j|j z}|j|}|j|}|j dj|} |j d} |j dj| ||| } |j| y) zGenerate temp LSHrrr z+CREATE TEMPORARY TABLE {} ON COMMIT DROP ASz GROUP BY 1a[ {query_prefix} SELECT h.id as hash_index, CASE WHEN SUM(e.embedding * h.hyperplane) > 0 THEN 1 ELSE 0 END as hash FROM {embedding_table} e INNER JOIN {hyperplanes} h ON e.embedding_id = h.hyperplane_id {group_by} DISTRIBUTE REPLICATE )rrrr!N) r>rrSrrTrLrrr) r*rtmp_embedding_tabler rrr"tmp_embedding_table_idtmp_hash_table_idrr!r&s r"rz$Yellowbrick._generate_tmp_lsh_hashess !+/<<R -s~~   KK$*C*CC "%0C!DNN>:wwLMTT  77<(gg    &%2,   " {#r!cddlm}|jr |jfnd}|jg||j|j z}|j |jdj||jddkDry|jg||j}|j |jdj||jd}|dz }|jd j|j||j|| }|j |y) z4Generate random hyperplanes and store in Yellowbrickrrr zSELECT COUNT(*) FROM {t})rNz!SELECT MAX(embedding_id) FROM {t}ra2 WITH parameters AS ( SELECT {num_hyperplanes} AS num_hyperplanes, {dims_per_hyperplane} AS dims_per_hyperplane ) INSERT INTO {hyperplanes_table} (id, hyperplane_id, hyperplane) SELECT id, hyperplane_id, (random() * 2 - 1) AS hyperplane FROM (SELECT range-1 id FROM sys.rowgenerator WHERE range BETWEEN 1 AND (SELECT num_hyperplanes FROM parameters) AND worker_lid = 0 AND thread_id = 0) a, (SELECT range-1 hyperplane_id FROM sys.rowgenerator WHERE range BETWEEN 1 AND (SELECT dims_per_hyperplane FROM parameters) AND worker_lid = 0 AND thread_id = 0) b )num_hyperplanesdims_per_hyperplanehyperplanes_table) r>rrSrrTrLrrrrr) r*rr-rrr/rnum_dimensionsr s r"_populate_hyperplanesz!Yellowbrick._populate_hyperplanes*s1 +/<<R *CNN   KK$*C*CC  sww9:AADUAVW ?? Q ! #  CNN 7M 74;; 7swwBCJJQJOP*1-!ww  $ &KK8 # N ;/  % . |$r!cHddlm}|jr |jfnd}|jg||j|j z}|jg||j|j z}|j|j|j zdz}|j|j|j zdz}|j|jdj|||||jr |jfnd}|jg||j|jz}|j|j|jzdz} |j|jd j|| y ) z&Create LSH index and hyperplane tablesrrr rra CREATE TABLE IF NOT EXISTS {t1} ( doc_id UUID NOT NULL, hash_index SMALLINT NOT NULL, hash SMALLINT NOT NULL, CONSTRAINT {c1} PRIMARY KEY (doc_id, hash_index), CONSTRAINT {c2} FOREIGN KEY (doc_id) REFERENCES {t2}(doc_id)) DISTRIBUTE ON (doc_id) SORT ON (doc_id) r _pk_id_hp_ida2 CREATE TABLE IF NOT EXISTS {t} ( id SMALLINT NOT NULL, hyperplane_id SMALLINT NOT NULL, hyperplane FLOAT NOT NULL, CONSTRAINT {c} PRIMARY KEY (id, hyperplane_id)) DISTRIBUTE REPLICATE SORT ON (id) rN) r>rrSrrTrKrMrrrrL) r*rrrrrrrrrs r"_create_lsh_index_tablesz$Yellowbrick._create_lsh_index_tablesTsv +/<<R S^^ O] ODKK$:N:N,N O S^^ M] MDKK$:L:L,L M ^^DKK$*>*>>M N ^^DKK$*>*>>M N GG f  &,0<<R CNN SM S4;;9R9R+R S NN4;;)B)BB^S T GG f r!c|j|j|j|jz||j|j|j|jz|y)zDrop LSH index tables)r5r=rN)r7rSrTrKrL)r*rs r"rYz"Yellowbrick._drop_lsh_index_tabless] <rrrXrSrrTrMrrrr[rrw) r*rrrrold_embeddingscontentalter_table_queryr insert_content_queryrys r"migrate_schema_v1_to_v2z#Yellowbrick.migrate_schema_v1_to_v2s # H++-37<<R +S^^H]HDKKH !/!T!T e@S!T(#.."$(KK$2D2D$D%(GG,M$N$U$U!%%V%!01""6*"ww  &!% |,'*ww( &G&7 % 34A.--B H!;A3?@a G Hs5E'D0EE'E$ E'$E'' F0E??F)r\rrNr2r=r2r5 Optional[str]r6zOptional[logging.Logger]r7boolr3r})r 'PgCursor'r3r}r1)r=r2r5rDrzOptional['PgCursor']r3r}r')rrFr=r2r5rDr3r})r3rE)rz Iterable[str]rOptional[List[dict]]rrr3 List[str])rrFrrrrr3r})Nr langchainpublicF)rizType[Yellowbrick]rrHr\rrrGrNr2r=r2r5r2r7rErrr3r)rzOptional[List[str]]rzOptional[bool]rrr3r})rJ)rrFrr2r5r2r3rE)r List[float]r3 uuid.UUID))r\rKrintrrr3List[Tuple[Document, float]])rr2rrNrrr3List[Document])rr2rrNrrr3rO)r\rKrrNrrr3rP)rrFrzOptional[uuid.UUID]r3r})rrFr)r2r r2r3r})rrFr-rNr3r})rr%r3r})rr%rrLr3r}r|)$rrrrr2enumEnumr#r4r+rOrZr[r7rrWrr classmethodrrrrrrrrr'rr1r4rYr9r;rrCr r!r"rr!suC 11,!%+/<'<'<' <'  <')<'<' <'|4#4#l"1 l!%'+ ;;;% ;  ;(!% ))) )  )..+/55(5 5  5nA A.6AGOA A8 +/!#  (      @$(%); ;#; ;  ;|BJ( (.1(;>( (001V$V),Vr^sS"  %   13:>6lH+lHr!