o rhO@sddlZddlTddlZddlZddZddZedfddZ d1d d Z d2d d Z e Z d1ddZ ddZeZd3ddZddZddZeZd4ddZeZd4ddZGdddZd5d d!ZGd"d#d#Zefd$d%Zd6d'd(ZGd)d*d*Zd+d,ZeZd-d.Ze Z!d7d/d0Z dS)8N)*cCtj|dd}|j\}}tj||fdd}tj||fdd}t}t||_t||_||_ ||_ | | |t|| ||fS)zPreturn k smallest values (and their indices) of the lines of a float32 arrayfloat32dtypeint64)npascontiguousarrayshapezerosfaissfloat_maxheap_array_tswig_ptridsvalnhkheapifyaddnreorderarrayrmnIDharR/var/www/html/alpaca_bot/venv/lib/python3.10/site-packages/faiss/extra_wrappers.pykmin   rcCr)zOreturn k largest values (and their indices) of the lines of a float32 arrayrrr)rr r r r float_minheap_array_trrrrrrrrrrrrkmax+r r"c Cstj|dd}tj|dd}|j\}}|j\}}||ksJtj||fdd}|tkr compute a checksum for quick-and-dirty comparisons of arrays uint8uint64r) viewndim bvec_checksumr7rr rr bvecs_checksum)arr.csrrrchecksumns   rMcCs(tj||fdd}t||t|||Sr5)rr#rand_smooth_vectors_cr)rr.r9r:rrrrand_smooth_vectorszsrPc Cstj|dd}tj|dd}|jd}|jd|ksJ|jd|jd}}d}t|D]}|t|t|||t||7}q-|S)z< size of intersection between each line of two result tablesrrrrE)rr r rangeranklist_intersection_sizer)I1I2rk1k2ninterirrreval_intersections  rYcCs t|jd|jdt|dS)NrEr)fvec_renorm_L2r rxrrr normalize_L2s r]cCs|tj|dd}|durt|d}tj|ddd}tj|jdd}t|jt| d|t|t||||fS)aPerform a bucket sort on a table of integers. Parameters ---------- tab : array_like elements to sort, max value nbucket - 1 nbucket : integer number of buckets, None if unknown nt : integer number of threads to use (0 = use unthreaded codepath) Returns ------- lims : array_like cumulative sum of bucket sizes (size vmax + 1) perm : array_like perm[lims[i] : lims[i + 1]] contains the indices of bucket #i (size tab.size) rrNrErF) rr intmaxr#r7 bucket_sort_cr rrG)tabnbucketntlimspermrrr bucket_sortsrfcCsn|jdks |jdks J|j\}}|durt|d}tj|ddd}t||t||t|||S)aPerform a bucket sort on a matrix, recording the original row of each element. Parameters ---------- tab : array_like array of size (N, ncol) that contains the bucket ids, maximum value nbucket - 1. On output, it the elements are shuffled such that the flat array tab.ravel()[lims[i] : lims[i + 1]] contains the row numbers of each bucket entry. nbucket : integer number of buckets (the maximum value in tab should be nbucket - 1) nt : integer number of threads to use (0 = use unthreaded codepath) Returns ------- lims : array_like cumulative sum of bucket sizes (size vmax + 1) int32rNrEr) rr r^r_rr#matrix_bucket_sort_inplace_cr r)rarbrcnrowncolrdrrrmatrix_bucket_sort_inplaces   rkc@s2eZdZdZd ddZddZddZd d Zd S) ResultHeapz_Accumulate query results from a sliced dataset. The final result will be in self.D, self.I.FcCs~tj||fdd|_tj||fdd|_|||_|_|r!t}nt}||_||_t |j|_ t |j|_ | ||_ dS)z nq: number of query vectors, k: number of results per query keep_max: keep the top-k maximum values instead of the minima rrrN)rr rrr-rr!r rrrrrheaps)selfr-rkeep_maxrmrrr__init__s   zResultHeap.__init__cCsd|j\}}tj|dd}tj|dd}|j||fksJ||jks#J|j|t|t||dS)z Add results for all heaps D, I should be of size (nh, nres) D, I do not need to be in a particular order (heap or sorted) rrrN)r rr r-rm addn_with_idsr)rnrrr-kdrrr add_results zResultHeap.add_resultcCs|j\}}|t|ks J|jdkr|j|jks%|jdkr#|j|fks%Jtj|dd}tj|dd}tj|dd}|jdkrAdn|}|j|t||t|t||dS)z Add results for a subset of heaps. D, I should hold resutls for all the subset as a special case, if I is 1D, then all ids are assumed to be the same rErrrrN)r lenrHrr rmaddn_query_subset_with_idsr)rnsubsetrrnsubsetrr id_striderrradd_result_subsets zResultHeap.add_result_subsetcCs|jdSN)rmr)rnrrrfinalizezResultHeap.finalizeNF)__name__ __module__ __qualname____doc__rprsrzr|rrrrrls   rlFc Cs||j|jksJ|j\}}}tj||f|jd}tj||f|jd}|r&tnt}||||t|t|t|t|||fS)z Merge a set of sorted knn-results obtained from different shards in a dataset Dall and Iall are of size (nshard, nq, k) each D[i, j] should be sorted returns D, I of size (nq, k) as the merged result set r)r rr#rmerge_knn_results_CMaxmerge_knn_results_CMinr) DallIallronshardrrDnewInewfuncrrrmerge_knn_resultss    rc@s$eZdZddZddZddZdS)MapInt64ToInt64cCsXtt||_|d|jksJd||_tj|dfdd|_t|jt |jdS)Nrtzneed power of 2 capacityrr) r^rlog2 log2_capacitycapacityr#rar hashtable_int64_to_int64_initr)rnrrrrrp3s zMapInt64ToInt64.__init__cCs>|j\}|j|fks Jt|jt|j|t|t|dSr{)r r hashtable_int64_to_int64_addrrra)rnkeysvalsrrrradd:s  zMapInt64ToInt64.addcCs>|j\}tj|fdd}t|jt|j|t|t||Sr=)r rr#r hashtable_int64_to_int64_lookuprrra)rnrrrrrrlookupAs zMapInt64ToInt64.lookupN)rrrrprrrrrrr1s rc Cstj|dd}tj|dd}|j\}}|j\}}||ksJtj||fdd}tj||fdd} |tkrKtt|t|||||t| t|| |fS|tkrftt|t|||||t| t|| |fSt d)a Compute the k nearest neighbors of a vector without constructing an index Parameters ---------- xq : array_like Query vectors, shape (nq, d) where the dimension d is that same as xb `dtype` must be float32. xb : array_like Database vectors, shape (nb, d) where dimension d is the same as xq `dtype` must be float32. k : int Number of nearest neighbors. distance_type : MetricType, optional distance measure to use (either METRIC_L2 or METRIC_INNER_PRODUCT) Returns ------- D : array_like Distances of the nearest neighbors, shape (nq, k) I : array_like Labels of the nearest neighbors, shape (nq, k) rrrz'only L2 and INNER_PRODUCT are supported) rr r r#r$ knn_L2sqrrr&knn_inner_productNotImplementedError) r)r*rr+r-r.r/r0rrrrrknnMs(      rhcc Cs|j\}}|j\}}||ksJtj||fdd}tj||fdd} |dkrPt} || _|| _t| | _t|| _ t | t|t|||d|| fS|dkrpt t|t|||||t|t| || fSt )a Compute the k nearest neighbors of a set of vectors without constructing an index. Parameters ---------- xq : array_like Query vectors, shape (nq, d) where d is the number of bits / 8 `dtype` must be uint8. xb : array_like Database vectors, shape (nb, d) where d is the number of bits / 8 `dtype` must be uint8. k : int Number of nearest neighbors. variant : string Function variant to use, either "mc" (counter) or "hc" (heap) Returns ------- D : array_like Distances of the nearest neighbors, shape (nq, k) I : array_like Labels of the nearest neighbors, shape (nq, k) rgrrrrEmc) r rr#r int_maxheap_array_trrrrrhammings_knn_hchammings_knn_mcr) r)r*rvariantr-r.r/r0rrheaprrr knn_hamming}s.      rc@s<eZdZdZddZddZd ddZdd d Zd d ZdS)KmeansaObject that performs k-means clustering and manages the centroids. The `Kmeans` class is essentially a wrapper around the C++ `Clustering` object. Parameters ---------- d : int dimension of the vectors to cluster k : int number of clusters gpu: bool or int, optional False: don't use GPU True: use all GPUs number: use this many GPUs progressive_dim_steps: use a progressive dimension clustering (with that number of steps) Subsequent parameters are fields of the Clustring object. The most important are: niter: int, optional clustering iterations nredo: int, optional redo clustering this many times and keep best verbose: bool, optional spherical: bool, optional do we want normalized centroids? int_centroids: bool, optional round centroids coordinates to integer seed: int, optional seed for the random number generator cKs||_||d|_d|vrt|_nt|_|D]$\}}|dkr3|dks,|dkr/t}||_qt|j|t |j||q| dS)zd: input dimension, k: nb of centroids. Additional parameters are passed on the ClusteringParameters object, including niter=25, verbose=False, spherical = False Fprogressive_dim_stepsgpuTN) r.resetr"ProgressiveDimClusteringParameterscpClusteringParametersitems get_num_gpusgetattrsetattr set_index)rnr.rkwargsvrrrrps    zKmeans.__init__cCsv|j}|jjtkr)|jjrt||_nt||_|jr't j |j|jd|_dSdS|jr3t |jd}nt }||_ dS)N)ngpu)r.r __class__r spherical IndexFlatIPindex IndexFlatL2rr index_cpu_to_all_gpusGpuProgressiveDimIndexFactoryProgressiveDimIndexFactoryfac)rnr.rrrrrs    zKmeans.set_indexNcCs(|dur t||_d|_d|_d|_dS)zg prepare k-means object to perform a new clustering, possibly with another number of centroids N)r^r centroidsobjiteration_stats)rnrrrrrs   z Kmeans.resetc sTtj|dd}|j\}}||jksJ|jjtkrBt||j|j}|dur9|j\}}||ks0Jt | |j | ||j|n$|dusHJ|dusNJ|jjrTJt||j|j}| |t||jt |j } | |j||_ |jfddtDtddD|_dfddD|_|jjd kr|jd Sd S) a  Perform k-means clustering. On output of the function call: - the centroids are in the centroids field of size (`k`, `d`). - the objective value at each iteration is in the array obj (size `niter`) - detailed optimization statistics are in the array iteration_stats. Parameters ---------- x : array_like Training vectors, shape (n, d), `dtype` must be float32 and n should be larger than the number of clusters `k`. weights : array_like weight associated to each vector, shape `n` init_centroids : array_like initial set of centroids, shape (n, d) Returns ------- final_obj: float final optimization objective rrNcsg|]}|qSr)at).0rX)statsrr 8sz Kmeans.train..cSsg|]}|jqSr)r)rstrrrr9sz,obj time time_search imbalance_factor nsplitcsg|] fddDqS)csi|]}|t|qSr)r)rfieldrrr =sz+Kmeans.train...r)r) stat_fieldsrrr<srrg)rr r r.rrr Clusteringrr copy_array_to_vectorravelrtrainrrProgressiveDimClusteringrrvector_float_to_arrayreshaperrQr7rrsplit) rnr\weightsinit_centroidsrr.clusncr0rr)rrrrs2         z Kmeans.traincCsZtj|dd}|jdusJd|j|j|j|j|d\}}||fS)Nrrzshould train before assigningrE)rr rrrrsearchr)rnr\rrrrrassignBs  z Kmeans.assignr{)NN) rrrrrprrrrrrrrrs    Pack a set integers (i, j) where i=0:n and j=0:M into n bitstrings. Output is an uint8 array of size (n, code_size), where code_size is such that at most 7 bits per code are wasted. If nbit is an integer: all entries takes nbit bits. If nbit is an array: entry (i, j) takes nbit[j] bits. rgrrD) r rr rr^sumr#pack_bitstrings_cr)rKnbitrM code_sizebrrrpack_bitstringsTs rcCs|j\}}|dur>tj|dd}t|}t|dd}||ks$Jtj||fdd}t||t|t||t||S|}||dd}||ksNJtj||fdd}t|||t||t||S)a Unpack a set integers (i, j) where i=0:n and j=0:M from n bitstrings (encoded as uint8s). Input is an uint8 array of size (n, code_size), where code_size is such that at most 7 bits per code are wasted. Two forms: - when called with (array, M, nbit): there are M entries of size nbit per row - when called with (array, nbits): element (i, j) is encoded in nbits[j] bits Nrgrrr) r rr rur^rr#unpack_bitstrings_cr)r M_or_nbitsrrrr min_code_sizerKrrrunpack_bitstringsos(     r)r3)r3N)rN)Nrr~)rr{)"numpyr faiss.loaderr collections.abcrrr"r$r2r;rAlrandrCrMrPrOrYr]rfr`rkrhrlrrrrrrrrrrrrrrs@       ' @  08