o ôrÒhÓ/ã@s”ddlZddlZddlZddlZe e¡Zejfdd„Z ej Z ddd„Z ejddfd d „Z dd d „Z dd d„Zddd„Z  ddd„Zddd„ZdS)éNcCst d|j|f¡t ¡}|j\}}t |¡}tj|||d}t ||¡} t ¡r8t dt ¡¡t  | ¡} d} |D]3} | jd} |   | ¡|   ||¡\} }|| 7}|  | |¡|   ¡| | 7} t d| t ¡|f¡q<| ¡t dt ¡|| f¡|j|jfS)z¤Computes the exact KNN search results for a dataset that possibly does not fit in RAM but for which we have an iterator that returns it block by block. z%knn_ground_truth queries size %s k=%d©Úkeep_maxúrunning on %d GPUsrú%d db elements, %.3f szGT time: %.3f s (%d vectors))ÚLOGÚinfoÚshapeÚtimeÚfaissÚis_similarity_metricÚ ResultHeapÚ IndexFlatÚ get_num_gpusÚindex_cpu_to_all_gpusÚaddÚsearchÚ add_resultÚresetÚfinalizeÚDÚI)ÚxqÚ db_iteratorÚkÚ metric_typeÚt0ÚnqÚdrÚrhÚindexÚi0ÚxbiÚnirr©r#ú]/var/www/html/alpaca_bot/venv/lib/python3.10/site-packages/faiss/contrib/exhaustive_search.pyÚknn_ground_truths,        r%éc! Csœ|j\}}t|tjƒ}t |j¡}|rt|ƒnt|ƒ}t|j |ƒ} t   d|›d| ›d|›d|›¡t   ¡} |  || ¡\} } t   ¡| } |rV|ddksQJ‚|  d¡} d}d }|d urÂ|sm| d d …| d f|k}n | d d …| d f|k}| ¡dkrÂt   d | ¡¡t   ¡} t|tjƒrª|}|ržt |d¡}nt ||j¡}| |¡| |||¡\}}}|r¼| d¡}t   ¡| }t   d ¡t   ¡} |rÐtjntj}||| ||ƒ} tj}|| ƒ|_|| ƒ|_|d ur||ƒ|_||ƒ|_|| d¡ƒ|_||ƒ|_tj |d dd}| !||ƒ¡|d}tj || j"d}tj |dd}| #||ƒ||ƒ¡t   ¡| } t   d| d›d|d›d| d›d¡|||fS)aAGPU does not support range search, so we emulate it with knn search + fallback to CPU index. The index_cpu can either be: - a CPU index that supports range search - a numpy table, that will be used to construct a Flat index if needed. - None. In that case, at most gpu_k results will be returned z GPU search z queries with k=z is_binary_index=z keep_max=éi€Úint16rNézCPU search remain %dÚcombineTÚint64©ÚdtypeéÿÿÿÿcSóg|]}t|ƒ‘qSr#©Úlen)Ú.0Údir#r#r$Ú ‹óz$range_search_gpu..ztimes z.3fzs Ús))rÚ isinstancer Ú IndexBinaryr rÚintÚfloatÚminÚntotalrÚdebugr rÚastypeÚsumÚnpÚndarrayÚIndexBinaryFlatr rÚ range_searchÚCombinerRangeKNNint16ÚCombinerRangeKNNfloatÚswig_ptrrrÚmaskÚD_remainÚviewÚ lim_remainÚI_remainÚemptyÚ compute_sizesr-Ú write_resultÚrangeÚ METRIC_L2ÚappendÚcumsumÚhstack)!rÚr2Ú index_gpuÚ index_cpuÚgpu_krrÚis_binary_indexrrrrrÚt1Út2rJrGÚxbrHrKÚCombinerRangeKNNÚcombinerÚspÚL_resÚnresÚD_resÚI_resÚnrÚiÚnvÚl0Úl1Út3r#r#r$Úrange_search_gpu5sr    ÿ         þ       $ riFr.cs|j\}}t ¡}tj|dd}t ||¡} |dkrt ¡}|r6t d|¡t  ¡} || _ tj | | |d} d} dd„t |ƒDƒ} d d„t |ƒDƒ}|D]r}|jd}|dkrk|   |¡t||| |ƒ\}}}|  ¡n|   |¡|  ||¡\}}}|  ¡|| 7}t |ƒD]'}||||d }}||kr¬| | |||…¡|| |||…¡q…| |7} t d | t ¡|f¡qLtjdd d‰tjddd‰‡fd d„| Dƒ} ‡fdd„|Dƒ}dd„|Dƒ}t|ƒ|ksîJ‚tj|d dd}t |¡|d d…<|t | ¡t |¡fS)z§Computes the range-search search results for a dataset that possibly does not fit in RAM but for which we have an iterator that returns it block by block. Úfloat32r,r.r©ÚcoÚngpurcSóg|]}g‘qSr#r#©r2Ú_ir#r#r$r4¨óz&range_ground_truth..cSrnr#r#ror#r#r$r4©rqr)rr+có"g|] }|gkr t |¡nˆ‘qSr#©r@rS©r2rd)Úempty_Dr#r$r4Àó"crrr#rsrt)Úempty_Ir#r$r4ÁrvcSr/r#r0rtr#r#r$r4Âr5Úuint64N)rr r@Úascontiguousarrayr r rrrÚGpuMultipleClonerOptionsÚshardrrOrrirrCrQÚzerosr1rRrS)rrÚ thresholdrr{rmrrrrrlrUr rrr!r"Úlims_iÚDiÚIiÚjrfrgÚsizesÚlimsr#)rurwr$Úrange_ground_truth“sR       €r„c Csp|r||k}n||k}t |¡}d}t|ƒD]\}} t| ƒ} |||| … ¡||<|| 7}q|||||fS)z select a set of results r)r@Ú zeros_likeÚ enumerater9r?) r`ÚdisÚidsÚthreshrrGÚnew_nresÚordrcr#r#r$Úthreshold_radius_nresÉs   rŒc Cs‚|r||k}n||k}t |¡}t|ƒd}t|ƒD]}||||d} } |||| | … ¡||d<q|||||fS)z= restrict range-search results to those below a given radius r))r@r…r1rOr?) rƒr‡rˆr‰rrGÚnew_limsÚnrdrfrgr#r#r$Úthreshold_radiusØs    "rc Csìt dd„|Dƒ¡}t|ƒ|ksJ‚|r&| t|ƒ|d¡|d|}n | |¡||}|jdkr9t|ƒ}nt|ƒ}t d|¡d}t |ƒD] \}\}}} t ||| ||d\}}} |t|ƒ7}||| f||<qJt d |¡||fS) z‘find radius that reduces number of results to target_nres, and applies it in-place to the result batches used in range_search_max_resultscSsg|]\}}}|‘qSr#r#)r2Ú_r‡r#r#r$r4êóz apply_maxres..r)r.rjz setting radius to %srrz. updated previous results, new nb results %d) r@rSr1Ú partitionr-r:r9rr=r†rŒ) Ú res_batchesÚ target_nresrÚalldisÚradiusÚtotresrdr`r‡rˆr#r#r$Ú apply_maxresæs(     ÿ r˜cCst|tjƒ}|dur|dusJ‚td|ƒ}|dur&|dus J‚t|dƒ}|dkr.t ¡}|rGt d|¡t ¡} || _tj || |d} nd} t   ¡} d} } d}}}g}|D]”}t   ¡}t  dt |ƒ›d ¡| ryt ||| |ƒ\}}}n | ||¡\}}}|d d…|dd…}|t |ƒ7}|t |ƒ7}t   ¡}|r¥| d ¡}|t |ƒ7}| |||f¡|durÑ||krÑt d ||f¡t|||jtjkd \}}t   ¡}| ||7} | ||7} t  dt   ¡| ||f¡q[t d| | ||f¡|r||krt|||jtjkd \}}t dd„|Dƒ¡}t dd„|Dƒ¡}t dd„|Dƒ¡}tjt |ƒd dd}t |¡|d d…<||||fS)aPerforms a range search with many queries (given by an iterator) and adjusts the threshold on-the-fly so that the total results table does not grow larger than max_results. If ngpu != 0, the function moves the index to this many GPUs to speed up search. Ngš™™™™™é?gø?r.rrkrz searching z vectorsr)r(z-too many results %d > %d, scaling back radiusrz' [%.3f s] %d queries done, %d resultszBsearch done in %.3f s + %.3f s, total %d results, end threshold %gcSsg|]\}}}|‘qSr#r#©r2Únres_iÚdis_iÚids_ir#r#r$r4Tr‘z,range_search_max_results..cSsg|]\}}}|‘qSr#r#r™r#r#r$r4Ur‘cSsg|]\}}}|‘qSr#r#r™r#r#r$r4Vr‘rxr,)r7r r8r9rrrrzr{rr r=r1rirCr>rQr˜rÚMETRIC_INNER_PRODUCTr@rSr|rR)rÚquery_iteratorr–Ú max_resultsÚ min_resultsr{rmÚ clip_to_minrXrlrUÚt_startÚt_searchÚt_post_processÚqtotr—Ú raw_totresr“Úxqirr~rr€ršrYrZr`r‡rˆrƒr#r#r$Úrange_search_max_resultss~          ÿ  þ   ÿÿÿ  þ r¨é é Nccs\t|ƒ}|}d}||kr,||||…}|V||kr |d9}|t|ƒ7}||ks dSdS)z¤ produces batches of progressively increasing sizes. This is useful to adjust the search radius progressively without overflowing with intermediate results réNr0)rÚstart_bsÚmax_bsrÚbsrdr§r#r#r$Úexponential_query_iterator^s€ ûr¯)r&)F)NNFrF)r©rª)r r Únumpyr@ÚloggingÚ getLoggerÚ__name__rrPr%Úknnrir„rŒrr˜r¨r¯r#r#r#r$Ús$ " ^ ÿ 6   þ\