Ë
    °Œj¦C  ã                   ó¬  — d dl Z d dlZd dlZd dlZddlmZmZmZm	Z	m
Z
mZ ddlmZ  G d„ d«      Z G d„ de«      Z ej                   «       Zd	d
de› d�fD ]#  ae j&                  j)                  t$        «      sŒ# n dad„ Z G d„ de«      Zd„ Z G d„ de«      Z G d„ de«      Z G d„ de«      Z G d„ de«      Z G d„ de«      Z G d„ de«      Zdd„Zy) é    Né   )Ú
fvecs_readÚ
ivecs_readÚ
bvecs_mmapÚ
fvecs_mmapÚ
bvecs_iterÚbvecs_iter_chunked)Úknnc                   óN   — e Zd ZdZd„ Zd„ Zdd„Zd„ Zdd„Zdd„Z	dd	„Z
d
„ Zd„ Zy)ÚDatasetz)Generic abstract class for a test datasetc                 óJ   — d| _         d| _        d| _        d| _        d| _        y)z0the constructor should set the following fields:éÿÿÿÿÚL2N©ÚdÚmetricÚnqÚnbÚnt©Úselfs    ú`/var/www/html/Fitness-lenito-AI-main/venv/lib/python3.12/site-packages/faiss/contrib/datasets.pyÚ__init__zDataset.__init__   s%   € àˆŒØˆŒØˆŒØˆŒØˆ�ó    c                 ó   — t        «       ‚)z%return the queries as a (nq, d) array©ÚNotImplementedErrorr   s    r   Úget_querieszDataset.get_queries"   ó   € ä!Ó#Ð#r   Nc                 ó   — t        «       ‚)z%return the queries as a (nt, d) arrayr   ©r   Úmaxtrains     r   Ú	get_trainzDataset.get_train&   r   r   c                 ó   — t        «       ‚)z%return the queries as a (nb, d) arrayr   r   s    r   Úget_databasezDataset.get_database*   r   r   c              #   óÐ   K  — | j                  «       }|\  }}| j                  |z  |z  | j                  |dz   z  |z  }}t        |||«      D ]  }||t        ||z   |«       –— Œ y­w)a7  returns an iterator on database vectors.
        bs is the number of vectors per batch
        split = (nsplit, rank) means the dataset is split in nsplit
        shards and we want shard number rank
        The default implementation just iterates over the full matrix
        returned by get_dataset.
        r   N)r%   r   ÚrangeÚmin©	r   ÚbsÚsplitÚxbÚnsplitÚrankÚi0Úi1Új0s	            r   Údatabase_iteratorzDataset.database_iterator.   sr   è ø€ ð ×ÑÓ ˆØ‰ˆ�Ø—‘˜4‘ 6Ñ)¨4¯7©7°d¸Q±hÑ+?À6Ñ+IˆBˆÜ˜˜B Ö#ˆBØ�Rœ#˜b 2™g rÓ*Ð+Ó+ñ $ùs   ‚A$A&c                 ó   — t        «       ‚)z5return the ground truth for k-nearest neighbor searchr   ©r   Úks     r   Úget_groundtruthzDataset.get_groundtruth<   r   r   c                 ó   — t        «       ‚)z(return the ground truth for range searchr   )r   Úthreshs     r   Úget_groundtruth_rangezDataset.get_groundtruth_range@   r   r   c           
      óˆ   — d| j                   › d| j                  › d| j                  › d| j                  › d| j                  › �
S )Nzdataset in dimension z, with metric z
, size: Q z B z T r   r   s    r   Ú__str__zDataset.__str__D   sD   € à# D§F¡F 8¨>¸$¿+¹+¸ð GØ—w‘w�i˜s 4§7¡7 )¨3¨t¯w©w¨ið9ð	
r   c                 óÒ  — | j                  «       j                  | j                  | j                  fk(  sJ ‚| j                  dkD  rA| j                  d¬«      }|j                  d| j                  fk(  sJ d|j                  ›�«       ‚| j                  «       j                  | j                  | j                  fk(  sJ ‚| j                  d¬«      j                  | j                  dfk(  sJ ‚y)z6runs the previous and checks the sizes of the matricesr   é{   )r"   zshape=é   )r5   N)	r   Úshaper   r   r   r#   r%   r   r6   )r   Úxts     r   Úcheck_sizeszDataset.check_sizesJ   sÁ   € à×ÑÓ!×'Ñ'¨D¯G©G°T·V±VÐ+<Ò<Ð<Ð<Ø�7‰7�QŠ;Ø—‘¨�Ó-ˆBØ—8‘8  T§V¡V˜}Ò,ÑF¸B¿HºHÐ.FÓFÐ,Ø× Ñ Ó"×(Ñ(¨T¯W©W°d·f±fÐ,=Ò=Ð=Ð=Ø×#Ñ# bÐ#Ó)×/Ñ/°D·G±G¸R°=Ò@Ð@Ñ@r   ©N©é€   )r   r   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r#   r%   r2   r6   r9   r;   rA   © r   r   r   r      s3   „ Ù3òò$ó$ò$ó,ó$ó$ò
óAr   r   c                   ó4   — e Zd ZdZdd„Zd„ Zd	d„Zd„ Zd
d„Zy)ÚSyntheticDatasetzOA dataset that is not completely random but still challenging to
    index
    c                 ó  — t         j                  | «       ||||f\  | _        | _        | _        | _        d}||z   |z   }t        j                  j                  |«      }	|	j                  ||f¬«      }
t        j                  |
|	j                  ||«      «      }
|
|	j                  |«      dz  dz   z  }
t        j                  |
«      }
|
j                  d«      }
|| _        |
d | | _        |
|||z    | _        |
||z   d  | _        y )Né
   )Úsizeé   gš™™™™™¹?Úfloat32)r   r   r   r   r   r   ÚnpÚrandomÚRandomStateÚnormalÚdotÚrandÚsinÚastyper   r@   r,   Úxq)r   r   r   r   r   r   ÚseedÚd1ÚnÚrsÚxs              r   r   zSyntheticDataset.__init__Y   sð   € Ü×Ñ˜ÔØ,-¨r°2°r¨MÑ)ˆŒ�”˜œ $¤'ØˆØ�‰G�b‰LˆÜ�Y‰Y×"Ñ" 4Ó(ˆØ�I‰I˜A˜r˜7ˆIÓ#ˆÜ�F‰F�1�b—g‘g˜b !“nÓ%ˆð �—‘˜“˜a‘ #Ñ%Ñ&ˆÜ�F‰F�1‹IˆØ�H‰H�YÓˆØˆŒØ�C�R�&ˆŒØ�B˜˜b™�/ˆŒØ�B˜‘G�I�,ˆ�r   c                 ó   — | j                   S rB   )rY   r   s    r   r   zSyntheticDataset.get_queriesk   ó   € Ø�w‰wˆr   Nc                 ó@   — |�|n| j                   }| j                  d | S rB   )r   r@   r!   s     r   r#   zSyntheticDataset.get_trainn   s#   € Ø'Ð3‘8¸¿¹ˆØ�w‰w�y˜Ð!Ð!r   c                 ó   — | j                   S rB   )r,   r   s    r   r%   zSyntheticDataset.get_databaser   r`   r   c                 ó´   — t        | j                  | j                  || j                  dk(  rt        j
                  «      d   S t        j                  «      d   S )Nr   r   )r
   rY   r,   r   ÚfaissÚ	METRIC_L2ÚMETRIC_INNER_PRODUCTr4   s     r   r6   z SyntheticDataset.get_groundtruthu   s`   € ÜØ�G‰GØ�G‰GØð —;‘; $Ò&ô —‘ó	
ð ñ	ð 		ô ×/Ñ/ó	
ð ñ	ð 		r   )r   i:  rB   )éd   ©	rE   rF   rG   rH   r   r   r#   r%   r6   rI   r   r   rK   rK   T   s    „ ñóò$ó"òô
r   rK   z/datasets01/simsearch/041218/z7/mnt/vol/gfsai-flash3-east/ai-group/datasets/simsearch/z/home/z/simsearch/data/zdata/c                 ó   — | a y rB   )Údataset_basedir)Úpaths    r   Úset_dataset_basedirrl   –   s   € à�Or   c                   ó2   — e Zd ZdZd„ Zd„ Zdd„Zd„ Zdd„Zy)	ÚDatasetSIFT1Mz_
    The original dataset is available at: http://corpus-texmex.irisa.fr/
    (ANN_SIFT1M)
    c                 ó€   — t         j                  | «       d\  | _        | _        | _        | _        t        dz   | _        y )N)rD   é † é@B é'  zsift1M/©r   r   r   r   r   r   rj   Úbasedirr   s    r   r   zDatasetSIFT1M.__init__¡   ó2   € Ü×Ñ˜ÔØ,GÑ)ˆŒ�”˜œ $¤'Ü&¨Ñ2ˆ�r   c                 ó2   — t        | j                  dz   «      S )Nzsift_query.fvecs©r   rt   r   s    r   r   zDatasetSIFT1M.get_queries¦   ó   € Ü˜$Ÿ,™,Ð);Ñ;Ó<Ð<r   Nc                 óX   — |�|n| j                   }t        | j                  dz   «      d | S )Nzsift_learn.fvecs©r   r   rt   r!   s     r   r#   zDatasetSIFT1M.get_train©   ó.   € Ø'Ð3‘8¸¿¹ˆÜ˜$Ÿ,™,Ð);Ñ;Ó<¸Y¸hÐGÐGr   c                 ó2   — t        | j                  dz   «      S )Nzsift_base.fvecsrw   r   s    r   r%   zDatasetSIFT1M.get_database­   ó   € Ü˜$Ÿ,™,Ð):Ñ:Ó;Ð;r   c                 ó^   — t        | j                  dz   «      }|�|dk  sJ ‚|d d …d |…f   }|S )Nzsift_groundtruth.ivecsrg   ©r   rt   ©r   r5   Úgts      r   r6   zDatasetSIFT1M.get_groundtruth°   ó<   € Ü˜Ÿ™Ð'?Ñ?Ó@ˆØˆ=Ø˜’8ˆO�8Ø’A�r˜�r�E‘ˆBØˆ	r   rB   rh   rI   r   r   rn   rn   ›   ó!   „ ñò
3ò
=óHò<ôr   rn   c                 ó0   — t        j                  | d¬«      S )NrP   ©Údtype)rQ   Úascontiguousarray)r^   s    r   Úsanitizerˆ   ¸   s   € Ü×Ñ ¨Ô3Ð3r   c                   ó<   — e Zd ZdZd	d„Zd„ Zd
d„Zd
d„Zd„ Zdd„Z	y)ÚDatasetBigANNz_
    The original dataset is available at: http://corpus-texmex.irisa.fr/
    (ANN_SIFT1B)
    c                 ó¬   — t         j                  | «       |dv sJ ‚|| _        |dz  }dd|df\  | _        | _        | _        | _        t        dz   | _        y )N)
r   é   é   rM   é   é2   rg   éÈ   iô  éè  rq   rD   é áõrr   zbigann/)	r   r   Únb_Mr   r   r   r   rj   rt   )r   r“   r   s      r   r   zDatasetBigANN.__init__Â   sZ   € Ü×Ñ˜ÔØÐAÑAÐAÐAØˆŒ	Ø�E‰\ˆØ,/°¸¸EÐ,AÑ)ˆŒ�”˜œ $¤'Ü&¨Ñ2ˆ�r   c                 óJ   — t        t        | j                  dz   «      d d  «      S )Nzbigann_query.bvecs)rˆ   r   rt   r   s    r   r   zDatasetBigANN.get_queriesÊ   s!   € Üœ
 4§<¡<Ð2FÑ#FÓGÉÐJÓKÐKr   Nc                 ój   — |�|n| j                   }t        t        | j                  dz   «      d | «      S )Nzbigann_learn.bvecs)r   rˆ   r   rt   r!   s     r   r#   zDatasetBigANN.get_trainÍ   s8   € Ø'Ð3‘8¸¿¹ˆÜÜ�t—|‘|Ð&:Ñ:Ó;¸I¸XÐFó
ð 	
r   c                 óx   — t        | j                  d| j                  z  z   «      }|�|dk  sJ ‚|d d …d |…f   }|S )Nzgnd/idx_%dM.ivecsrg   )r   rt   r“   r€   s      r   r6   zDatasetBigANN.get_groundtruthÓ   sE   € Ü˜Ÿ™Ð':¸T¿Y¹YÑ'FÑFÓGˆØˆ=Ø˜’8ˆO�8Ø’A�r˜�r�E‘ˆBØˆ	r   c                 óŠ   — | j                   dk  sJ d«       ‚t        t        | j                  dz   «      d | j                   «      S )Nrg   údataset too large, use iteratorúbigann_base.bvecs)r“   rˆ   r   rt   r   r   s    r   r%   zDatasetBigANN.get_databaseÚ   sB   € Ø�y‰y˜3ŠÐAÐ AÓAˆÜÜ�t—|‘|Ð&9Ñ9Ó:¸9¸T¿W¹WÐEó
ð 	
r   c           	   #   óò   K  — t        | j                  dz   «      }|\  }}| j                  |z  |z  | j                  |dz   z  |z  }}t        |||«      D ]  }t	        ||t        ||z   |«       «      –— Œ! y ­w)Nr™   r   )r   rt   r   r'   rˆ   r(   r)   s	            r   r2   zDatasetBigANN.database_iteratorà   s{   è ø€ Ü˜Ÿ™Ð':Ñ:Ó;ˆØ‰ˆ�Ø—‘˜4‘ 6Ñ)¨4¯7©7°d¸Q±hÑ+?À6Ñ+IˆBˆÜ˜˜B Ö#ˆBÜ˜2˜b¤3 r¨B¡w°Ó#3Ð4Ó5Ó5ñ $ùó   ‚A5A7)r‘   rB   rC   ©
rE   rF   rG   rH   r   r   r#   r6   r%   r2   rI   r   r   rŠ   rŠ   ¼   s&   „ ñó
3òLó
óò
ô6r   rŠ   c                   ó<   — e Zd ZdZd	d„Zd„ Zd
d„Zd
d„Zd„ Zdd„Z	y)ÚDatasetDeep1Bzv
    See
    https://github.com/facebookresearch/faiss/tree/main/benchs#getting-deep1b
    on how to get the data
    c                 óì   — t         j                  | «       ddddddœ}||v sJ ‚dd|d	f\  | _        | _        | _        | _        t        d
z   | _        | j                  ›d|| j                     ›d�| _        y )NÚ100kÚ1MÚ10MÚ100MÚ1B)rp   rq   é€–˜ r’   é Êš;é`   i€ø]rr   zdeep1b/Údeepz_groundtruth.ivecs)	r   r   r   r   r   r   rj   rt   Úgt_fname)r   r   Ú
nb_to_names      r   r   zDatasetDeep1B.__init__ï   s}   € Ü×Ñ˜ÔàØØØØñ
ˆ
ð �ZÑÐÐØ,.°	¸2¸uÐ,DÑ)ˆŒ�”˜œ $¤'Ü&¨Ñ2ˆŒà�L‹LØ�t—w‘wÓð
ˆ�r   c                 óD   — t        t        | j                  dz   «      «      S )Nzdeep1B_queries.fvecs)rˆ   r   rt   r   s    r   r   zDatasetDeep1B.get_queries   s   € Üœ
 4§<¡<Ð2HÑ#HÓIÓJÐJr   Nc                 ój   — |�|n| j                   }t        t        | j                  dz   «      d | «      S )Nzlearn.fvecs)r   rˆ   r   rt   r!   s     r   r#   zDatasetDeep1B.get_train  s2   € Ø'Ð3‘8¸¿¹ˆÜœ
 4§<¡<°-Ñ#?Ó@ÀÀ(ÐKÓLÐLr   c                 óX   — t        | j                  «      }|�|dk  sJ ‚|d d …d |…f   }|S )Nrg   )r   r©   r€   s      r   r6   zDatasetDeep1B.get_groundtruth  s6   € Ü˜Ÿ™Ó&ˆØˆ=Ø˜’8ˆO�8Ø’A�r˜�r�E‘ˆBØˆ	r   c                 óŠ   — | j                   dk  sJ d«       ‚t        t        | j                  dz   «      d | j                    «      S )Nr’   r˜   ú
base.fvecs)r   rˆ   r   rt   r   s    r   r%   zDatasetDeep1B.get_database  s>   € Ø�w‰w˜%ÒÐBÐ!BÓBÐÜœ
 4§<¡<°,Ñ#>Ó?À	À$Ç'Á'ÐJÓKÐKr   c           	   #   óò   K  — t        | j                  dz   «      }|\  }}| j                  |z  |z  | j                  |dz   z  |z  }}t        |||«      D ]  }t	        ||t        ||z   |«       «      –— Œ! y ­w)Nr¯   r   )r   rt   r   r'   rˆ   r(   r)   s	            r   r2   zDatasetDeep1B.database_iterator  sz   è ø€ Ü˜Ÿ™ |Ñ3Ó4ˆØ‰ˆ�Ø—‘˜4‘ 6Ñ)¨4¯7©7°d¸Q±hÑ+?À6Ñ+IˆBˆÜ˜˜B Ö#ˆBÜ˜2˜b¤3 r¨B¡w°Ó#3Ð4Ó5Ó5ñ $ùr›   )r¦   rB   rC   rœ   rI   r   r   rž   rž   è   s(   „ ñó
ò"KóMóòLô6r   rž   c                   ó,   — e Zd ZdZdd„Zd„ Zd„ Zdd„Zy)	ÚDatasetGlovezD
    Data from http://ann-benchmarks.com/glove-100-angular.hdf5
    Nc                 ó  — dd l }|rJ d«       ‚|s	t        dz   }|j                  |d«      | _        d| _        d\  | _        | _        | j                  d   j                  d   | _        | j                  d   j                  d   | _	        y )	Nr   znot implementedzglove/glove-100-angular.hdf5ÚrÚIP)rg   r   ÚtrainÚtest)
Úh5pyrj   ÚFileÚ
glove_h5pyr   r   r   r?   r   r   )r   ÚlocÚdownloadr¸   s       r   r   zDatasetGlove.__init__  s   € ÛáÐ.Ð.Ó.ˆ|ÙÜ!Ð$BÑBˆCØŸ)™) C¨Ó-ˆŒð ˆŒØ ‰ˆŒ�”Ø—/‘/ 'Ñ*×0Ñ0°Ñ3ˆŒØ—/‘/ &Ñ)×/Ñ/°Ñ2ˆ�r   c                 ót   — t        j                  | j                  d   «      }t        j                  |«       |S )Nr·   ©rQ   Úarrayrº   rd   Únormalize_L2©r   rY   s     r   r   zDatasetGlove.get_queries-  s,   € Ü�X‰X�d—o‘o fÑ-Ó.ˆÜ×Ñ˜2ÔØˆ	r   c                 ót   — t        j                  | j                  d   «      }t        j                  |«       |S )Nr¶   r¾   ©r   r,   s     r   r%   zDatasetGlove.get_database2  s,   € Ü�X‰X�d—o‘o gÑ.Ó/ˆÜ×Ñ˜2ÔØˆ	r   c                 óL   — | j                   d   }|�|dk  sJ ‚|d d …d |…f   }|S )NÚ	neighborsrg   )rº   r€   s      r   r6   zDatasetGlove.get_groundtruth7  s6   € Ø�_‰_˜[Ñ)ˆØˆ=Ø˜’8ˆO�8Ø’A�r˜�r�E‘ˆBØˆ	r   )NFrB   ©rE   rF   rG   rH   r   r   r%   r6   rI   r   r   r²   r²     s   „ ñó3òò
ô
r   r²   c                   ó*   — e Zd ZdZd„ Zd„ Zd„ Zdd„Zy)ÚDatasetMusic100zO
    get dataset from
    https://github.com/stanis-morozov/ip-nsw#dataset
    c                 óŽ   — t         j                  | «       d\  | _        | _        | _        | _        d| _        t        dz   | _        y )N)rg   r   rq   rr   rµ   z
music-100/)	r   r   r   r   r   r   r   rj   rt   r   s    r   r   zDatasetMusic100.__init__E  s9   € Ü×Ñ˜ÔØ,@Ñ)ˆŒ�”˜œ $¤'ØˆŒÜ&¨Ñ5ˆ�r   c                 ór   — t        j                  | j                  dz   d¬«      }|j                  dd«      }|S )Nzquery_music100.binrP   r…   r   rg   ©rQ   Úfromfilert   ÚreshaperÁ   s     r   r   zDatasetMusic100.get_queriesK  s1   € Ü�[‰[˜Ÿ™Ð(<Ñ<ÀIÔNˆØ�Z‰Z˜˜CÓ ˆØˆ	r   c                 ór   — t        j                  | j                  dz   d¬«      }|j                  dd«      }|S )Nzdatabase_music100.binrP   r…   r   rg   rË   rÃ   s     r   r%   zDatasetMusic100.get_databaseP  s6   € Ü�[‰[Ø�L‰LÐ2Ñ2¸)ô
ˆð �Z‰Z˜˜CÓ ˆØˆ	r   Nc                 ór   — t        j                  | j                  dz   «      }|�|dk  sJ ‚|d d …d |…f   }|S )Nzgt.npyrg   )rQ   Úloadrt   r€   s      r   r6   zDatasetMusic100.get_groundtruthW  s?   € Ü�W‰W�T—\‘\ HÑ,Ó-ˆØˆ=Ø˜’8ˆO�8Ø’A�r˜�r�E‘ˆBØˆ	r   rB   rÆ   rI   r   r   rÈ   rÈ   ?  s   „ ñò
6òò
ôr   rÈ   c                   ó2   — e Zd ZdZd„ Zd„ Zdd„Zd„ Zdd„Zy)	ÚDatasetGIST1Mz_
    The original dataset is available at: http://corpus-texmex.irisa.fr/
    (ANN_GIST1M)
    c                 ó€   — t         j                  | «       d\  | _        | _        | _        | _        t        dz   | _        y )N)iÀ  rp   rq   rr   zgist1M/rs   r   s    r   r   zDatasetGIST1M.__init__e  ru   r   c                 ó2   — t        | j                  dz   «      S )Nzgist_query.fvecsrw   r   s    r   r   zDatasetGIST1M.get_queriesj  rx   r   Nc                 óX   — |�|n| j                   }t        | j                  dz   «      d | S )Nzgist_learn.fvecsrz   r!   s     r   r#   zDatasetGIST1M.get_trainm  r{   r   c                 ó2   — t        | j                  dz   «      S )Nzgist_base.fvecsrw   r   s    r   r%   zDatasetGIST1M.get_databaseq  r}   r   c                 ó^   — t        | j                  dz   «      }|�|dk  sJ ‚|d d …d |…f   }|S )Nzgist_groundtruth.ivecsrg   r   r€   s      r   r6   zDatasetGIST1M.get_groundtrutht  r‚   r   rB   rh   rI   r   r   rÒ   rÒ   _  rƒ   r   rÒ   c                   óJ   — e Zd ZdZdd„Zd„ Zdd„Zd„ Zdd„Zdd„Z	dd	„Z
d
„ Zy)ÚDatasetDINO10Ba)  
    Data from https://dl.fbaipublicfiles.com/large_objects/dino_vitl_10B/
    The dataset contains 10 billion 1024-d vectors extracted from image
    patches from the YFCC100M dataset, using a Dino-ViT-L 16 model
    (facebook/dinov3-vitl16-pretrain-lvd1689m).
    The dataset is sharded in multiple chunked .bvecs files. Downloading
    instructions can be obtained with
    "wget https://dl.fbaipublicfiles.com/large_objects/dino_vitl_10B/README.md".
    Supported sizes : 100k 200k 500k 1M ... 5B 10B listed in supported_nbs
    (see __init__).
    c                 óô  — t         j                  | «       g d¢}||vr|st        d|› d|› �«      ‚t        j                  j                  t        «      st        dt        › �«      ‚t        dz   | _        | j                  dz   | _        t        j                  j                  | j                  «      sJ d| j                  › �«       ‚| j                  dz   | _	        t        j                  j                  | j                  «      sJ d	|› d
| j                  › �«       ‚| j                  dz   dz   t        |«      z   dz   dz   | _        | j                  dz   | _        || _        d| _        d| _        d| _        d| _        y )N)rp   i@ i ¡ rq   i€„ i@KL r¥   i -1i€ðúr’   i Âëi eÍr¦   i ”5wl    rT ì    d(	 zUnsupported dataset size: z, supported values are: z0Provided dataset base directory does not exist: zdino_vitl_10B/Úchunked_base_10Bz2Index path should exist, check your dataset path: zqueries_clean.bvecsz*Queries path should exist as dataset size z is supported: zgts/Úgts_dino_patch_Ú_zk10.npyztrain_queries_99M.bvecsi   rp   iÀžær   )r   r   Ú
ValueErrorÚosrk   Úexistsrj   rt   ÚindexdirÚ
queriesdirÚstrÚgtsdirÚtrain_queriesdirr   r   r   r   r   )r   r   Úignore_supportedÚsupported_nbss       r   r   zDatasetDINO10B.__init__‰  s�  € Ü×Ñ˜Ôò
ˆð$ �]Ñ"Ñ+;ÜØ,¨R¨DÐ0HØ �/ð#óð ô �w‰w�~‰~œoÔ.ÜØBÜ"Ð#ð%óð ô 'Ð)9Ñ9ˆŒØŸ™Ð'9Ñ9ˆŒÜ�w‰w�~‰~Ø�M‰Mô
ð 	Pà?ÀÇÁ¸ÐOó	Pð 
ð Ÿ,™,Ð)>Ñ>ˆŒÜ�w‰w�~‰~˜dŸo™oÔ.ð 	
Ø8¸¸¸OØ�‰Ðð!ó	
Ð.ð
 �L‰LØñàñ ô �"‹gñð ñ	ð
 ñð 	Œð !%§¡Ð/HÑ HˆÔØˆŒØˆŒØˆŒØˆŒØˆ�r   c                 óB   — t        | j                  «      }t        |«      S )z!Get all vectors as a single array)r   rã   rˆ   )r   Úqueriess     r   r   zDatasetDINO10B.get_queriesÀ  s   € ä˜TŸ_™_Ó-ˆÜ˜Ó Ð r   Nc                 óh   — |�|dkD  rt        d«      ‚t        t        | j                  «      d| «      S )z,Get training query vectors as a single arrayNr¥   zºThe training set is potentially too large to fit in RAM (400 GB of data). Please use train_iterator or use maxtrain parameter below 10_000_000 to get the first maxtrain training vectors.)r   rˆ   r   ræ   r!   s     r   r#   zDatasetDINO10B.get_trainÅ  s@   € àÐ˜x¨*Ò4Ü%ð-óð ô œ
 4×#8Ñ#8Ó9¸)¸8ÐDÓEÐEr   c                 ó¦   — | j                   dkD  rt        d«      ‚t        t        | j                  | j                   ¬«      j                  «       «      S )z*Get all database vectors as a single arrayr¥   z„The dataset is potentially too large to fit in RAM. Please use database_iterator or use a dataset size equal to or below 10_000_000.©Ú
batch_size)r   r   rˆ   r	   râ   Ú__next__r   s    r   r%   zDatasetDINO10B.get_databaseÐ  sJ   € à�7‰7�ZÒÜ%ð*óð ô Ü" 4§=¡=¸T¿W¹WÔE×NÑNÓPóð r   c              #   ó
  K  — d}t        | j                  |¬«      D ]c  }||j                  d   z   | j                  kD  r|d| j                  |z
   }t	        |«      –— ||j                  d   z  }|| j                  k\  sŒc y y­w)zgIterator over the database of size nb, corresponding to the
        first nb vectors in the .bvecs filer   rí   N)r	   râ   r?   r   rˆ   )r   r*   Ú
total_readÚbatchs       r   r2   z DatasetDINO10B.database_iteratorÝ  s}   è ø€ ð ˆ
Ü'¨¯©À"×EˆEØ˜EŸK™K¨™NÑ*¨T¯W©WÒ4ØÐ4 §¡¨*Ñ 4Ð5�Ü˜5“/Ò!Ø˜%Ÿ+™+ a™.Ñ(ˆJØ˜TŸW™WÓ$Ùñ Fùs   ‚A<BÁ?Bc              #   ó^   K  — t        | j                  |¬«      D ]  }t        |«      –— Œ y­w)z;Iterator over all training query vectors in the .bvecs filerí   N)r   ræ   rˆ   )r   r*   rò   s      r   Útrain_iteratorzDatasetDINO10B.train_iteratoré  s(   è ø€ ä × 5Ñ 5À"×EˆEÜ˜5“/Ó!ñ Fùs   ‚+-c                 óz   — |dkD  rt        d«      ‚t        j                  | j                  «      }|dd…d|…f   }|S )zGet ground truth from .npy filerM   z+Ground truth files only available for k<=10N)r   rQ   rÐ   rå   )r   r5   Úgtss      r   r6   zDatasetDINO10B.get_groundtruthî  sC   € àˆrŠ6Ü%Ø=óð ô �g‰g�d—k‘kÓ"ˆØ’!�R�a�R�%‰jˆØˆ
r   c                  ó   — y)NÚ	euclideanrI   r   s    r   ÚdistancezDatasetDINO10B.distanceø  s   € Ør   )FrB   )rr   )rM   )rE   rF   rG   rH   r   r   r#   r%   r2   rô   r6   rù   rI   r   r   rÙ   rÙ   |  s1   „ ñ
ó5òn!ó
	Fòó
ó"ó
ór   rÙ   c                 ó:  — | dk(  r
t        «       S | dk(  r
t        «       S | j                  d«      r!| dk(  rdnt        | dd «      }t	        |¬«      S | j                  d	«      r[| d
d }|d   dk(  rdt        |dd «      z  }n0|dk(  rd}n(|d   dk(  rdt        |dd «      z  }nt        d|z   «      ‚t        |¬«      S | dk(  r
t        «       S | dk(  rt        |¬«      S | j                  d«      r!| dk(  rdnt        | d
d «      }t        |¬«      S t        d| z   «      ‚)zŒconverts a string describing a dataset to a Dataset object
    Supports sift1M, bigann1M..bigann1B, deep1M..deep1B, music-100 and glove
    Úsift1MÚgist1MÚbigannÚbigann1Br‘   é   r   )r“   r¨   rO   NÚMrq   r¤   r¦   r5   zdid not recognize suffix )r   z	music-100Úglove)r¼   ÚdinoÚdino10BrÛ   zunknown dataset )rn   rÒ   Ú
startswithÚintrŠ   ÚAssertionErrorrž   rÈ   r²   rÙ   ÚRuntimeError)Údatasetr¼   ÚdbsizeÚszsufs       r   Údataset_from_namer  ü  s@  € ð
 �(ÒÜ‹Ðà	�HÒ	Ü‹Ðà	×	Ñ	˜HÔ	%Ø  JÒ.‘´C¸ÀÀ"¸Ó4FˆÜ &Ô)Ð)à	×	Ñ	˜FÔ	#à˜˜�ˆØ�‰9˜ÒØœS  s¨ ›_Ñ,‰FØ�dŠ]Ø‰FØ�2‰Y˜#ÒØœC  c r 
›OÑ+‰Fä Ð!<¸uÑ!DÓEÐEÜ Ô'Ð'à	�KÒ	ÜÓ Ð à	�GÒ	Ü XÔ.Ð.à	×	Ñ	˜FÔ	#Ø#*¨iÒ#7‘¼SÀÈÈÀÓ=MˆÜ Ô(Ð(ô Ð-°Ñ7Ó8Ð8r   )Údeep1MF)rà   ÚnumpyrQ   rd   ÚgetpassÚvecs_ior   r   r   r   r   r	   Úexhaustive_searchr
   r   rK   ÚgetuserÚusernamerj   rk   rá   rl   rn   rˆ   rŠ   rž   r²   rÈ   rÒ   rÙ   r  rI   r   r   Ú<module>r     só   ðó 
Û Û Û ÷÷ õ #÷:Añ :Aôz+�wô +ðh ˆ7�?‰?Ó€ð $Ø=ØˆXˆJÐ&Ð'ó€Oð
 
‡w�w‡~�~”oÕ&Ùðð €Oòô
�Gô ò:4ô)6�Gô )6ôX/6�Gô /6ôd"�7ô "ôJ�gô ô@�Gô ô:}�Wô }ô@'9r   