o ڭj*@sdZddlZddlZddlZddlZddlZddlmZddlm Z ddl m Z m Z m Z mZmZmZmZmZmZmZmZmZmZddlmZddlZddlZddlm ZddlmZdd l m!Z!d d l"m#Z#d d l$m%Z%m&Z&m'Z'd d l(m)Z)m*Z*d dl+m,Z,m-Z-d dl.m/Z0e rddl1m2Z3ne Z3e4dZ5e5j6dddZ7de8de8deeeej9ej9fee:e:ffddffddZ;defddZe7j?deej9ej9ffd d!Z@e7j?deej9ej9ffd"d#ZAe7j?deej9ej9ffd$d%ZBe7j?deej9ej9ffd&d'ZCe7j?dee3ej9ffd(d)ZDe7j?d*eEdee!jFej9ej9e!jFej9ej9e!jFej9ej9f fd+d,ZG -dld-d.d/d0e8de8d1e8d2eHd3eHd4e8deeej9eej9eej9ffd5d6ZIee!jFejJejKejJejKfZLe Gd7d8d8ZMGd9d:d:eZNGd;d<d<ZOd=ejJejKdeejJejJejJffd>d?ZP @dmdAe!jFdBejJejKdCejJejKdDeQdejJejRf dEdFZSdGee!jFejJejKejJejKfdHejJejRdeMfdIdJZTdKeNdeeMeeMffdLdMZUdAe!jFdBejJejKdCejJejKdNejJejKdOejJejVdee!jFejJejKejJejKejJejKff dPdQZWdRe dSee&dTeEddfdUdVZXe7j?de8de8dWeQdXeHdeee!jFej9ff dYdZZYd[e8d\e8deeEfd]d^ZZd_d`d-d.ejVdadbde8de8dce8ddeHdWeQdeeQdfeHd4e8dgej j[dTeEdee,ej9ffdhdiZ\Gdjdkdke%Z]dS)nzUtilities for data generation.N)ThreadPoolExecutor) dataclass) TYPE_CHECKINGAnyCallableDict GeneratorList NamedTupleOptionalSequenceSetTupleTypeUnion)request)typing)r)sparse)concat)DataIterDMatrixQuantileDMatrix)is_pd_cat_dtypepandas_pyarrow_mapper) ArrayLike XGBRanker)train) DataFramejoblibz ./cachedir)verbose n_samples n_featuresreturnc csvtd}tjd}|jdd||d||}tjtjtj tj tj tj tj tjtjtjtjtjtjtjtjtjtjtjtjtjg}|D]}tj||d}||fV||fVqF|D]}tj||d}||}||} || fVq`|jdd||d ||}tjt fD]} tj|| d}||fVqtjt fD]} tj|| d}||}||} || fVqd S) z*Enumerate all supported dtypes from numpy.pandasrlowhighsizedtypeg?r*N)!pytest importorskipnprandom RandomStaterandintreshapeint32int64byteshortintcint_longlonguint32uint64ubyteushortuintcuint ulonglongfloat16float32float64halfsingledoublearraytolistrbinomialbool_bool) r!r"pdrngorigdtypesr,Xdf_origdfdtype1dtype2rXQ/var/www/html/CropPilot/venv/lib/python3.10/site-packages/xgboost/testing/data.py np_dtypes1s`          rZc csVtd}|||||||| g}t j }|j dd|dgdd|dgdt j d}t j d|jfD]}|D]}|j dd|dgdd|dgd|d}||fVqAq=t j }||g}|j d d |d gd d |d gdt j d}t j d|jfD];}|D]6}|j d d |d gd d |d gd|d}||fV|d }|d }t||jsJt||jsJ||fVqq|d }|jD] }||jt||<qt j d|jfD]}|j dd|dgdd|dgd|d}||fVqd|jfD].}dd|dgdd|dgd} |j | |durt jn|d}|j | |d}||fVqdS)z/Enumerate all supported pandas extension types.r$r-rf0f1r+N?g@g@r^categoryTF)r/r0 UInt8Dtype UInt16Dtype UInt32Dtype UInt64Dtype Int8Dtype Int16Dtype Int32Dtype Int64Dtyper1nanrrENA Float32Dtype Float64Dtype isinstanceSeriesastypecolumnscatrename_categoriesintCategoricalDtyperM BooleanDtype) rOrRNullrQr,rUser_origsercdatarXrXrY pd_dtypeslsl       " r|ccs0td}td}t}d|jdfD]J}|D]E}|ds#|dr$q||s/|dkr/|ntj}|jdd|d gd d |dgd tj d }|jdd|d gd d |dgd |d }||fVqq|jd fD]2}|jdd |dgd d|dgd | d }|jdd |dgd d|dgd | | d }||fVqcdS)z*Pandas DataFrame with pyarrow backed type.r$pyarrowNrrDrNr-rr[r\r]r+FT) r/r0rrk startswithisnar1rjrrErv ArrowDtyperM)rOparRrwr, orig_nullrQrUrXrXrYpd_arrow_dtypess:     rrPcCs|jdddd}|jdd}tj|d<tjtdd t||Wdn1s,wYtjtddt||WddS1sIwYdS) zValidate there's no inf in X. r.r\)rzInput data contains `inf`matchN) r2r5r1infr/raises ValueErrorrr)rPrSyrXrXrY check_infs    "rcsdtjdtd}dttdttdttdtjffdd }d td tdtjffd d }||ddgddgddg|ddgddgddg|ddd|ddd|dd d|d!d"d|d#d$d|d%d&d|d'd(dd) }||j d*g }|d* }||fS)+zSynthesize a dataset similar to the sklearn California housing dataset. The real one can be obtained via: .. code-block:: import sklearn.datasets X, y = sklearn.datasets.fetch_california_housing(return_X_y=True) iPir$meanssigmasweightsr#csXjt|d|d|dd}j|jd|d|dd}tj||gddS)Nr)r*locscaler-axis)normalrtshaper1 concatenate)rrrl0l1r!rPrXrY mixture_2comps "z-get_california_housing..mixture_2compmeanstdcsj||fdS)Nrrr*)rrrrrXrYnormsz$get_california_housing..normg5ŀ]g~(Fv^gr-|E?g3mE^1?gDi-T?gÅv-W?gXcB@g& @@g6 ?g](?g8W nx?gd?g|["@g2{e?rgVb<@g>+)@gZK@g@g)P=?g˧ ^T?g/E@g@gI@gtbO$@gg9h@gk}v?) LongitudeLatitudeMedIncHouseAgeAveRooms AveBedrms PopulationAveOccup MedHouseValr) r1r2 default_rngr/r0r floatndarrayrrq differenceto_numpy)rOrrrUrSrrXrrYget_california_housingsH           rcCstd}|}|j|jfS)z&Fetch the digits dataset from sklearn.sklearn.datasets)r/r0 load_digitsr{target)datasetsr{rXrXrY get_digits,s  rcCstd}|jddS)z-Fetch the breast cancer dataset from sklearn.rT) return_X_y)r/r0load_breast_cancer)rrXrXrY get_cancer4s  rc Cstd}tjd}d}d}|j||d\}}|d||j}t|jdD]}t|jdD]}|||fr?tj |||f<q0q'||fS)zGenerate a sparse dataset.rig?) random_stater-r) r/r0r1r2r3make_regressionrLrrangerj) rrPnsparsityrSrflagijrXrXrY get_sparse;s   rc strddlntdtjdd}dtt t t ft fdt dj ffd d }|d d d dddd|d<|ddddd|d<|dddddd|d<|dd d!d"d#d$d%d&d|d'<|d(d)d*d"d+d,|d-<|d.d)d/d0d1d#d2d3d4d|d5<|d6d7d8d9d:d;d<|d=<|d>d?d@dAd%dBd|dC<|dDdEd d#dFd|dG<|dAdHdHdIdJ|dK<dLt dMt dt dj ffdNdO }|dPdQd|dR<|dSdTd|dU<|dVdWd|dX<|dYdZd|d[<|d\d]d|d^<|d_d`d|da<|dbdcd|dd<|dedfd|dg<|dhdid|dj<|dkdld|dm<t |j}|||}tjfdn}|jD]"}t||jjrD|||jjtj7}q*|||j7}q*|do|9}|dp|7}||fS)qamGet a synthetic version of the amse housing dataset. The real one can be obtained via: .. code-block:: from sklearn import datasets datasets.fetch_openml(data_id=42165, as_frame=True, return_X_y=True) Number of samples: 1460 Number of features: 20 Number of categorical features: 10 Number of numerical features: 10 rNr$r%i name_probadensityr#c std|}td|dko|dk}|r d|}||tj<t|}t|}|ddt|7<j||d}j | t dd|d }|S) Nr-r`ư>r)r*pcSs t|tSN)rnstr)xrXrXrYxs z5get_ames_housing..synth_cat..r+) rtr1absrjlistkeysvaluessumchoicerorufilter) rrn_nullshas_nanrrrrseriesr!rOrPrXrY synth_catfs     z#get_ames_housing..synth_catgqu]?gqh.?gsmB.synth_numgmtF @gOfKhttps://s3-us-west-2.amazonaws.com/xgboost-examples/MQ2008.zipz MQ2008.zip)urlfilenamer)pathNzMQ2008/Fold1/train.txtzMQ2008/Fold1/test.txtzMQ2008/Fold1/vali.txtTF)query_id zero_based) r/r0osrjoinexistsr urlretrievezipfileZipFile extractallload_svmlight_files)rrsrcrfx_trainy_train qid_trainx_testy_testqid_testx_validy_valid qid_validrXrXrY get_mq2008sH      r/Fr%) vary_sizern_samples_per_batch n_batchesuse_cupyr0rcCsg}g}g}|rddl} | jt|} ntj|} t|D].} |r*|| dn|} | | |} | | }| jdd| d}|| ||||q |||fS)zMake batches of dense data.rN r-r') cupyr2r3r1r>rrandnuniformappend)r1r"r2r3r0rrSrwr5rPrr!_X_y_wrXrXrY make_batches<s          r=c@sleZdZUdZejed<eje j ed<eje j ed<eje j ed<eje j ed<eje j ed<dS) ClickFoldzCA structure containing information about generated user-click data.rSrqidscoreclickposN) __name__ __module__ __qualname____doc__r csr_matrix__annotations__nptNDArrayr1r6rEr7rXrXrXrYr>]s  r>c@s8eZdZUdZeed<eed<eed<defddZdS) RelDataCVzPSimple data struct for holding a train-test split of a learning to rank dataset.rtestmax_relr#cCs |jdkS)z6Whether the label consists of binary relevance degree.r-)rMselfrXrXrY is_binaryps zRelDataCV.is_binaryN) rCrDrErFRelDatarHrtrNrPrXrXrXrYrKis rKc@sPeZdZdZdeddfddZdejej dejej dejej fd d Z dS) PBMaSimulate click data with position bias model. There are other models available in `ULTRA `_ like the cascading model. References ---------- Unbiased LambdaMART: An Unbiased Pairwise Learning-to-Rank Algorithm etar#NcCs0tgd|_tgd}t|||_dS)N)rg{Gz?Q?gp= ף?r`) g(\?gQ?gQ?g(\?rT皙?g)\(?rg{Gz?gQ?)r1rJ click_probpower exam_prob)rOrSrXrXrXrY__init__s z PBM.__init__labelspositionc Cstj|dd}t|j}d||dk<d||t|jk<|j|}t|j}|j|jks/Jtj|dd}d|||jjk<|j|}tj d}|j|jdtj d}tj|jtj d}d||||k<|S) zSample clicks for one query based on input relevance degree and position. Parameters ---------- labels : relevance_degree T)copyrrr%)r*r,r+r-) r1rJrrlenrVr*rXr2rrEr6) rOrZr[rVrXranksrPprobclicksrXrXrYsample_clicks_for_querys      zPBM.sample_clicks_for_query) rCrDrErFrrYrIrJr1r6r7rarXrXrXrYrRus    rRrc Cst|}|j}tjdttj|dd|dddddf}ttj||f}||}t|t|jg}|||fS)zzRun length encoding using numpy, modified from: https://gist.github.com/nvictus/66627b580c13068589957d6ab0919e66 rr-NrT) equal_nan) r1asarrayr*r_ flatnonzeroisclosediffr8rJ)rrstartslengthsrindptrrXrXrYrlencodes 6 rkrrSrr? sample_ratec Cstjd}t|jd|}tjd|jdtjd}|||d|}||}||}||} t| } || }|| }| | } t ddd} | j ||| d| |} | S) zWe use XGBoost to generate the initial score instead of SVMRank for simplicity. Sample rate is set to 0.1 by default so that we can test with small datasets. r%rr+Nz rank:ndcghist) objective tree_method)r?) r1r2rrtraranger>rargsortrfitpredict) rSrr?rlrPr!indexX_trainr'r( sorted_idxltrscoresrXrXrYinit_rank_scores      ryfold scores_foldcCs|\}}}|jtjks Jt|}tj|jftjd}tj|jftjd}tdd}|D].} | |k} | | j d} || } t | ddd} | || <|| } | | | }||| <q-|j d|j dksnJ|j |j f|j d|j dksJ|j |j ft ||||||S)zSimulate clicks for one fold.r+r`)rSrNr) r,r1r6uniqueemptyr*r7rRr5rrqrar>)rzr{X_foldy_foldqid_foldqidsr[r`pbmqqid_mask query_scoresquery_positionrelevance_degrees query_clicksrXrXrYsimulate_one_folds$     $$rcv_datac stt|j|j\}}}tdgdd|Dttdks&Jt |}t |}t |}t |||fddt dj D}ggggggf\t j dD]6}t||||||f||} | j| j| j| j| j| jq^fddt j dD} t dD]}| |||ksJqtdkrtdddddd} d } | | fSfd d t tD\} } | | fS) z6Simulate click data using position biased model (PBM).rcSsg|]}|jdqS)rr).0vrXrXrY sz#simulate_clicks..r[cs$g|]}|d|qS)r-rXrr)rj scores_fullrXrYrs$r-csg|]}|qSrXrXr)s_lstrXrYrsrNc 3s:|]}t||||||VqdSr)r>r)X_lstc_lstp_lstq_lstry_lstrXrY &s ( z"simulate_clicks..)rziprrLr1rJcumsumr]rvstackrryrr*rr8rSrr?r@rArBallr>) rrSrr?X_fully_fullqid_fullrxrrzscores_check_1rrLrX)rrrjrrrrrrYsimulate_clickss:             *  rr`rBc Csjt|}||}||}||}||}t|\}}}td|jD]}||d} ||} | | ks8J| | ft|| | jdksJJ| | f|| | } | dks\J| | | jdkswJ| | j|t|| | ft| }|| | ||| | <|| | ||| | <|| | ||| | <|| | ||| | <q"||||f} | S)z,Sort data based on query index and position.r-r)r1rqrkrr*r|minmax) rSrr?r`rBrvrj_rbegend query_posr{rXrXrYsort_ltr_samples-s4  $   rDTypeDMatrixTdevicec Cstj}||jddddtjdd}t|dr&|jdddf}n|dddf}|}||||d }t j t d d t d |d |Wdn1sQwYt|ds|| dd}||ksmJ|jjjdusvJ|jjjdusJ|j|jd || dd}||jksJ|}||| } ||d|j| } | | ksJ|tj}||| } | | ksJ|dddd}t j t d d ||WddS1swYdSdS)zRun tests for base margin.rr`dr.2rilocN) base_marginz.*base_margin.*rrm)rorFTr-r)r1r2rrrprEr5hasattrrr/rrtrain_fnget_base_marginrTflags c_contiguous f_contiguousset_infoset_base_marginr*rF) rrrrPrSrrXygotbm_colbm_rowbm_f64rXrXrYrun_base_margin_info^sB $      "rras_densecsttjds$tjd}tjd|dd}|jddd}||fSttdt d tj ffd d }g}t d } t D] } | | || qFWd n1s\wYg} g} |D]} | \}}| || |qgt| ksJtj| dd}t| }||jd|jdfj}tj|dd}|jdksJ|jdksJ|jdksJ|r|}|jdksJ|jdksJtj||dk<||fS||fS)zMake sparse matrix. Parameters ---------- as_dense: Return the matrix as np.ndarray with missing values filled by NaN rr%r`csr)mrrrformatrt_idr#cstjd|}}|dkr||}n|}tj|d|d}tdf}t|jdD]'}|j|d|j|}|dkr]||dd|f |dfd7}q6||fS)Nr%r-r`)rrrrrrU) r1r2rrtocscrrrrjtoarray)rrP thread_sizen_features_tlocrSrrr*r"r! n_threadsrrXrY random_cscs( *z*make_sparse_regression..random_csc) max_workersN)rrr-r)rr1r2r3rrrmultiprocessing cpu_countrt csc_matrixrrr8submitresultr]hstackrcr5rrrrrj)r!r"rrrPrSrrfuturesexecutorr X_results y_resultsr%rarrrXrrYmake_sparse_regressionsP        r n_stringsseedcCs\d}t}tj|}t||kr*d|jttj |dd}| |t||kst|S)zGenerate n unique strings.rTr) setr1r2rr]rrrstring ascii_lettersadd)rrname_lenunique_stringsrP random_strrXrXrYunique_random_stringss    rrr`cpu)r cat_ratiorr cat_dtyper n_categoriesonehotrrrcCsHtd} tj|} tj|d} | } t|D]d}| jd|ddd}|dkrkt|tj rCt t ||}| j ||dd}nt d|}| jd||d}| j|dd | t|<| t|j|| t|<q| jd||d}| j||jd | t|<qtj|fd }| jD]}t| |j| jr|| |jj7}q|| |7}q|d7}|d krt|D]/}| jd|dt||d}tj| j||f<t| jj|r|t| jj|jj ksJq| j!d|ksJ|r| "| } |rt#| j}| $|| |} | d kr | d vsJddl%}ddl&}|'| } | |}| |fS)a/Generate categorical features for test. Parameters ---------- n_categories: Number of categories for categorical features. onehot: Should we apply one-hot encoding to the data? sparsity: The ratio of the amount of missing values over the number of all entries. cat_ratio: The ratio of features that are categorical. shuffle: Whether we should shuffle the columns. cat_dtype : The dtype for categorical features, might be string or numeric. Returns ------- X, y r$r-r.rTrr'rar+rrr)cudagpuN)(r/r0r1r2r3rrrL issubdtypestr_rJrrrpr4rorrrset_categoriesr,rrqrnrurrtrjrrrRr| categoriesr*r get_dummiesrrcudfr5 from_pandas)r!r"rrrrrrrrrOrProw_rngrUrrrrznumlabelcolrtrqrr5rXrXrYmake_categoricalsZ "           rcseZdZdZddddededeedeed ed eed dffd d Z de d efddZ dddZ d e eejejfeeeffddZZS)IteratorForTestzCIterator for testing streaming DMatrix. (external memory, quantile)FN)on_hostmin_cache_page_bytesrSrr9cacherrr#csBt|t|ks J||_||_||_d|_tj|||ddS)Nr) cache_prefixrr)r]rSrr9itsuperrY)rOrSrr9rrr __class__rXrYrYVs  zIteratorForTest.__init__ input_datacCs|jt|jkr dStjtdd||j|j|j|jdWdn1s*wY||j|j|j|j|jrI|j|jnddt |jd7_dS)NFzKeyword argumentr)r{rweightr-T) rr]rSr/r TypeErrorrr\r9gccollect)rOrrXrXrYnextkszIteratorForTest.nextcCs d|_dS)Nr)rrNrXrXrYreset|s zIteratorForTest.resetcCs:t|j}t|j}|jrtj|jdd}nd}|||fS)zReturn concatenated arrays.rrN)rrSrr9r1r)rOrSrr9rXrXrY as_arrayss   zIteratorForTest.as_arrays)r#N)rCrDrErFr r rrNrtrYrrrrrr1rrrGrr __classcell__rXrXrrYrSs0   r)F)r)^rFrrrrr concurrent.futuresr dataclassesrrrrrrrr r r r r rrrurllibrnumpyr1r/rI numpy.randomRNGscipyrcompatrcorerrrr{rrsklearnrrtrainingrrr$r DataFrameTr0rMemorymemoryrtrrrZr|rrrrrrrrrrGr/rNr=rJr6rQr>rKrRrkrrEryrrr7rrrr DTypeLikerrrXrXrXrYsV  <       ( ;A: 77 <   *7    $  #+        1/ U    `