o
    Ú­j=1  ã                   @   st  d Z ddlmZmZmZmZmZmZ ddlZ	ddl
ZddlmZ ddlmZ ddlmZmZ ddlmZ ddlmZ ddlZddlmZ dd	lmZ dd
l m!Z! ddl"mZ# ddl$m%Z% ddlm&Z& ddl'm(Z( ddl)m*Z* ddl)m+Z, ddl-m.Z. ddl/m0Z0m1Z1 de2ded deddfdd„Z3de2ded deddfdd„Z4de2ded deddfdd„Z5dede2ded d e6ddf
d!d"„Z7d#e6d e6de2d$e8d%e9ddfd&d'„Z:ded e6defd(d)„Z;dedee2 fd*d+„Z<ded,e6d-e6d.e6d/e6de2deej=ej>ej>f fd0d1„Z?dede2ddfd2d3„Z@d4e	jAd5œded,e6d-e6d6e6d7e9d8e	jjBdeej=ej>f fd9d:„Z+dede0ddfd;d<„ZCdS )=z0Tests for dask shared by different test modules.é    )ÚAnyÚListÚLiteralÚTupleÚTypeÚcastN)Úarray)Ú	dataframe)ÚClientÚ
get_worker)Úparse)Úmake_classification)Úconcat)Úget_basescoreé   )Údask)ÚEvalsLog)Ú_get_rabit_args)Ú_DASK_VERSIONé   )Úmake_batches)Úmake_categorical)Úmake_recoded)ÚDeviceÚassert_allcloseÚtree_methodÚdevice)ÚcpuÚcudaÚclientÚreturnc                 C   sš   t dddd�\}}tjdd| |d�}| ||¡ t|ƒ}t |¡jdd�}t |¡jd	d�}tj	dd| |d�}	||	_
|	 ||¡ t|	ƒ}
tj ||
¡ d
S )z%Test init estimation for classsifier.é    é    éÊ  ©Ú	n_samplesÚ
n_featuresÚrandom_stater   ©Ún_estimatorsÚ	max_depthr   r   ©r"   N©Úchunks©r"   N)r   ÚxgbÚXGBClassifierÚfitr   ÚdaÚ
from_arrayÚrechunkÚdxgbÚDaskXGBClassifierr   ÚnpÚtestingr   )r   r   r   ÚXÚyÚclfÚ
base_scoreÚdxÚdyÚdclfÚdbase_score© rA   úQ/var/www/html/CropPilot/venv/lib/python3.10/site-packages/xgboost/testing/dask.pyÚcheck_init_estimation_clf   s$   ÿürC   c                 C   s¦   ddl m} |dddd�\}}tjdd| |d�}| ||¡ t|ƒ}t |¡jd	d
�}t |¡jdd
�}	t	j
dd| |d�}
||
_|
 ||	¡ t|
ƒ}tj ||¡ dS )z#Test init estimation for regressor.r   )Úmake_regressionr!   r"   r#   r$   r   r(   r+   r,   r.   N)Úsklearn.datasetsrD   r/   ÚXGBRegressorr1   r   r2   r3   r4   r5   ÚDaskXGBRegressorr   r7   r8   r   )r   r   r   rD   r9   r:   Úregr<   r=   r>   Údregr@   rA   rA   rB   Úcheck_init_estimation_reg5   s    ÿÿrJ   c                 C   s   t | ||ƒ t| ||ƒ dS )zTest init estimation.N)rJ   rC   )r   r   r   rA   rA   rB   Úcheck_init_estimationN   s   rK   Ú	n_workersc                 C   sÄ   |dksJ ‚|   ¡ �N tj||d�}t tdƒtdddƒdœ¡}t g dgd ¢dgd ¢¡}tj|j	d	d
d…df< | j
|d� | tj||d�tj||d�¡ W d	  ƒ d	S 1 s[w   Y  d	S )z0Issue #9271, not every worker has missing value.r   )r   r   i'  r   éÿÿÿÿ)ÚaÚbiˆ  r   Ni¸  iè  rN   )rL   ©Únpartitions)Ú
as_currentr5   r6   ÚpdÚ	DataFrameÚrangeÚSeriesr7   ÚnanÚlocÚwait_for_workersr1   ÚddÚfrom_pandas)r   r   r   rL   r;   r9   r:   rA   rA   rB   Úcheck_uneven_nanV   s   
þ"÷r\   Ú	worker_idÚ	comm_argsÚis_qdmc              	   C   s¼  d}d}d}|dk}t ƒ jj}	tjjdddi|¤Ž�K tjt||||| d�ddiŽ}
|r4tj	|
|	d	�}ntj
|
|	d	�}i }tjd
|	|dœ||dfgd|d� t ttt |d d ƒ¡s^J ‚W d  ƒ n1 shw   Y  g g g }}}t|ƒD ]}t|||||d�\}}}| |¡ | |¡ | |¡ qyt|ƒ}t|ƒ}t|ƒ}|r¯tj||||	d�}n	tj
||||	d�}i }tjd
|	|dœ||dfgd|d� tjj|d d |d d dd� dS )z-Basic checks for distributed external memory.r"   é   é   r   Údmlc_communicatorÚrabit)Úuse_cupyr'   Úcache)ÚnthreadÚhist)r   rf   r   ÚTrain)ÚevalsÚnum_boost_roundÚevals_resultÚrmseN)Úweightrf   g-Cëâ6?©ÚrtolrA   )r   ÚstateÚnthreadsr/   Ú
collectiveÚCommunicatorContextÚtmÚIteratorForTestr   ÚExtMemQuantileDMatrixÚDMatrixÚtrainÚnon_increasingr   r   ÚfloatrU   Úextendr   ÚQuantileDMatrixr7   r8   r   )r]   rL   r   r^   r_   Ún_samples_per_batchr&   Ú	n_batchesrd   Ú	n_threadsÚitÚXyÚresultsÚlxÚlyÚlwÚiÚxr:   Úwr9   ÚyconcatÚwconcatÚresults_localrA   rA   rB   Úcheck_external_memoryk   sv   
ûÿø

û"éû


û
ÿrŒ   c                 C   s   |   t| |¡S )z6Get RABIT collective communicator arguments for tests.)Úsyncr   )r   rL   rA   rA   rB   Úget_rabit_args´   s   rŽ   c                 C   s:   t ƒ tdƒkrddini }| jdi |¤Žd }t| ¡ ƒS )zGet workers from a dask client.z2025.4.0rL   rM   ÚworkersNrA   )r   Úparse_versionÚscheduler_infoÚlistÚkeys)r   Úkwargsr�   rA   rA   rB   Úget_client_workers¹   s   r•   r%   r&   Ún_query_groupsÚmax_relc             	      s"  t | ƒ}|t|ƒ }|dkrddlm‰  nddlm‰  dtdtdtjf‡ ‡‡‡fdd„}g }	d}
td||ƒD ]}| j|||||
t|ƒ  gd	�}|	 	|¡ |
d
7 }
q6||t|ƒ  }|dkro| j|||t|ƒ d�}|	 	|¡ |d
dƒ}t
j|	|d�}t|t
jƒsƒJ ‚|jddgd
d�|j|jfS )z'Synthetic dataset for learning to rank.r   r   )rT   ÚnÚseedr    c                    sb   t j |¡}t| ˆˆdˆd�\}}|j| fdˆd�}ˆ |dd„ tˆƒD ƒd�}||d< ||d< |S )	Nr   )Ún_informativeÚn_redundantÚ	n_classes)ÚsizeÚlowÚhighc                 S   s   g | ]}d |› �‘qS )ÚfrA   )Ú.0r†   rA   rA   rB   Ú
<listcomp>Ø   s    z*make_ltr.<locals>.make.<locals>.<listcomp>)ÚcolumnsÚqidr:   )r7   ÚrandomÚdefault_rngr   ÚintegersrU   )r˜   r™   Úrngr9   r:   r¤   Údf©ÚDFr—   r&   r–   rA   rB   ÚmakeÒ   s   

ÿzmake_ltr.<locals>.make)r˜   r™   r�   r   )r˜   r™   ©Úmetar¤   r:   )Úaxis)r•   ÚlenÚpandasrT   ÚcudfÚintrS   rU   ÚsubmitÚappendrZ   Úfrom_delayedÚ
isinstanceÚdropr:   r¤   )r   r%   r&   r–   r—   r   r�   Ún_samples_per_workerr¬   Úfuturesr†   ÚkÚfutÚlastr®   r©   rA   rª   rB   Úmake_ltrÀ   s,   
"ÿ



r¾   c           
      C   sÖ   t | dddd|d�\}}}t | dddd|d�\}}}tjdd|d	d
�}|j|||||f||fg||gdd� |jdks=J ‚|jd |jksGJ ‚| ¡ d d }	tj|	dd… dd�s^J |	ƒ‚t	j
j|	d ddd� dS )z)Test for the allow_group_split parameter.i   é€   r`   é   )r–   r—   r   i   Fé$   zrank:pairwise)Úallow_group_splitr)   r   Ú	objectiveT)r¤   Úeval_setÚeval_qidÚverboser   Úvalidation_0zndcg@32Nra   g{®Gáz„?)Ú	tolerancerM   g      ð?rn   )r¾   r5   ÚDaskXGBRankerr1   Ún_features_in_Úshaperk   rt   Únon_decreasingr7   r8   r   )
r   r   ÚX_trÚq_trÚy_trÚX_vaÚq_vaÚy_vaÚltrÚndcgrA   rA   rB   Úcheck_no_group_splitñ   s2   ÿÿüú	rÕ   F)ÚonehotÚ	cat_dtypeÚn_categoriesrÖ   r×   c             
   C   sè   t | ƒ}t|ƒ}g }dtdtjfdd„}	|	d||d|d�}
t|ƒD ]-\}}t|| ||||   ƒ}|dkr<|dkr<d}| j|	||||d|gd	�}| |¡ q!t	tjtj
||
d
�ƒ}|d }||j dg¡ }|rpt |¡|fS ||fS )z&Synthesize categorical data with dask.r”   r    c                  [   s   t di | ¤Ž\}}||d< |S )NÚlabelrA   )Úmake_cat_local)r”   r9   r:   rA   rA   rB   Úpack  s   zmake_categorical.<locals>.packr   F)r%   r&   rØ   rÖ   r×   r   )r%   r&   rØ   r×   rÖ   r�   r­   rÙ   )r•   r°   r   rZ   rT   Ú	enumerateÚminr´   rµ   r   r¶   r£   Ú
differenceÚget_dummies)r   r%   r&   rØ   rÖ   r×   r�   rL   ÚdfsrÛ   r®   r†   ÚworkerÚl_n_samplesÚfuturer©   r:   r9   rA   rA   rB   r     sB   
ûÿù	r   c                    s`   dt tj dtdtdtjfdd„‰dt tj ddf‡ ‡‡fdd	„}tjtjfD ]}||ƒ q'dS )
z+Run re-coding test with the Dask interface.ÚDMatrixTÚargsr”   r    c                 _   sB   | t ju r| dd ¡}| |d|i|¤ŽS | dd ¡ | |i |¤ŽS )NÚref)r5   ÚDaskQuantileDMatrixÚpop)rä   rå   r”   ræ   rA   rA   rB   Úcreate_dmatrixJ  s
   
z"run_recode.<locals>.create_dmatrixNc                    sÌ  t ˆdd�\}}}}}tˆ ƒ}tj|dd�j|d�tj|dd�j|d�tj||jd d fd�j|d�}}}ˆ| ˆ ||dd	�}	ˆ| ˆ ||d|	d
�}
tj	ˆ dˆi|	|
dfgd�}ˆ| ˆ ||dd	�}	ˆ| ˆ ||d|	d
�}
tj	ˆ dˆi|	|
dfg|d d�}ˆ| ˆ ||dd	�}	ˆ| ˆ ||d|	d
�}
tj	ˆ dˆi|	|
dfg|d d�}t
j |d d d |d d d ¡ t ˆ ||¡ ¡ }t ˆ ||¡ ¡ }tˆ||ƒ t ˆ ||	¡ ¡ }t ˆ ||
¡ ¡ }tˆ||ƒ d S )Né`   )r&   é   rP   )r�   r   r,   T)Úenable_categorical)rì   ræ   r   ÚValid)ri   Úbooster)ri   Ú	xgb_modelÚhistoryrl   )r   r•   rZ   r[   Úpersistr2   r3   rË   r5   rx   r7   r8   r   Úinplace_predictÚcomputeÚpredict)rä   ÚencÚreencr:   Ú_ÚtoÚdencÚdreencr>   r�   ÚXy_validr‚   Ú	results_1Ú	results_2Úpredt_0Úpredt_1©r   ré   r   rA   rB   ÚrunT  sX    
ýÿÿÿû	ÿûÿzrun_recode.<locals>.run)r   r5   ÚDaskDMatrixr   rç   )r   r   r  rä   rA   r   rB   Ú
run_recodeG  s   ÿÿÿ
þ 
8
ÿr  )DÚ__doc__Útypingr   r   r   r   r   r   Únumpyr7   r±   rS   r   r   r2   r	   rZ   Údistributedr
   r   Úpackaging.versionr   r�   rE   r   Úxgboostr/   Úxgboost.testingr8   rt   Úxgboost.compatr   Úxgboost.testing.updaterr   Ú r5   Ú_typingr   r   Ú
dask.utilsr   Údatar   r   rÚ   Úordinalr   Úutilsr   r   ÚstrrC   rJ   rK   r³   r\   ÚdictÚboolrŒ   rŽ   r•   rT   rV   r¾   rÕ   Úint64Ú	DTypeLiker  rA   rA   rA   rB   Ú<module>   sØ     ÿÿÿ
þÿÿÿ
þÿÿÿ
þÿÿÿÿ
þÿþýüû
úIÿþýûúù
ø1%ùÿþýüúù
ø7