o
    Ù­jÝ5  ã                   @   sÄ  d Z ddlZddlmZ ddlmZmZmZmZm	Z	m
Z
mZmZmZmZ ddlZddlZddlZddlZddlmZ ddlmZ ddlmZ dd	lmZmZ dd
lm Z m!Z! ddl"m#Z#m$Z$m%Z%m&Z& ddl'm(Z( ddl)m*Z*m+Z+ ddl,m-Z- e .d¡Z/eee0ef  Z1g d¢Z2G dd„ de$ƒZ3edej4dej5de
ej4e0f fdd„ƒZ6edej4ddde
ej4df fdd„ƒZ6dej4de	ej5 de
ej4e	e0 f fdd„Z6de0dB dej4dej5dej5de	ej5 de	ej5 de
ej4ej5ej5e	ej5 e	ej5 f fdd „Z7d!ee dee0ee f fd"d#„Z8d$e1dee0ee f fd%d&„Z9d'e1d(e	e# d)e	e d*e	e de
ee0ee f e	eeef  f f
d+d,„Z:d'e	e1 de;fd-d.„Z<d/e;dej=fd0d1„Z>dMd2d3„Z?dd4œd5e	e d)e	e d6e	e d7e@d8eAd'e	e1 d9eAd:e;d;e	eA d<e	e% d(e	e# d=e	e de&fd>d?„ZBd5e	e d)e	e d6e	e d7e@d8eAd:e;d'e	e1 d(e	e# d=e	e de%fd@dA„ZCdBe;d!ede%fdCdD„ZDdEeEdFeAdGeEdHeeA dIee0 dJeAd(e	e# de
e%ee
e%e0f  f fdKdL„ZFdS )Nz)Copyright 2019-2025, XGBoost contributorsé    N)ÚSequence)
ÚAnyÚCallableÚDictÚListÚOptionalÚTupleÚTypeVarÚUnionÚcastÚoverload)Ú	dataframeé   )Ú
collective)Ú
Categories)ÚFeatureNamesÚFeatureTypes)ÚconcatÚimport_cupy)ÚBoosterÚDataIterÚDMatrixÚQuantileDMatrix)Ú
is_on_cuda)Úget_model_categoriesÚpick_ref_categories)Ú	_RefErrorz[xgboost.dask])ÚlabelÚweightÚbase_marginÚqidÚlabel_lower_boundÚlabel_upper_boundc                       s¢   e Zd ZdZ			ddee dee deee	e
f  dee deee  ddf‡ fd	d
„Zdedee fdd„Zdefdd„Zddd„Zdedefdd„Z‡  ZS )ÚDaskPartitionIterz.A data iterator for the `DaskQuantileDMatrix`.NÚdataÚfeature_namesÚfeature_typesÚfeature_weightsÚkwargsÚreturnc                    s~   t td ƒf}|| _tD ]}t| || |d ¡ƒ tt| |ƒ|ƒs!J ‚q|| _|| _	|| _
t| jt ƒs3J ‚d| _tƒ jdd� d S )Nr   T)Úrelease_data)r   ÚtypeÚ_dataÚmetaÚsetattrÚgetÚ
isinstanceÚgetattrÚ_feature_namesÚ_feature_typesÚ_feature_weightsÚ_iterÚsuperÚ__init__)Úselfr$   r%   r&   r'   r(   ÚtypesÚk©Ú	__class__© úN/var/www/html/CropPilot/venv/lib/python3.10/site-packages/xgboost/dask/data.pyr7   4   s   zDaskPartitionIter.__init__Úattrc                 C   s"   t | |ƒd urt | |ƒ| j S d S ©N)r1   r5   )r8   r?   r=   r=   r>   Ú_getM   s   zDaskPartitionIter._getc                 C   s   | j | j S )z5Utility function for obtaining current batch of data.)r,   r5   ©r8   r=   r=   r>   r$   R   s   zDaskPartitionIter.datac                 C   s
   d| _ dS )zReset the iteratorr   N)r5   rB   r=   r=   r>   ÚresetV   s   
zDaskPartitionIter.resetÚ
input_datac                    s\   ˆ j tˆ jƒkr
dS ‡ fdd„tD ƒ}|dˆ  ¡ dˆ jˆ jˆ jdœ|¤Ž ˆ  j d7  _ dS )	zYield next batch of dataFc                    s   i | ]}|ˆ   |¡“qS r=   )rA   )Ú.0r:   rB   r=   r>   Ú
<dictcomp>`   ó    z*DaskPartitionIter.next.<locals>.<dictcomp>N)r$   Úgroupr%   r&   r'   é   Tr=   )r5   Úlenr,   r-   r$   r2   r3   r4   )r8   rD   r(   r=   rB   r>   ÚnextZ   s   ûúzDaskPartitionIter.next)NNN©r)   N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   r   r
   r   r   r7   ÚstrrA   r$   rC   r   ÚboolrK   Ú__classcell__r=   r=   r;   r>   r#   1   s,    ûþýüû
úù
r#   ÚdfÚcolr)   c                 C   ó   d S r@   r=   ©rT   rU   r=   r=   r>   Ú_add_columnm   ó   rX   c                 C   rV   r@   r=   rW   r=   r=   r>   rX   q   rY   c                 C   sl   |d u r| |fS d}|j › d|› �}|| jv r(|d7 }|j › d|› �}|| jv s| jdi ||i¤Ž} | |fS )Nr   Ú_rI   r=   )ÚnameÚcolumnsÚassign)rT   rU   ÚtrailsÚuidr=   r=   r>   rX   u   s   

þÚdevicer    ÚyÚsample_weightr   c                    sd  t ˆ |ƒ\‰ }t ˆ |ƒ\‰ }t ˆ |ƒ\‰ }t ˆ |ƒ\‰ }	| du s$| dkr&dnd}
tj d|
i¡�J ˆ  ¡ ‰ ˆ |  d¡j ¡ jjˆ |< ˆ j	|d�‰ ˆ  
|¡|  ¡ }|j ¡ j ¡ }t|ƒ}t||d d	 g ƒ}ˆ j|d
|d� ¡ ‰ W d  ƒ n1 s}w   Y  ˆ | }ˆ | }‡ fdd„||	fD ƒ\}}dd„ ||||	fD ƒ}ˆ j|d	d� ¡ ‰ ˆ ||||fS )zÏA function to prevent query group from being scattered to different
    workers. Please see the tutorial in the document for the implication for not having
    partition boundary based on query groups.

    NÚcpuÚp2pÚtaskszdataframe.shuffle.methodÚcategory)ÚbyéÿÿÿÿrI   F)ÚdropÚ	divisionsc                 3   s,   � | ]}|d urt tjˆ | ƒnd V  qd S r@   )r   ÚddÚSeries©rE   r_   ©rT   r=   r>   Ú	<genexpr>¯   s   € 
ÿz!no_group_split.<locals>.<genexpr>c                 S   s   g | ]}|d ur|‘qS r@   r=   rm   r=   r=   r>   Ú
<listcomp>³   rG   z"no_group_split.<locals>.<listcomp>)Úaxis)rX   ÚdaskÚconfigÚsetÚpersistÚastypeÚcatÚas_knownÚcodesÚsort_valuesÚgroupbyÚcountÚindexÚcomputeÚvaluesÚtolistÚsortedÚtupleÚ	set_indexri   )r`   rT   r    ra   rb   r   Úqid_uidÚy_uidÚw_uidÚbm_uidÚshuffleÚcntÚdivÚuidsr=   rn   r>   Úno_group_split…   s:   ýüõ

ÿrŒ   r(   c            
         sp  ˆ  d¡}|dusJ ‚t|ƒ}t|d ƒrddlm‰  nddlm‰  dtdttt	f f‡fdd„‰dtdt
jf‡ ‡fd	d
„‰‡fdd„t|ƒD ƒ}t|ƒ}|jjrSˆS t dt ¡ |¡ t|ƒ}t|ƒrntƒ }| |j¡}nt |j¡}|j|dd…f }t|dƒrŒ|j|dd…f }n||dd…f }ˆ d|gi¡ t|jƒD ]\}}	|	ˆv s«J ‚ˆ |	||	 gi¡ q¡ˆS )z>Sort worker-local data by query ID for learning to rank tasks.r$   Nr   )Ú	DataFrameÚir)   c                    sL   dt tt  dt t f‡fdd„‰ ‡ ‡fdd„tD ƒ}dd„ | ¡ D ƒ}|S )zDReturn a dictionary containing all the meta info and all partitions.r?   r)   c                    s   | d ur| ˆ  S d S r@   r=   )r?   )rŽ   r=   r>   rA   Æ   s   z0sort_data_by_qid.<locals>.get_dict.<locals>._getc                    s   i | ]}|ˆ ˆ  |d ¡ƒ“qS r@   )r/   )rE   r[   )rA   r(   r=   r>   rF   Ë   ó    z6sort_data_by_qid.<locals>.get_dict.<locals>.<dictcomp>c                 S   s   i | ]\}}|d ur||“qS r@   r=   )rE   r:   Úvr=   r=   r>   rF   Í   r�   )r   r   r   Úlistr-   Úitems)rŽ   Údata_optr$   )r(   )rA   rŽ   r>   Úget_dictÃ   s   "z"sort_data_by_qid.<locals>.get_dictc                    s   ˆ| ƒ}ˆ |ƒS r@   r=   )rŽ   r$   )r�   r”   r=   r>   Úmap_fnÐ   s   z sort_data_by_qid.<locals>.map_fnc                    s   g | ]}ˆ |ƒ‘qS r=   r=   )rE   rŽ   )r•   r=   r>   rp   Ô   s    z$sort_data_by_qid.<locals>.<listcomp>a  [r%d]: Sorting data with %d partitions for ranking. This is a costly operation and will increase the memory usage significantly. To avoid this warning, sort the data based on qid before passing it into XGBoost. Alternatively, you can use set the `allow_group_split` to False.Úiloc)r/   rJ   r   Úcudfr�   ÚpandasÚintr   rQ   r‘   ÚpdÚranger   r    Úis_monotonic_increasingÚLOGGERÚwarningÚcollÚget_rankr   ÚargsortÚnpr–   ÚhasattrÚupdateÚ	enumerater\   )
r(   Ú
data_partsÚn_partsÚ
meta_partsÚdfqÚdfxÚcpÚ
sorted_idxrŽ   Úcr=   )r�   r”   r(   r•   r>   Úsort_data_by_qid¸   s>   
ú
r®   Úlist_of_partsc                    s„   t ˆ tƒsJ ‚i ‰dtdtddf‡ ‡fdd„}tˆ ƒD ]\}}||dƒ tD ]}|||ƒ q&qˆ dd¡}|dur@td	i ˆ¤Ž‰ˆS )
z8Convert list of dictionaries into a dictionary of lists.rŽ   r[   r)   Nc                    sL   |ˆ |  v rˆ |  | }nd }|d ur$|ˆvrg ˆ|< ˆ|   |¡ d S d S r@   )Úappend)rŽ   r[   Úpart©r¯   Úresultr=   r>   r°     s   ýz!_get_worker_parts.<locals>.appendr$   r    r=   )r0   r‘   r™   rQ   r¥   r-   r/   r®   )r¯   r°   rŽ   rZ   r:   r    r=   r²   r>   Ú_get_worker_parts   s   

ÿr´   ÚpartsÚmodelr&   Úxy_catsc                 C   s8   t | ƒ}|d d }t|||ƒ\}}t|||ƒ}||fS )Nr$   r   )r´   r   r   )rµ   r¶   r&   r·   Úunzipped_dictÚXrZ   Ú
model_catsr=   r=   r>   Ú_extract_data  s
   r»   c                 C   sJ   | d urt | d  d¡ƒ}nd}tt tj|gtjd�tjj	¡d ƒ}|S )Nr   r$   F)Údtype)
r   r/   rR   rŸ   Ú	allreducer¢   ÚarrayÚint32ÚOpÚMAX)rµ   Úis_cudar=   r=   r>   Ú_get_is_cuda'  s
   &rÃ   rÂ   c                 C   s&   | rt ƒ }| d¡}|S t d¡}|S )N)r   r   )r   Úemptyr¢   )rÂ   r«   rÄ   r=   r=   r>   Ú_make_empty1  s   

ÿrÅ   c                  C   s   t  ¡ } t d| j¡ d S )NzWorker %s has an empty DMatrix.)ÚdistributedÚ
get_workerr�   rž   Úaddress)Úworkerr=   r=   r>   Ú_warn_empty:  s   rÊ   )Úrefr%   r'   ÚmissingÚnthreadÚmax_binÚenable_categoricalÚmax_quantile_batchesrË   ÚXy_catsc              	   C   sn   t |ƒ}|d u rtƒ  tt|ƒ| |||	||d�S t||
||ƒ\}}ttdi |¤|| |dœ¤Ž||||	||d�S )N)r%   r&   rÎ   rË   rÏ   rÐ   )r&   r%   r'   )rÌ   rÍ   rÎ   rË   rÏ   rÐ   r=   )rÃ   rÊ   r   rÅ   r»   r#   )r%   r&   r'   rÌ   rÍ   rµ   rÎ   rÏ   rÐ   rË   r¶   rÑ   rÂ   r¸   rº   r=   r=   r>   Ú_create_quantile_dmatrix?  s8   ù
ÿüôrÒ   c        	      
   C   s¨   t |ƒ}	|du rtƒ  tt|	ƒ| ||d�S tdƒ}
dtt|
  dt|
 fdd„}t||||ƒ\}}i }| ¡ D ]\}}||ƒ}|||< q7td	i |¤|| ||||dœ¤ŽS )
zdGet data that local to worker from DaskDMatrix.

    Returns
    -------
    A DMatrix object.

    N)r%   r&   rÏ   ÚTr$   r)   c                 S   s   t dd„ | D ƒƒrd S t| ƒS )Nc                 s   s   � | ]}|d u V  qd S r@   r=   )rE   r±   r=   r=   r>   ro   �  s   € z:_create_dmatrix.<locals>.concat_or_none.<locals>.<genexpr>)Úanyr   )r$   r=   r=   r>   Úconcat_or_noneŒ  s   z'_create_dmatrix.<locals>.concat_or_none)rÌ   r%   r&   rÍ   rÏ   r'   r=   )	rÃ   rÊ   r   rÅ   r	   r   r   r»   r’   )r%   r&   r'   rÌ   rÍ   rÏ   rµ   r¶   rÑ   rÂ   rÓ   rÕ   r¸   rº   Úconcated_dictÚkeyÚvaluer�   r=   r=   r>   Ú_create_dmatrixm  s4   ü
ÿ
ùrÙ   Úis_quantilec                 K   s    | r	t di |¤ŽS tdi |¤ŽS )Nr=   )rÒ   rÙ   )rÚ   r(   r=   r=   r>   Ú_dmatrix_from_list_of_parts£  s   rÛ   Ú	train_refÚtrain_idÚrefsÚevals_idÚ
evals_nameÚ	n_threadsc             	   G   sÔ   t di | ¤||d dœ¤Ž}g }| ¡ }	t|ƒD ]N\}
}||
 |kr+| |||
 f¡ q| dd ¡d urO|d |kr=ttƒ‚|d= t di |¤|||	|dœ¤Ž}nt di |¤||	|dœ¤Ž}| |||
 f¡ q||fS )N)rÍ   r¶   rÑ   rË   )rÍ   rË   rÑ   r¶   )rÍ   rÑ   r¶   r=   )rÛ   Úget_categoriesr¥   r°   r/   Ú
ValueErrorr   )rÜ   rÝ   rß   rà   rá   r¶   rÞ   ÚXyÚevalsrÑ   rŽ   rË   Úeval_xyr=   r=   r>   Ú_get_dmatrices©  s6   
ÿ
ÿÿÿÿ
ÿrç   rL   )GrP   ÚloggingÚcollections.abcr   Útypingr   r   r   r   r   r   r	   r
   r   r   rr   rÆ   Únumpyr¢   r˜   rš   r   rk   Ú r   rŸ   Ú_data_utilsr   Ú_typingr   r   Úcompatr   r   Úcorer   r   r   r   r$   r   Úsklearnr   r   Útrainingr   Ú	getLoggerr�   rQ   Ú
_DataPartsr-   r#   r�   rl   rX   rŒ   r®   r´   r»   rR   rÃ   ÚndarrayrÅ   rÊ   Úfloatr™   rÒ   rÙ   rÛ   Údictrç   r=   r=   r=   r>   Ú<module>   s  0

<&$ÿÿ
þÿþýüûúÿ
ù"3Hÿþýü"
û

	õþýüûúùø	÷
öõôó
ò.þýüûúùø	÷
ö
õ6ÿþýüûúùø