o
    tBhB                     @   s  d dl mZmZmZ d dlZd dlZd dlmZ d dlmZ d dl	m
Z
 ddlmZmZmZ ddlmZmZ dd	lmZ ed
Zd dlmZmZmZ d dlm	Z	 d dlmZmZ dd ZdPddZdd Zdd Z dd Z!dd Z"dd Z#ej$%dddgd d! Z&d"d# Z'G d$d% d%e(Z)dQd&d'Z*d(d) Z+d*d+ Z,ej$%d,e-e.dd-d. Z/d/d0 Z0d1d2 Z1d3d4 Z2d5d6 Z3d7d8 Z4d9d: Z5d;d< Z6d=d> Z7d?d@ Z8dAdB Z9dCdD Z:ej$%dEdFdGgej$j;ej<dHkoej<dIkdJdKdLdM Z=dNdO Z>dS )R    )print_functiondivisionabsolute_importNrandom)uuid4)sleep   )Paralleldelayedparallel_backend)ThreadingBackendAutoBatchingMixin)DaskDistributedBackenddistributed)ClientLocalCluster
get_client)time)clusterincc                  O      d S N )argskwargsr   r   l/var/www/html/riverr-enterprise-integrations-main/venv/lib/python3.10/site-packages/joblib/test/test_dask.pynoop   s   r   皙?c                 C   s   t | | r
tdd S )Nzcondition evaluated to True)r   
ValueError)	conditiondurationr   r   r   slow_raise_value_error   s   r"   c                    sF   | dd }i }| D ]\}}t fddt|D ||< q|S )Nc                 S   s   | j S r   )log)dask_workerr   r   r   <lambda>   s    zcount_events.<locals>.<lambda>c                    s   g | ]
}|d   kr|qS    r   ).0event
event_namer   r   
<listcomp>!   s    z count_events.<locals>.<listcomp>)runitemslenlist)r+   clientworker_eventsevent_countsweventsr   r*   r   count_events   s
    r6   c                 C   sF  t  \}\}}t|d | dq}td]\}}t dd tdD }|dd tdD ks1J tt t d	d tdD  W d    n1 sMw   Y  t d
d tdD }|dd tdD kskJ W d    n1 suw   Y  W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )Naddressloopdaskc                 s       | ]	}t t|V  qd S r   r   r   r(   ir   r   r   	<genexpr>*       ztest_simple.<locals>.<genexpr>
   c                 S      g | ]}t |qS r   r   r=   r   r   r   r,   +       ztest_simple.<locals>.<listcomp>c                 s   s     | ]}t t|d kV  qdS )   N)r   r"   r=   r   r   r   r?   .   s    c                 s   r;   r   r<   r=   r   r   r   r?   1   r@   c                 S   rB   r   rC   r=   r   r   r   r,   2   rD   )r   r   r   r
   rangepytestraisesr   r9   sabr1   ba_seqr   r   r   test_simple&   s$   

"rP   c           	      C   s.  t jtju sJ t \}\}}t|d | d]}tdI\}}t 4}|j}t|t s.J |j	|u s5J |j
dks<J |dd ttdD  |j
dksPJ W d    n1 sZw   Y  W d    n1 siw   Y  W d    n1 sxw   Y  W d    d S W d    d S 1 sw   Y  d S )	Nr7   r8   r:   r'   c                 s   s    | ]
}t d d  V  qdS )c                   S   r   r   r   r   r   r   r   r%   G   s    z?test_dask_backend_uses_autobatching.<locals>.<genexpr>.<lambda>Nr   r(   rN   r   r   r   r?   F   
    
z6test_dask_backend_uses_autobatching.<locals>.<genexpr>     @rA   )r   compute_batch_sizer   r   r   r   r
   _backend
isinstanceparallel_effective_batch_sizerF   int)	r9   rJ   rK   rL   r1   rM   rN   rX   backendr   r   r   #test_dask_backend_uses_autobatching5   s.   
"r\   c                   C   s   t  S r   r   r   r   r   r   random2M   s   r]   c           	   
   C   s   t  W\}\}}t|d | d2}td\}}t dd tdD \}}||ks,J W d    n1 s6w   Y  W d    n1 sEw   Y  W d    d S W d    d S 1 s]w   Y  d S )Nr7   r8   r:   c                 s       | ]}t t V  qd S r   )r   r]   r=   r   r   r   r?   U   s    z3test_dont_assume_function_purity.<locals>.<genexpr>r	   r   r   r   r
   rF   )	r9   rJ   rK   rL   r1   rM   rN   xyr   r   r    test_dont_assume_function_purityQ   s   "rb   mixedTFc              
   C   s6  ddl m} |sdd tdD }d}ndd tdD }d}t|||ks)J t e\}\}}t|d	 | d
@}td\}	}
tddd|}
W d    n1 sSw   Y  dd }|dd}|	|}t
dd |D srJ W d    n1 s|w   Y  W d    d S W d    d S 1 sw   Y  d S )Nr   )Batchc                 S   s   g | ]}t t|qS r   r<   r=   r   r   r   r,   ]       z&test_dask_funcname.<locals>.<listcomp>   batch_of_inc_4_callsc                 S   s,   g | ]}|d  rt t|nt t|qS )r	   )r   absr   r=   r   r   r   r,   `   s     mixed_batch_of_inc_4_callsr7   r8   r:   r	   all)
batch_sizepre_dispatchc                 S   s
   t | jS r   )r0   transition_log)dask_schedulerr   r   r   fl      
ztest_dask_funcname.<locals>.f42c                 s   s    | ]	}d |d v V  qdS )batch_of_incr   Nr   )r(   tupr   r   r   r?   p   r@   z%test_dask_funcname.<locals>.<genexpr>)joblib._daskrd   rF   reprr   r   r   r
   replacerun_on_schedulerrj   )r9   rc   rd   tasks
batch_reprrJ   rK   rL   r1   rM   rN   ro   r#   r   r   r   test_dask_funcnameY   s,   
"r{   c                    s  dd t dD }tdtd d fdd	tdd	d
}t|}ztd\}}t fdd|D }W d    n1 sEw   Y  |dd t dD ksWJ t	d|}t
| dksfJ tdd |D sqJ td\}}t  fdd|D }W d    n1 sw   Y  t	d|}t
| dksJ tdd |D sJ W |  |  d S |  |  w )Nc                 S      g | ]}g qS r   r   rR   r   r   r   r,   ~       z;test_no_undesired_distributed_cache_hit.<locals>.<listcomp>d   numpyg    .Ac                    s(   |d urj |  | t j | S r   )testingassert_array_equalappendr   hex)list_data)Xnpr   r   isolated_operation   s   zCtest_no_undesired_distributed_cache_hit.<locals>.isolated_operationr'   r	   )	n_workersthreads_per_workerr:   c                 3   s    | ]	}t  |V  qd S r   rQ   r(   r   )r   r   r   r?          
z:test_no_undesired_distributed_cache_hit.<locals>.<genexpr>c                 S   r|   r   r   rR   r   r   r   r,      r}   receive-from-scatterr   c                 S      g | ]}t |d kqS r&   r/   r(   rr   r   r   r,      re   c                 3   s     | ]}t | d V  qdS ))r   NrQ   r   )r   r   r   r   r?      s    
c                 S   r   r&   r   r   r   r   r   r,      re   r   )rF   rG   importorskiparangerZ   r   r   r   r
   r6   sumvaluesrj   close)r9   listsr   r1   rM   rN   rescountsr   )r   r   r   r   'test_no_undesired_distributed_cache_hits   s8   





r   c                   @   s(   e Zd Zdd Zdd ZeZdd ZdS )CountSerializedc                 C   s   || _ d| _d S )Nr   )r`   count)selfr`   r   r   r   __init__   s   
zCountSerialized.__init__c                 C   s   | j t|d| S )Nr`   )r`   getattr)r   otherr   r   r   __add__   s   zCountSerialized.__add__c                 C   s   |  j d7  _ t| jffS )Nr'   )r   r   r`   )r   r   r   r   
__reduce__   s   zCountSerialized.__reduce__N)__name__
__module____qualname__r   r   __radd__r   r   r   r   r   r      s
    r   c                 C   s   | | | | | S r   r   )rK   rL   cder   r   r   add5   s   r   c                 C   s  t d}t d}t d}t \}\}}t|d | d}td||gd<\}}	tt}
|
|||dd	d
|
|||d	dd
|
||||d	d
|
|||||d
g}dd |D }t |}W d    n1 sdw   Y  tt	 td| dd W d    n1 sw   Y  W d    n1 sw   Y  W d    n1 sw   Y  W d    n1 sw   Y  ||ksJ |j
dksJ |j
dksJ |j
dv sJ d S )Nr'   r	   rE   r7   r8   r:   )scatterrf      )r   r   c                 S   s    g | ]\}}}||i |qS r   r   )r(   funcr   r   r   r   r   r,      s    z'test_manual_scatter.<locals>.<listcomp>)r9   r   )rf      )r   r   r   r   r   r   r
   rG   rH   	TypeErrorr   )r9   r`   ra   zrJ   rK   rL   r1   rM   rN   ro   ry   expectedresultsr   r   r   test_manual_scatter   s>   r   c              
      s   t d}|jtd|jd |jtd|jd} gd |gd  }t [\}\}}t|d | d?}td\}}	t dd	 t	|D  W d    n1 sQw   Y  t
d
|}
|
|d  |
|d   dkskJ W d    n1 suw   Y  W d    n1 sw   Y  t j\}\}}t|d | dE}td\}}	t  fdd	tdD  W d    n1 sw   Y  t
d
|}
|
|d  dksJ |
|d  dksJ W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )Nr   rT   )dtyperE   r7   r8   r:   c                 s   s(    | ]\}}t t||||d V  qdS ))optNr   r   )r(   r>   r   r   r   r   r?      s    z$test_auto_scatter.<locals>.<genexpr>r   r	   c                 3   s&    | ]}t t d d |V  qd S )NrE   r   r=   data1r   r   r?      s   $ r   r   )rG   r   onesrZ   uint8r   r   r   r
   	enumerater6   rF   )loop_in_threadr   data2data_to_processrJ   rK   rL   r1   rM   rN   r   r   r   r   test_auto_scatter   s8   



"
"r   retry_noc              
      s   t dd d}fdd fddt U\}\}}t|d | d0}td	 d
t fddt|D }W d    n1 sJw   Y  W d    n1 sYw   Y  W d    d S W d    d S 1 sqw   Y  d S )Nr   rA   c                    s
     | S r   )r   )r`   r>   j)r   r   r   my_sum
  rp   z#test_nested_scatter.<locals>.my_sumc                    s\   t  }td t  fddtD }W d    t|S 1 s%w   Y  t|S )Nr:   c                 3   s(    | ]}t  |d  |V  qd S r   rQ   )r(   r   )arrayr>   r   r   r   r?     s    
zEtest_nested_scatter.<locals>.outer_function_joblib.<locals>.<genexpr>)r   r   r
   rF   r   )r   r>   r1   r   )NUM_INNER_TASKSr   )r   r>   r   outer_function_joblib  s   


z2test_nested_scatter.<locals>.outer_function_joblibr7   r8   r:   i'  c                 3   s&    | ]}t  |d  |V  qd S r   rQ   r=   )my_arrayr   r   r   r?     s    
z&test_nested_scatter.<locals>.<genexpr>)rG   r   r   r   r   r   r
   rF   )r9   r   NUM_OUTER_TASKSrJ   rK   rL   rN   r   )r   r   r   r   r   r   test_nested_scatter  s$   
	


"r   c           	   
      s  dd  t  \}\}}t|d | d=}td)\}}tdd fdd	td
D }|D ]}tt|dks:J q.W d    n1 sEw   Y  W d    n1 sTw   Y  t|d | d=}td)\}}tdd fdd	td
D }|D ]}tt|dksJ q{W d    n1 sw   Y  W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )Nc                  S   sH   t tdddd tdD } | t tdddd tdD O } | S )Nr	   n_jobsc                 s       | ]	}t tj V  qd S r   r   osgetpidrR   r   r   r   r?   "  r@   zOtest_nested_backend_context_manager.<locals>.get_nested_pids.<locals>.<genexpr>c                 s   r   r   r   rR   r   r   r   r?   #  r@   )setr
   rF   )pidsr   r   r   get_nested_pids!  s    $z<test_nested_backend_context_manager.<locals>.get_nested_pidsr7   r8   r:   r	   r   c                 3       | ]}t   V  qd S r   rQ   rR   r   r   r   r?   )  
    
z6test_nested_backend_context_manager.<locals>.<genexpr>rA   c                 3   r   r   rQ   rR   r   r   r   r?   3  r   )r   r   r   r
   rF   r/   r   )	r   rJ   rK   rL   r1   rM   rN   
pid_groups	pid_groupr   r   r   #test_nested_backend_context_manager   s6   
"r   c                    s8  dd   fddt  \}\}}t|d | d`}tdL\}}t $} |dks.J |jd	ks5J |fd
dtdD }W d    n1 sLw   Y  |D ]\}	}
|	dks]J |
d	kscJ qSW d    n1 snw   Y  W d    n1 s}w   Y  W d    d S W d    d S 1 sw   Y  d S )Nc                 S   s
   | j jjS r   )rV   	__class__r   pr   r   r   _backend_type?  rp   zJtest_nested_backend_context_manager_implicit_n_jobs.<locals>._backend_typec                     s8   t  }  | | jfW  d    S 1 sw   Y  d S r   )r
   r   r   )r   r   r   get_nested_implicit_n_jobsB  s   $zWtest_nested_backend_context_manager_implicit_n_jobs.<locals>.get_nested_implicit_n_jobsr7   r8   r:   r   c                 3   r   r   rQ   rR   )r   r   r   r?   L  r   zFtest_nested_backend_context_manager_implicit_n_jobs.<locals>.<genexpr>r	   )r   r   r   r
   r   rF   )r9   rJ   rK   rL   r1   rM   rN   r   all_nested_n_jobsbackend_typenested_n_jobsr   )r   r   r   3test_nested_backend_context_manager_implicit_n_jobs;  s,   
"r   c              	   C   sl   t t}td W d    n1 sw   Y  W d    n1 s$w   Y  dt|j v s4J d S )Nr:   zcreate a dask client)rG   rH   r   r   strvaluelower)r9   infor   r   r   test_errorsU  s   
r   c              
   C   s8  t  \}\}}t|d | dj}td%\}}tdddd tdD }t|d	 d	 d	 ts3J W d    n1 s=w   Y  td%\}}tddd
d tdD }t|d	 d	 d	 tsdJ W d    n1 snw   Y  W d    n1 s}w   Y  W d    d S W d    d S 1 sw   Y  d S )Nr7   r8   r:   r	   r   c                 s   s    | ]
}t td dV  qd S )Nnested_requirer   outerrR   r   r   r   r?   b  s    
z.test_correct_nested_backend.<locals>.<genexpr>r'   r   c                 s   s    | ]
}t td dV  qdS )	sharedmemr   Nr   rR   r   r   r   r?   h  rS   )r   r   r   r
   rF   rW   r   r   )r9   rJ   rK   rL   r1   rM   rN   resultr   r   r   test_correct_nested_backend]  s&   "r   c                    s"   t ddd fddtdD S )Nr	   threads)r   preferc                 3   s    | ]	}t t V  qd S r   )r   middlerR   r   r   r   r?   o  r   zouter.<locals>.<genexpr>r'   r
   rF   r   r   r   r   r   n  s   r   c                 C   s   t d| ddd tdD S )Nr	   )r   requirec                 s   r^   r   )r   innerrR   r   r   r   r?   u  s    

zmiddle.<locals>.<genexpr>r'   r   )r   r   r   r   r   t  s   r   c                   C   s   t  jS r   )r
   rV   r   r   r   r   r   z  s   r   c              	   C   s   t | ddd3 td tdddd td	D  W d    n1 s%w   Y  W d    d S W d    d S 1 s=w   Y  d S )
NFT)r9   	processesset_as_defaultr:   rf   r   c                 s   r;   r   )r   idr=   r   r   r   r?     r@   z0test_secede_with_no_processes.<locals>.<genexpr>r	   )r   r   r
   rF   r8   r   r   r   test_secede_with_no_processes~  s   
"r   c                 C   s   ddl m} | jS )Nr   )
get_worker)r   r   r7   )rN   r   r   r   r   _worker_address  s   r   c              
   C   s8  t  \}\}}t|d | dj}td|d d!\}}t dd tdD }||d gd ks3J W d    n1 s=w   Y  td|d d!\}}t dd tdD }||d gd ksdJ W d    n1 snw   Y  W d    n1 s}w   Y  W d    d S W d    d S 1 sw   Y  d S )	Nr7   r8   r:   )workersc                 s   r;   r   r   r   r=   r   r   r   r?     r   z-test_dask_backend_keywords.<locals>.<genexpr>rA   c                 s   r;   r   r   r=   r   r   r   r?     r   r_   rI   r   r   r   test_dask_backend_keywords  s&   

"r   c              	   C   s   t d| dH}td t dd tdD  W d    n1 s"w   Y  t }|jjjrAtd t |d k s<J |jjjs/|j	rFJ W d    d S 1 sQw   Y  d S )	NF)r   r9   r:   c                 s   r;   r   r<   r=   r   r   r   r?     r@   ztest_cleanup.<locals>.<genexpr>rA   g{Gz?r   )
r   r   r
   rF   r   r   	schedulerry   r   futures)r9   r1   startr   r   r   test_cleanup  s   


"r   cluster_strategyadaptivelate_scalingz2.1.1z1.28.0z?distributed bug - https://github.com/dask/distributed/pull/2841)reasonc                 C   s   t dddd}t|}| dkr|jddd n	| dkr |d z6td t d	d
 tdD  W d    n1 s<w   Y  W |  |  d S W |  |  d S |  |  w )Nr   Fr	   r   r   r   r  )minimummaximumr  r:   c                 s   r;   r   r<   r=   r   r   r   r?     r@   z(test_wait_for_workers.<locals>.<genexpr>rA   )r   r   adaptscaler   r
   rF   r   )r  r   r1   r   r   r   test_wait_for_workers  s"   


r
  c               	   C   sB  t dddd} t| }ztddd- d}tjt|d	 t d
d tdD  W d    n1 s3w   Y  W d    n1 sBw   Y  tddd- d}tjt|d	 t dd tdD  W d    n1 snw   Y  W d    n1 s}w   Y  W |	  | 	  d S W |	  | 	  d S |	  | 	  w )Nr   Fr	   r  r:   g?)wait_for_workers_timeoutz7DaskDistributedBackend has no worker after 0.1 seconds.)matchc                 s   r;   r   r<   r=   r   r   r   r?     r@   z0test_wait_for_workers_timeout.<locals>.<genexpr>rA   z+DaskDistributedBackend has no active workerc                 s   r;   r   r<   r=   r   r   r   r?     r@   )
r   r   r   rG   rH   TimeoutErrorr
   rF   RuntimeErrorr   )r   r1   msgr   r   r   test_wait_for_workers_timeout  s0   
r  )r   )r   r   )?
__future__r   r   r   r   rG   r   uuidr   r   r    r
   r   r   rX   r   r   _daskr   r   r   r   r   r   distributed.metricsdistributed.utils_testr   r   r   r"   r6   rP   r\   r]   rb   markparametrizer{   r   objectr   r   r   r   r0   rF   r   r   r   r   r   r   r   r   r   r   r   r   skipif__version__r
  r  r   r   r   r   <module>   s^    

	
9
%
