a
    !fF                     @   s"  d dl mZmZmZ d dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
mZ ddlmZmZmZ ddlmZmZ dd	lmZ ed
ZedZd dlmZmZmZ d dlm
Z
 d dlmZmZ dd ZdVddZ dd Z!dd Z"dd Z#dd Z$dd Z%ej&'ddd gd!d" Z(d#d$ Z)G d%d& d&e*Z+dWd'd(Z,d)d* Z-d+d, Z.ej&'d-e/e0dd.d/ Z1d0d1 Z2d2d3 Z3d4d5 Z4d6d7 Z5d8d9 Z6d:d; Z7d<d= Z8d>d? Z9d@dA Z:dBdC Z;dDdE Z<ej&'dFdGdHgej&j=ej>dIkoej>dJkdKdLdMdN Z?dOdP Z@ej&'dQdRdSgdTdU ZAdS )X    )print_functiondivisionabsolute_importNrandom)uuid4)sleep   )Paralleldelayedparallel_config)ThreadingBackendAutoBatchingMixin)DaskDistributedBackenddistributeddask)ClientLocalCluster
get_client)time)clusterincc                  O   s   d S N )argskwargsr   r   V/var/www/html/python-backend/venv/lib/python3.9/site-packages/joblib/test/test_dask.pynoop   s    r   皙?c                 C   s   t | | rtdd S )Nzcondition evaluated to True)r   
ValueError)	conditiondurationr   r   r   slow_raise_value_error   s    r"   c                    sF   | dd }i }| D ]&\}}t fddt|D ||< q|S )Nc                 S   s   | j S r   )log)Zdask_workerr   r   r   <lambda>"       zcount_events.<locals>.<lambda>c                    s   g | ]}|d   kr|qS    r   ).0event
event_namer   r   
<listcomp>%   s   z count_events.<locals>.<listcomp>)runitemslenlist)r+   clientZworker_eventsZevent_countsweventsr   r*   r   count_events!   s
     r4   c                 C   s8  t  \}\}}t|d | d}tdd t dd tdD }|dd	 tdD ksbJ tt( t d
d tdD  W d    n1 s0    Y  t dd tdD }|dd	 tdD ksJ W d    n1 s0    Y  W d    n1 s
0    Y  W d    n1 s*0    Y  d S )Naddressloopr   backendc                 s   s   | ]}t t|V  qd S r   r   r   r(   ir   r   r   	<genexpr>.   r%   ztest_simple.<locals>.<genexpr>
   c                 S   s   g | ]}t |qS r   r   r;   r   r   r   r,   /   r%   ztest_simple.<locals>.<listcomp>c                 s   s   | ]}t t|d kV  qdS )   N)r   r"   r;   r   r   r   r=   2   s   c                 s   s   | ]}t t|V  qd S r   r:   r;   r   r   r   r=   5   r%   c                 S   s   g | ]}t |qS r   r?   r;   r   r   r   r,   6   r%   )r   r   r   r
   rangepytestraisesr   r7   sabr1   seqr   r   r   test_simple*   s    
&rI   c                 C   s  t jtju sJ t \}\}}t|d | d}tdd t h}|j}t|t sZJ |j	|u shJ |j
dksvJ |dd ttdD  |j
d	ksJ W d    n1 s0    Y  W d    n1 s0    Y  W d    n1 s0    Y  W d    n1 s0    Y  d S )
Nr5   r6   r   r8   r'   c                 s   s   | ]}t d d  V  qdS )c                   S   s   d S r   r   r   r   r   r   r$   K   r%   z?test_dask_backend_uses_autobatching.<locals>.<genexpr>.<lambda>Nr   r(   _r   r   r   r=   J   s   z6test_dask_backend_uses_autobatching.<locals>.<genexpr>     @r>   )r   Zcompute_batch_sizer   r   r   r   r
   _backend
isinstanceparallelZ_effective_batch_sizerA   int)r7   rE   rF   rG   r1   rP   r9   r   r   r   #test_dask_backend_uses_autobatching9   s    
rR   c                   C   s   t  S r   r   r   r   r   r   random2Q   s    rS   c              
   C   s   t  \}\}}t|d | db}tdd8 t dd tdD \}}||ksVJ W d    n1 sj0    Y  W d    n1 s0    Y  W d    n1 s0    Y  d S )Nr5   r6   r   r8   c                 s   s   | ]}t t V  qd S r   )r   rS   r;   r   r   r   r=   Y   r%   z3test_dont_assume_function_purity.<locals>.<genexpr>r	   r   r   r   r
   rA   )r7   rE   rF   rG   r1   xyr   r   r    test_dont_assume_function_purityU   s
    rW   mixedTFc              
   C   s$  ddl m} |s(dd tdD }d}ndd tdD }d}t|||ksRJ t \}\}}t|d	 | d
~}tdd  tddd|}	W d    n1 s0    Y  dd }
|dd}|	|
}t
dd |D sJ W d    n1 s0    Y  W d    n1 s0    Y  d S )Nr   )Batchc                 S   s   g | ]}t t|qS r   r:   r;   r   r   r   r,   a   r%   z&test_dask_funcname.<locals>.<listcomp>   Zbatch_of_inc_4_callsc                 S   s,   g | ]$}|d  rt t|n
t t|qS )r	   )r   absr   r;   r   r   r   r,   d   s   Zmixed_batch_of_inc_4_callsr5   r6   r   r8   r	   all)Z
batch_sizeZpre_dispatchc                 S   s
   t | jS r   )r0   Ztransition_log)Zdask_schedulerr   r   r   fp   s    ztest_dask_funcname.<locals>.f42c                 s   s   | ]}d |d v V  qdS )Zbatch_of_incr   Nr   )r(   tupr   r   r   r=   t   r%   z%test_dask_funcname.<locals>.<genexpr>)Zjoblib._daskrY   rA   reprr   r   r   r
   replaceZrun_on_schedulerr\   )r7   rX   rY   tasksZ
batch_reprrE   rF   rG   r1   rL   r]   r#   r   r   r   test_dask_funcname]   s"    .
rd   c                    s  dd t dD }tdtd d fdd	tdd	d
}t|}z"tdd( t fdd|D }W d    n1 s0    Y  |dd t dD ksJ t	d|}t
| dksJ tdd |D sJ tdd* t  fdd|D }W d    n1 s0    Y  t	d|}t
| dksHJ tdd |D s`J W |  |  n|  |  0 d S )Nc                 S   s   g | ]}g qS r   r   rK   r   r   r   r,      r%   z;test_no_undesired_distributed_cache_hit.<locals>.<listcomp>d   numpyg    .Ac                    s(   |d urj |  | t j | S r   )testingZassert_array_equalappendr   hex)list_data)Xnpr   r   isolated_operation   s    zCtest_no_undesired_distributed_cache_hit.<locals>.isolated_operationr'   r	   )	n_workersthreads_per_workerr   r8   c                 3   s   | ]}t  |V  qd S r   rJ   r(   rj   )rn   r   r   r=      s   z:test_no_undesired_distributed_cache_hit.<locals>.<genexpr>c                 S   s   g | ]}g qS r   r   rK   r   r   r   r,      r%   receive-from-scatterr   c                 S   s   g | ]}t |d kqS r&   r/   r(   rr   r   r   r,      r%   c                 3   s   | ]}t | d V  qdS ))rk   NrJ   rq   )rl   rn   r   r   r=      s   c                 S   s   g | ]}t |d kqS r&   rs   rt   r   r   r   r,      r%   )N)rA   rB   importorskipZarangerQ   r   r   r   r
   r4   sumvaluesr\   close)r7   listsr   r1   rescountsr   )rl   rn   rm   r   'test_no_undesired_distributed_cache_hitw   s4    
&
(

r}   c                   @   s(   e Zd Zdd Zdd ZeZdd ZdS )CountSerializedc                 C   s   || _ d| _d S )Nr   )rU   count)selfrU   r   r   r   __init__   s    zCountSerialized.__init__c                 C   s   | j t|d| S )NrU   )rU   getattr)r   otherr   r   r   __add__   s    zCountSerialized.__add__c                 C   s   |  j d7  _ t| jffS )Nr'   )r   r~   rU   )r   r   r   r   
__reduce__   s    zCountSerialized.__reduce__N)__name__
__module____qualname__r   r   __radd__r   r   r   r   r   r~      s   r~   c                 C   s   | | | | | S r   r   )rF   rG   cder   r   r   add5   s    r   c                 C   s  t d}t d}t d}t <\}\}}t|d | d }td||gdt tt}||||dd	d
||||d	dd
|||||d	d
||||||d
g}	dd |	D }
t |	}W d    n1 s0    Y  tt	@ td| dd W d    n1 s0    Y  W d    n1 s$0    Y  W d    n1 sD0    Y  W d    n1 sd0    Y  ||
ks|J |j
dksJ |j
dksJ |j
dv sJ d S )Nr'   r	   r@   r5   r6   r   )r9   scatterrZ      )r   r   c                 S   s    g | ]\}}}||i |qS r   r   )r(   funcr   r   r   r   r   r,      s   z'test_manual_scatter.<locals>.<listcomp>)r9   r7   r   )rZ      )r~   r   r   r   r   r   r
   rB   rC   	TypeErrorr   )r7   rU   rV   zrE   rF   rG   r1   r]   rc   expectedresultsr   r   r   test_manual_scatter   s.    (r   c           	   
      s  t d}|jtd|jd |jtd|jd} gd |gd  }t \}\}}t|d | d|}tdd( t d	d
 t	|D  W d    n1 s0    Y  t
d|}||d  ||d   dksJ W d    n1 s0    Y  W d    n1 s0    Y  t \}\}}t|d | d}tdd, t  fdd
tdD  W d    n1 sr0    Y  t
d|}||d  dksJ ||d  dksJ W d    n1 s0    Y  W d    n1 s0    Y  d S )Nrf   rM   )Zdtyper@   r5   r6   r   r8   c                 s   s&   | ]\}}t t||||d V  qdS ))optNr   r   )r(   r<   rk   r   r   r   r=      s   z$test_auto_scatter.<locals>.<genexpr>rr   r	   c                 3   s$   | ]}t t d d |V  qd S )Nr@   r   r;   Zdata1r   r   r=      r%   r   r   )rB   rv   onesrQ   Zuint8r   r   r   r
   	enumerater4   rA   )	loop_in_threadrm   Zdata2Zdata_to_processrE   rF   rG   r1   r|   r   r   r   test_auto_scatter   s&    

&
^<
r   retry_noc              
      s   t dd d}fdd fddt \}\}}t|d | db}td	d
8 dt fddt|D }W d    n1 s0    Y  W d    n1 s0    Y  W d    n1 s0    Y  d S )Nrf   r>   c                    s
     | S r   )rw   )rU   r<   j)rm   r   r   my_sum  s    z#test_nested_scatter.<locals>.my_sumc                    sX   t  }tdd0 t  fddtD }W d    n1 sF0    Y  t|S )Nr   r8   c                 3   s&   | ]}t  |d  |V  qd S r   rJ   )r(   r   )arrayr<   r   r   r   r=     s   zEtest_nested_scatter.<locals>.outer_function_joblib.<locals>.<genexpr>)r   r   r
   rA   rw   )r   r<   r1   r   )NUM_INNER_TASKSr   )r   r<   r   outer_function_joblib  s    &z2test_nested_scatter.<locals>.outer_function_joblibr5   r6   r   r8   i'  c                 3   s$   | ]}t  |d  |V  qd S r   rJ   r;   )my_arrayr   r   r   r=     s   z&test_nested_scatter.<locals>.<genexpr>)rB   rv   r   r   r   r   r
   rA   )r7   r   ZNUM_OUTER_TASKSrE   rF   rG   rL   r   )r   r   r   rm   r   r   test_nested_scatter  s    
	
r   c              
      sr  dd  t  L\}\}}t|d | dx}tddN tdd fd	d
tdD }|D ]}tt|dks\J q\W d    n1 s0    Y  W d    n1 s0    Y  t|d | dz}tddN tdd fdd
tdD }|D ]}tt|dksJ qW d    n1 s$0    Y  W d    n1 sD0    Y  W d    n1 sd0    Y  d S )Nc                  S   sH   t tdddd tdD } | t tdddd tdD O } | S )Nr	   n_jobsc                 s   s   | ]}t tj V  qd S r   r   osgetpidrK   r   r   r   r=   &  r%   zOtest_nested_backend_context_manager.<locals>.get_nested_pids.<locals>.<genexpr>c                 s   s   | ]}t tj V  qd S r   r   rK   r   r   r   r=   '  r%   )setr
   rA   )Zpidsr   r   r   get_nested_pids%  s     $z<test_nested_backend_context_manager.<locals>.get_nested_pidsr5   r6   r   r8   r	   r   c                 3   s   | ]}t   V  qd S r   rJ   rK   r   r   r   r=   -  s   z6test_nested_backend_context_manager.<locals>.<genexpr>r>   c                 3   s   | ]}t   V  qd S r   rJ   rK   r   r   r   r=   7  s   )r   r   r   r
   rA   r/   r   )r   rE   rF   rG   r1   Z
pid_groupsZ	pid_groupr   r   r   #test_nested_backend_context_manager$  s     Rr   c           	         s&  dd   fddt  \}\}}t|d | d}tdd t H} |d	ksZJ |jd
kshJ |fddtdD }W d    n1 s0    Y  |D ] \}}|d	ksJ |d
ksJ qW d    n1 s0    Y  W d    n1 s0    Y  W d    n1 s0    Y  d S )Nc                 S   s
   | j jjS r   )rN   	__class__r   pr   r   r   _backend_typeC  s    zJtest_nested_backend_context_manager_implicit_n_jobs.<locals>._backend_typec                     s8   t  }  | | jfW  d    S 1 s*0    Y  d S r   )r
   r   r   )r   r   r   get_nested_implicit_n_jobsF  s    zWtest_nested_backend_context_manager_implicit_n_jobs.<locals>.get_nested_implicit_n_jobsr5   r6   r   r8   r   c                 3   s   | ]}t   V  qd S r   rJ   rK   )r   r   r   r=   P  s   zFtest_nested_backend_context_manager_implicit_n_jobs.<locals>.<genexpr>r	   )r   r   r   r
   r   rA   )	r7   rE   rF   rG   r1   r   Zall_nested_n_jobsZbackend_typeZnested_n_jobsr   )r   r   r   3test_nested_backend_context_manager_implicit_n_jobs?  s    &r   c              	   C   sn   t t:}tdd W d    n1 s,0    Y  W d    n1 sJ0    Y  dt|j v sjJ d S )Nr   r8   zcreate a dask client)rB   rC   r   r   strvaluelower)r7   infor   r   r   test_errorsY  s    <r   c              
   C   s&  t  \}\}}t|d | d}tddF tdddd td	D }t|d
 d
 d
 tsfJ W d    n1 sz0    Y  tddF tdddd td	D }t|d
 d
 d
 tsJ W d    n1 s0    Y  W d    n1 s0    Y  W d    n1 s0    Y  d S )Nr5   r6   r   r8   r	   r   c                 s   s   | ]}t td dV  qd S )NZnested_requirer   outerrK   r   r   r   r=   f  s   z.test_correct_nested_backend.<locals>.<genexpr>r'   r   c                 s   s   | ]}t td dV  qdS )Z	sharedmemr   Nr   rK   r   r   r   r=   l  s   )r   r   r   r
   rA   rO   r   r   )r7   rE   rF   rG   r1   resultr   r   r   test_correct_nested_backenda  s    8r   c                    s"   t ddd fddtdD S )Nr	   threads)r   Zpreferc                 3   s   | ]}t t V  qd S r   )r   middlerK   r   r   r   r=   s  s   zouter.<locals>.<genexpr>r'   r
   rA   r   r   r   r   r   r  s    r   c                 C   s   t d| ddd tdD S )Nr	   )r   requirec                 s   s   | ]}t t V  qd S r   )r   innerrK   r   r   r   r=   y  s   zmiddle.<locals>.<genexpr>r'   r   )r   r   r   r   r   x  s    r   c                   C   s   t  jS r   )r
   rN   r   r   r   r   r   ~  s    r   c              	   C   sx   t | dddV tdd, tdddd	 td
D  W d    n1 sL0    Y  W d    n1 sj0    Y  d S )NFT)r7   	processesZset_as_defaultr   r8   rZ   r   c                 s   s   | ]}t t|V  qd S r   )r   idr;   r   r   r   r=     r%   z0test_secede_with_no_processes.<locals>.<genexpr>r	   )r   r   r
   rA   r6   r   r   r   test_secede_with_no_processes  s    r   c                 C   s   ddl m} | jS )Nr   )
get_worker)r   r   r5   )rL   r   r   r   r   _worker_address  s    r   c              
   C   s"  t  \}\}}t|d | d}td|d d> t dd tdD }||d gd ksdJ W d    n1 sx0    Y  td|d d> t dd tdD }||d gd ksJ W d    n1 s0    Y  W d    n1 s0    Y  W d    n1 s0    Y  d S )	Nr5   r6   r   )r9   workersc                 s   s   | ]}t t|V  qd S r   r   r   r;   r   r   r   r=     s   z-test_dask_backend_keywords.<locals>.<genexpr>r>   c                 s   s   | ]}t t|V  qd S r   r   r;   r   r   r   r=     s   rT   rD   r   r   r   test_dask_backend_keywords  s    
4
r   c              	   C   s   t d| d}tdd( t dd tdD  W d    n1 sF0    Y  t }|jjjr|td t |d	 k sVJ qV|j	rJ W d    n1 s0    Y  d S )
NF)r   r7   r   r8   c                 s   s   | ]}t t|V  qd S r   r:   r;   r   r   r   r=     r%   ztest_cleanup.<locals>.<genexpr>r>   g{Gz?r   )
r   r   r
   rA   r   r   Z	schedulerrc   r   futures)r7   r1   startr   r   r   test_cleanup  s    6
r   cluster_strategyadaptivelate_scalingz2.1.1z1.28.0z?distributed bug - https://github.com/dask/distributed/pull/2841)reasonc                 C   s   t dddd}t|}| dkr.|jddd n| dkr@|d zVtdd	( t d
d tdD  W d    n1 sz0    Y  W |  |  n|  |  0 d S )Nr   Fr	   ro   r   rp   r   )minimummaximumr   r   r8   c                 s   s   | ]}t t|V  qd S r   r:   r;   r   r   r   r=     r%   z(test_wait_for_workers.<locals>.<genexpr>r>   )r   r   Zadaptscaler   r
   rA   ry   )r   r   r1   r   r   r   test_wait_for_workers  s    
8
r   c               	   C   s0  t dddd} t| }z tdddZ d}tjt|d	( t d
d tdD  W d    n1 sh0    Y  W d    n1 s0    Y  tdddZ d}tjt|d	( t dd tdD  W d    n1 s0    Y  W d    n1 s0    Y  W |	  | 	  n|	  | 	  0 d S )Nr   Fr	   r   r   g?)r9   Zwait_for_workers_timeoutz7DaskDistributedBackend has no worker after 0.1 seconds.)matchc                 s   s   | ]}t t|V  qd S r   r:   r;   r   r   r   r=     r%   z0test_wait_for_workers_timeout.<locals>.<genexpr>r>   z+DaskDistributedBackend has no active workerc                 s   s   | ]}t t|V  qd S r   r:   r;   r   r   r   r=     r%   )
r   r   r   rB   rC   TimeoutErrorr
   rA   RuntimeErrorry   )r   r1   msgr   r   r   test_wait_for_workers_timeout  s    TV
r   r9   Zlokymultiprocessingc                    sl   t dd}t|} fdd}||}| }t|dks@J |d j}t|tsXJ dt|v shJ d S )Nr	   )ro   c                     sN   t jdd.} td ddd tdD  W d    n1 s@0    Y  | S )NTrecordr	   )r   r9   c                 s   s   | ]}t t|V  qd S r   r:   r;   r   r   r   r=     s   zftest_joblib_warning_inside_dask_daemonic_worker.<locals>.func_using_joblib_parallel.<locals>.<genexpr>r>   )warningscatch_warningsr
   rA   r   r8   r   r   func_using_joblib_parallel  s
    &zStest_joblib_warning_inside_dask_daemonic_worker.<locals>.func_using_joblib_parallelr'   r   zdistributed.worker.daemon)	r   r   submitr   r/   messagerO   UserWarningr   )r9   r   r1   r   futr   warningr   r8   r   /test_joblib_warning_inside_dask_daemonic_worker  s    


r   )r   )r   r   )B
__future__r   r   r   r   r   rB   r   uuidr   r   r    r
   r   r   rP   r   r   Z_daskr   rv   r   r   r   r   r   Zdistributed.metricsZdistributed.utils_testr   r   r   r"   r4   rI   rR   rS   rW   markZparametrizerd   r}   objectr~   r   r   r   r0   rA   r   r   r   r   r   r   r   r   r   r   r   r   Zskipif__version__r   r   r   r   r   r   r   <module>   sd   


	
9
%
