a !f @s( dZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z ddlm Z ddl mZddlmZddlmZddlZddlZdd lmZdd lmZmZdd lmZdd lmZmZdd lmZddlmZmZddl m!Z!m"Z"m#Z#m$Z$m%Z%edurddl&m'Z'ddl(m)Z)z ddl*Z*Wne+yJdZ*Yn0zddl,m-Z-Wne+ytdZ-Yn0z ddl.Z.Wne+ydZ.Yn0ddl/m0Z0ddl/m1Z1ddl/m2Z2ddl/m3Z3ddl/m4Z4ddl5m6Z6m7Z7ddl5m8Z8ddl5m9Z9ddl5m:Z:ddl5m;Z;mZ>e=?Z@e@AdddgeBe=CZDeDd d!e=D7ZDedurxgZEndd"gZEeEd#gZFeGed$reDHeId%e=e>ZJd&d'ZKd(d)ZLd*d+ZMGd,d-d-eNZOd6d/d0ZPd1d2ZQd7d3d4ZRd5d6ZSd7d8ZTd9d:ZUd;d<ZVe!d=e8e9gejWj!d>d?d@e;d@dAfdBggdCdDedEdFZXe!dGeDe!dHgdIe!dJgdKdLdMZYe!dGeDdNdOZZdPdQZ[ee!dRgdSdTdUZ\ee!dGgdVdWdXZ]dYdZZ^e!d[e=e!d\e=d]d^Z_d_d`Z`edadbZadcddZbe!dHgdedfdgZce!dGeFdhdiZdedjdkZee!dGeFdldmZfee!dGeFdndoZgee!dGeEdpdqZhdrdsZie!dGe=e!dtdgdufdvgdwfgdxdyZjee!dGeFdzd{Zkd|d}Zlee!dGeEd~dZmddZnddZoddZpee!dGeFejWjqddddZrGddde0ZsddZte!dGeDddZuddZvddZwe$eduddddZxddZyeFddZzez{dd!e|dDee!dGeze!d=e9e8gddZ}Gddde0Z~e!d=e8e9gddZe!d=e8e9gddZddZddZGdddejj4Ze:deee!dGgde!d=e8e9gddZee!dHgde!dGeFe!d=e8e9gddZddZee!deFe!deFe!d=e8e9gddZee!d=e8e9ge!ddd.gddZe!dgdddZe!dgdddZeddÄZeee!dGeEddńZdZee!dGeEddȄZdZdZdZdZee!dGeEedurgnd%ge!deeege!dgdϢddфZdjejejejdӍZeddՄZdjejejejdӍZedd؄ZddڄZdd܄ZddބZeee!dGeEddZddZeeddZddZddZejW!dHgdddZe!dGeDe!dHgdddZddZee!dGe@e!dHgdddZe!dGe@e!dHgdddZe!dGe@e!dHgdddZe!dGe@e!dHgdddZe!dgdddZeee!dGeEddZe!dGedu rndd#gngdddZee!dGeEddZee!d=e8e9gddZe!d=e8e9gddZdd Zd8d d Zee!dGd"d#ge!d=e8e9gd d Zee$e.dudde!d=e8e9gddZd9ddZejWje!d=e8e9ge!dGedu rd#gnd"d#gddZddZe!dGedu rdd"gndge$e-duddddZe$e.dudde!d=e8e9gddZe!d=e8e9gddZdd Zd!d"Zd#d$Zd%d&Zeee!dHgd'd(d)Zeee!d*gd+e!dHd,d@ge!d=e8e9gd-d.Zee!dHd,d@ge!d/gd0e!d=e8e9gd1d2Zee!dHgd3d4d5ZdS(:z Test the parallel module. N)format_exceptionsqrt)sleep) PicklingError) nullcontext TimeoutErrorparallel)dumpload)mp)np with_numpy)with_multiprocessing)IS_PYPY force_gc_pypy) parametrizeraisescheck_subprocess_callskipifwarns)get_reusable_executor)Queue) parallel_sum)SequentialBackend)ThreadingBackend)MultiprocessingBackend)ParallelBackendBase) LokyBackend)Paralleldelayed)parallel_config)parallel_backend)register_parallel_backend)effective_n_jobs cpu_count)rBACKENDSDEFAULT_BACKENDmultiprocessingcCsg|]}t|qS)r)).0Z backend_strr,r,Z/var/www/html/python-backend/venv/lib/python3.9/site-packages/joblib/test/test_parallel.py Mr/loky threading get_contextspawncCst|dt|ddS)N_poolZ_workers)getattrbackendr,r,r. get_workers[sr9cCs||SNr,)xyr,r,r.division_sr=cCs|dSNr,r;r,r,r.squarecsrAc@seZdZdZddZdS)MyExceptionWithFinickyInitz1An exception class with non trivial __init__ cCsdSr:r,)selfabcdr,r,r.__init__jsz#MyExceptionWithFinickyInit.__init__N__name__ __module__ __qualname____doc__rHr,r,r,r.rBgsrBFcCs"|dkr|rtddddnt|S)NrDrErFrG)rB ValueError)r;custom_exceptionr,r,r.exception_raiserns rQcCstdtdS)Ng?)timerKeyboardInterruptr@r,r,r.interrupt_raiserus rTcCs|d||S)zO A module-level function so that it can be spawn with multiprocessing. r?r,)r;r<zr,r,r.fzsrVcCsttdSNr)typer get_active_backendr,r,r,r._active_backend_typesrZcCst||dddtdDS)Nn_jobsr8css|]}tt|VqdSr:r#rAr-ir,r,r. sz parallel_func..r"range) inner_n_jobsr8r,r,r. parallel_funcsrecCstdksJdSrW)r(r,r,r,r.test_cpu_countsrfcCstdksJdSrWr'r,r,r,r.test_effective_n_jobssrhcontextzbackend_n_jobs, expected_n_jobs)rarar\Nr)z positive-intz negative-intNone)ZidscCsT|d|d"tdd|ks JWdn1s40YtdddksPJdS)Nr2rkrrg)riZbackend_n_jobsZexpected_n_jobsr,r,r.test_effective_n_jobs_Nones 0rnr8r\)rr?rjverbose)r? dcCs:ddtdDt|||dddtdDks6JdS)NcSsg|] }t|qSr,rAr-r;r,r,r.r/r0z(test_simple_parallel..)r\r8rpcss|]}tt|VqdSr:r]rtr,r,r.r`sz'test_simple_parallel..rcr")r8r\rpr,r,r.test_simple_parallelsrwcCs|jtdddtjdd>}td|ddd td D}|gd ksPJWdn1sd0Yd d |D}t|dksJdS)NnameZ!some_new_name_for_the_main_thread)targetrxvalueTrecordr?r[css|]}tt|VqdSr:r]rtr,r,r.r`sz6test_main_thread_renamed_no_warning..ra)rrcSsg|]}dt|jvr|qS)zworker timeout)strmessager-wr,r,r.r/r0z7test_main_thread_renamed_no_warning..r)setattrr2current_threadwarningscatch_warningsr"rclen)r8 monkeypatchwarninforesultsr,r,r.#test_main_thread_renamed_no_warnings .rcCstjdd&}tdt||dWdn1s80Ydd|D}|r|rtdd|D}ttjd d rt|d kn t|d k}|o|Sd S|rJdSdS) NTr{always)r8rdcSsg|] }|jqSr,)rrr,r,r.r/r0z*_assert_warning_nested..css|]}d|jdvVqdS)zbacked parallel loops cannotrN)args)r-Zeachr,r,r.r`sz)_assert_warning_nested..ZnogilFr) rr simplefilterreallr6sysflagsr)r8rdexpectedrZwarnings_are_correctZwarnings_have_the_right_lengthr,r,r._assert_warning_nesteds  * rz%parent_backend,child_backend,expected))r1r+T)r1r1F)r+r+T)r+r1T)r2r+T)r2r1Tcstd|dfddtdDtd|dfddtdD}|dkrttrft|sftjdd t|sJn t|sJdS) Nr?r[c3s |]}ttdddVqdS)rFr8rdrNr#rr-_ child_backendr,r.r`s z0test_nested_parallel_warnings..ruc3s |]}ttddVqdS)r?rNrrrrr,r.r`s r2zThis test often fails in PyPy.reason)r"rcranypytestxfailr)parent_backendrrresr,rr.test_nested_parallel_warningss   r)r1r+r2cCs>dg}dd}tj||fd}|||ds:JdS)NFcSshtjdd,}tddddtdDWdn1s>0Ytt|t|dk|d<dS) NTr{r?rkcss|]}ttdVqdS)皙?Nr#rrr,r,r.r`szPtest_background_thread_parallelism..background_thread..r}r)rrr"rcprintr)is_run_parallelrr,r,r.background_thread s & z=test_background_thread_parallelism..background_thread)ryrr)r2Threadstartjoin)r8rrtr,r,r."test_background_thread_parallelisms rcCs"td|dddtdDdS)Nr?r[css|]}ttdVqdS){Gz?Nr]rr,r,r.r`sznested_loop..rbr7r,r,r. nested_loopsrrrcs&td|dfddtdDdS)Nr?r[c3s|]}ttVqdSr:)r#rrrr,r.r`!sz#test_nested_loop..rb)rrr,rr.test_nested_loopsrcCstdSr:)rOr7r,r,r.raise_exception%src CstttXtddd,}|ttdttdgWdn1sH0YWdn1sf0YdS)Nr?r1r[)rrOr"r#rrr r,r,r.)test_nested_loop_with_exception_with_loky)s    rcs<tddtdddfddtdDs8JdS) z1Input is mutable when using the threading backendru)maxsizer?r2r[c3s|]}tjdVqdSrN)r#putrqr,r.r`4sz1test_mutate_input_with_threads..N)rr"rcfullr,r,rr.test_mutate_input_with_threads1s  r)rr?racCs6td}dd|Dt|ddd|Dks2JdS)z.Check the keyword argument processing of pmap. cSsg|]}t|ddqSrr<rVrtr,r,r.r/=r0z(test_parallel_kwargs..rkcss|]}tt|ddVqdSrrNr#rVrtr,r,r.r`>r0z'test_parallel_kwargs..Nrv)r\lstr,r,r.test_parallel_kwargs9s rcCstd}dd|D}td|dh}|j}||dd|DksDJ||dd|Dks^Jtdur|t|t|jus|JWdn1s0Ytdurt|jdusJ||d d|DksJtdurt|jdusJdS) NrcSsg|]}t|ddqSrrrtr,r,r.r/Dr0z4test_parallel_as_context_manager..r}r[css|]}tt|ddVqdSrrrtr,r,r.r`Mr0z3test_parallel_as_context_manager..css|]}tt|ddVqdSrrrtr,r,r.r`Nr0css|]}tt|ddVqdSrrrtr,r,r.r`Zr0)rcr"_backendrr9)r8rrpZmanaged_backendr,r,r. test_parallel_as_context_managerAs4rcsbGdddtttdd2tdddfdd td DWd n1sT0Yd S) ze Check that pmap captures the errors when it is passed an object that cannot be pickled. c@seZdZddZdS)z1test_parallel_pickling..UnpicklableObjectcSs tddS)NZ123) RuntimeErrorrCr,r,r. __reduce__esz.UnpicklableObject.__reduce__N)rJrKrLrr,r,r,r.UnpicklableObjectdsrzthe task to sendmatchr?r1r[c3s|]}ttVqdSr:r#idrrr,r.r`isz)test_parallel_pickling..rN)objectrrr"rcr,r,rr.test_parallel_pickling_s rcCs0ttd|ddddtdDdks,JdS)Nr?r\r8timeoutcss|]}ttdVqdS)gMbP?Nrrtr,r,r.r`psz0test_parallel_timeout_success..r)rr"rcr7r,r,r.test_parallel_timeout_successms rcCsLtt0td|ddddtdDWdn1s>0YdS)Nr?rrcss|]}ttdVqdSrNrrtr,r,r.r`ysz-test_parallel_timeout_fail..r)rr r"rcr7r,r,r.test_parallel_timeout_failts rc Cstdur.)rr)rrcSsg|]}tt|qSr,r#rTrtr,r,r.r/r0cSsg|]\}}tt||qSr,rrr,r,r.r/scSsg|]}t|ddqSrrrtr,r,r.r/r0rcss|]}tt|ddVqdSrrrtr,r,r.r`r0z%test_error_capture..cSsg|]}tt|qSr,rrtr,r,r.r/r0cSsg|]}t|ddqSrrrtr,r,r.r/r0css|]}tt|ddVqdSrrrtr,r,r.r`r0rkcSsg|]}tt|qSr,rrtr,r,r.r/r0cSsg|]\}}tt||qSr,rrr,r,r.r/r0r)r\rpcss|]}tt|ddVqdS)T)rPNr#rQr^r,r,r.r`sr)rrZeroDivisionErrorr"ziprSr9rrc _iteratingZn_completed_tasksZn_dispatched_tasks _abortingrB)r8r Zoriginal_workersr,r,r.test_error_capture}sd   "   "  &   2$  $ $  rcCs|d|dS)Nz Consumed %s)append)queueitemr,r,r.consumersrzbatch_size, expected_queue) Produced 0 Consumed 0 Produced 1 Consumed 1 Produced 2 Consumed 2 Produced 3 Consumed 3 Produced 4 Consumed 4 Produced 5 Consumed 5r}) rrrrrrrrrrrrcsTtfdd}td||dfdd|D|ks@JtdksPJdS) zC Test that with only one job, Parallel does act as a iterator. c3s&tdD]}d||VqdSNz Produced %ircrr_rr,r.producers z'test_dispatch_one_job..producerrr\ batch_sizer8c3s|]}tt|VqdSr:r#rrtrr,r.r`sz(test_dispatch_one_job.. N)listr"r)r8rZexpected_queuerr,rr.test_dispatch_one_jobs  rcst}|fdd}tddd|dfdd|Dt}|d d ksXJ|d d d }|dksvJ|d}||ksJtdksJd S)zW Check that using pre_dispatch Parallel does indeed dispatch items lazily. c3s&tdD]}d||VqdSrrrrr,r.rs z/test_dispatch_multiprocessing..producerr?rra)r\r pre_dispatchr8c3s|]}ttdVqdS)rNrrrr,r.r`sz0test_dispatch_multiprocessing..rrNr}z Consumed anyrjrr)rManagerrr"indexr)r8managerrZqueue_contentsZfirst_consumption_indexZproduced_3_indexr,rr.test_dispatch_multiprocessings    rcCsZtdddd8}|ddtdD|jdks8JWdn1sL0YdS) Nr?autor2rcss|]}tt|VqdSr:rr^r,r,r.r` r0z/test_batching_auto_threading..rr"rcrZcompute_batch_size)rr,r,r.test_batching_auto_threadingsrcCsZtdd|d8}|ddtdD|jdks8JWdn1sL0YdS)Nr?rrcss|]}tt|VqdSr:rr^r,r,r.r`r0z2test_batching_auto_subprocesses..rrr)r8rr,r,r.test_batching_auto_subprocesses srcCsLtt0tddddddtdDWdn1s>0YdS) zCMake sure that exception raised during dispatch are indeed capturedr?r)r\rrpcss|]}tt|VqdSr:rr^r,r,r.r`sz*test_exception_dispatch..rN)rrOr"rcr,r,r,r.test_exception_dispatchs rcCs tddddtdDdS)Nr?rkcss|]}tt|VqdSr:rr-jr,r,r.r`!sz(nested_function_inner..rrbrr,r,r.nested_function_inner srcCs tddddtdDdS)Nr?rkcss|]}tt|VqdSr:)r#rrr,r,r.r`&sz(nested_function_outer..rrbrr,r,r.nested_function_outer%srz'https://github.com/joblib/loky/pull/255rcCstt.}td|dddtdDWdn1s<0Yt|j|j|j}d|}d|vsnJd |vszJd |vsJt|jtusJdS) zEnsure errors for nested joblib cases gets propagated We rely on the Python 3 built-in __cause__ system that already report this kind of information to the user. r?r[css|]}tt|VqdSr:)r#rr^r,r,r.r`4sz1test_nested_exception_dispatch..rNrrrQ) rrOr"rcrrXrztbr)r8excinfoZ report_linesreportr,r,r.test_nested_exception_dispatch*s &    r c@s$eZdZdZdddZd ddZdS) FakeParallelBackendz8Pretends to run concurrently while running sequentially.rNcKs|||_||_|Sr:)r'r\r )rCr\r Z backend_argsr,r,r. configureEs zFakeParallelBackend.configurecCs"|dkrttd|d}|S)Nrr)maxrr(rCr\r,r,r.r'Jsz$FakeParallelBackend.effective_n_jobs)rN)r)rJrKrLrMrr'r,r,r,r.rBs rc CsttddtddWdn1s,0Yttdd:tddWdn1sd0YWdn1s0Yttdd:tddWdn1s0YWdn1s0YdS)NzInvalid backend:rz unit-testingr7)rrOr"r$r,r,r,r.test_invalid_backendPs( < rcCsNtt }td|dWdn1s.0Ydt|jvsJJdS)Nrr[z&n_jobs == 0 in Parallel has no meaning)rrOr"Z_initialize_backendr~rz)r8r r,r,r.test_invalid_njobs]s .rcCs>z0tdtdtvsJtdtks(JWtd=ntd=0dS)NZ test_backend)r&rr)r,r,r,r.test_register_parallel_backendds   rcCsTttksJz*tdtdddttks0JWtt_ntt_0ttksPJdS)Nr2T)Z make_default)rZDefaultBackendr&r)rr*r r,r,r,r.test_overwrite_default_backendms rzOnly without multiprocessingcCsttdd,tddddtdDWdn1s>0Ytdd(tddtdDWdn1s0YdS) Nz)joblib backend '.*' is not available on.*rr1r7css|]}tt|VqdSr:r]r^r,r,r.r`}r0z2test_backend_no_multiprocessing..racss|]}tt|VqdSr:r]r^r,r,r.r`r0)r UserWarningr"rcr$r,r,r,r.test_backend_no_multiprocessingys : rcCs||ddt\}}|dks&Jtddks6Jt}|jdksJJ|dkrvt|tksbJt|jtksJn|dkrt|tksJt|jtksJnX|dkrt|t ksJt|jt ksJn,| drt|t ksJt|jt ksJWdn1s0YdS)Nrarkr+r1r2Ztest_) r rYr'r"r\rXrrr!r startswithr)ri backend_nameactive_backend active_n_jobsrr,r,r.check_backend_context_managers$   rcCsg|] }d|qS)ztest_backend_%dr,r^r,r,r.r/r0racsl|tvr|t|tttks$Jt|ttks.)r)setitemrrZrrr" all_backends_for_context_manager)rr8rir,rr.test_backend_context_managers r!c@seZdZdZdddZdS)ParameterizedParallelBackendz9Pretends to run conncurrently while running sequentially.NcCs|durtd||_dS)Nzparam should not be None)rOparam)rCr#r,r,r.rHsz%ParameterizedParallelBackend.__init__)NrIr,r,r,r.r"sr"cCs|tdtttksJ|dddd~t\}}t|tksHJ|jdksVJ|dksbJt }|j dksvJ|j |usJ|ddt dD}Wdn1s0Y|dd t dDksJttksJdS) NZ param_backend*ra)r#r\css|]}tt|VqdSr:r#rr^r,r,r.r`r0z=test_parameterized_backend_context_manager..rucSsg|] }t|qSr,rr^r,r,r.r/r0z>test_parameterized_backend_context_manager..) rr)r"rZrr rYrXr#r"r\rrc)rrirrrrr,r,r.*test_parameterized_backend_context_managers  4r&cCsttksJ|tdddd~t\}}t|tks>J|jdksLJ|dksXJt}|jdkslJ|j |uszJ|ddt dD}Wdn1s0Y|ddt dDksJttksJdS) N+)r#rurkcss|]}tt|VqdSr:r%r^r,r,r.r`r0zFtest_directly_parameterized_backend_context_manager..cSsg|] }t|qSr,rr^r,r,r.r/r0zGtest_directly_parameterized_backend_context_manager..) rZrr"r rYrXr#r"r\rrc)rirrrrr,r,r.3test_directly_parameterized_backend_context_managers  4r(cCstdtS)Nr)rosgetpidr,r,r,r.sleep_and_return_pidsr+cCs<ttksJtdks JtddddtdDS)Nrr?rkcss|]}ttVqdSr:)r#r+rr,r,r.r`sz"get_nested_pids..)rZrr"_effective_n_jobsrcr,r,r,r.get_nested_pidss r-cs eZdZdZfddZZS) MyBackendz:Backend to test backward compatibility with older backendscstt|dSrW)superr.get_nested_backendr __class__r,r.r0szMyBackend.get_nested_backend)rJrKrLrMr0 __classcell__r,r,r1r.r.sr.back_compat_backend)r2r1r+r4cCsf||JtddddtdD}|D]}tt|dks*Jq*Wdn1sX0YdS)Nr?rkcss|]}ttVqdSr:)r#r-rr,r,r.r` sz6test_nested_backend_context_manager..rr)r"rcrset)rir8Z pid_groupsZ pid_groupr,r,r.#test_nested_backend_context_managers  r6)r?rjNcs|ddtddfddtdD|d4tddfddtdDWdn1sn0YdS) NcSs0tt|ksJt|}t|ks,JdSr:)rZr)r'r"r,)Zexpected_backend_typeZexpected_n_jobr,r,r.check_nested_backendsz?test_nested_backend_in_sequential..check_nested_backendrrkc3s|]}ttdVqdSr)r#r*r)r7r,r.r`&sz4test_nested_backend_in_sequential..rc3s|]}tVqdSr:r#rr8r7r\r,r.r`,srb)r8r\rir,r9r.!test_nested_backend_in_sequentials r:cCsZ||>}|tur|d}|tur*|d}|j|ks8JWdn1sL0YdS)Nr8r)r$r% nesting_level)ri inner_backendZexpected_levelctxr8r,r,r.check_nesting_level2s  r> outer_backendr<cs~t|dtd|dfddtdDdd.tfddtdDWdn1sp0YdS) Nrr?r[c3s|]}ttdVqdSrr#r>rrir<r,r.r`Csz-test_backend_nesting_level..rrkc3s|]}ttdVqdSrr@rrAr,r.r`Is)r>r"rc)rir?r<r,rAr.test_backend_nesting_level;s rBwith_retrieve_callbackTcsddlGfdddt}td|dd|dd}tddfd d td D|turt|d jd kstJ|tur|djd ksJWdn1s0YdS)Nrcs"eZdZdZZjddZdS)z)test_retrieval_context..MyBackendrcss|jd7_dVdSrlrrr,r,r.retrieval_contextWsz;test_retrieval_context..MyBackend.retrieval_contextN)rJrKrLr_Zsupports_retrieve_callbackcontextmanagerrDr,) contextlibrCr,r.r.Ssr.Z retrievalcSstddddt|DS)Nr?rkcss|]}tt|VqdSr:rr^r,r,r.r`_r0z>test_retrieval_context..nested_call..rb)nr,r,r. nested_call^sz+test_retrieval_context..nested_callr?rkc3s|]}t|VqdSr:r8r^)rHr,r.r`bsz)test_retrieval_context..rur8r)rFrr&r"rcr$r_r%)rirCr.r=r,)rFrHrCr.test_retrieval_contextMs  rIr)rrjgQ?cCs6ttt|dWdn1s(0YdS)Nr)rrOr"rJr,r,r.test_invalid_batch_sizeos rKz)n_tasks, n_jobs, pre_dispatch, batch_size) )r?r?rr)r?r?r\rrr?r\r)ir?r\rrL)rr}r\r)rr\r)r 2 * n_jobsr)rrr)rPrrOrN)rMrrOrcCsN|||d}ddt|D}tfi|ddt|D}||ksJJdS)N)r\rrcSsg|] }t|qSr,rsr^r,r,r.r/r0z0test_dispatch_race_condition..css|]}tt|VqdSr:r]r^r,r,r.r`r0z/test_dispatch_race_condition..rv)Zn_tasksr\rrparamsrrr,r,r.test_dispatch_race_conditionus  rRcCs8t}tddd}|jd}|}||ks4JdS)Nr?r+r[ri)rget_start_methodr"Z _backend_argsget)Zmp_start_methodrriZ start_methodr,r,r.test_default_mp_contexts   rUcs^|dkrtd}tjd}|ddtjtd|dfddt dDdS) Nr+r4r$ir?r[c3s |]}ttjjVqdSr:)r#rdotTr^rDr,r.r`szAtest_no_blas_crash_or_freeze_with_subprocesses..) rr3rrandomZ RandomStaterandnrVrWr"rc)r8rngr,rXr..test_no_blas_crash_or_freeze_with_subprocessess   r\afrom joblib import Parallel, delayed def square(x): return x ** 2 backend = "{}" if backend == "spawn": from multiprocessing import get_context backend = get_context(backend) print(Parallel(n_jobs=2, backend=backend)( delayed(square)(i) for i in range(5))) cCsB|dkrtdkrtdt|}ttjd|gddddS)Nr+forkzVRequire fork start method to use interactively defined functions with multiprocessing.z-cr\[0, 1, 4, 9, 16\]r stdout_regex) rrSrskip,UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_NO_MAINformatrr executable)r8coder,r,r.2test_parallel_with_interactively_defined_functionss   rfaimport sys # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed def run(f, x): return f(x) {define_func} if __name__ == "__main__": backend = "{backend}" if backend == "spawn": from multiprocessing import get_context backend = get_context(backend) callable_position = "{callable_position}" if callable_position == "delayed": print(Parallel(n_jobs=2, backend=backend)( delayed(square)(i) for i in range(5))) elif callable_position == "args": print(Parallel(n_jobs=2, backend=backend)( delayed(run)(square, i) for i in range(5))) else: print(Parallel(n_jobs=2, backend=backend)( delayed(run)(f=square, x=i) for i in range(5))) z!def square(x): return x ** 2 zcdef gen_square(): def square(x): return x ** 2 return square square = gen_square() zsquare = lambda x: x ** 2 define_funccallable_position)r#rkwargsc Csv|dvr$|tkstjdkr$tdtj|||tj tj t j d}| d}| |ttj|jgddddS) N)r+r4win32zNot picklable with pickle)rgr8rhjoblib_root_folderzunpicklable_func_script.pyrr^r_) SQUARE_MAINrplatformrra)UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_MAINrcr)pathdirnamejoblib__file__rwriterrdstrpath)r8rgrhtmpdirre code_filer,r,r.0test_parallel_with_unpicklable_functions_in_argss"    rwaimport sys import faulthandler # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed from functools import partial class MyClass: '''Class defined in the __main__ namespace''' def __init__(self, value): self.value = value def square(x, ignored=None, ignored2=None): '''Function defined in the __main__ namespace''' return x.value ** 2 square2 = partial(square, ignored2='something') # Here, we do not need the `if __name__ == "__main__":` safeguard when # using the default `loky` backend (even on Windows). # To make debugging easier faulthandler.dump_traceback_later(30, exit=True) # The following baroque function call is meant to check that joblib # introspection rightfully uses cloudpickle instead of the (faster) pickle # module of the standard library when necessary. In particular cloudpickle is # necessary for functions and instances of classes interactively defined in the # __main__ module. print(Parallel(backend="loky", n_jobs=2)( delayed(square2)(MyClass(i), ignored=[dict(a=MyClass(1))]) for i in range(5) )) )rkcCs.|d}|tttj|jgddddS)Nz(joblib_interactively_defined_function.pyr^)r`r)rrs5INTERACTIVE_DEFINED_FUNCTION_AND_CLASS_SCRIPT_CONTENTrrrdrtruscriptr,r,r.7test_parallel_with_interactively_defined_functions_loky=s   r{aimport sys # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed, hash import multiprocessing as mp mp.util.log_to_stderr(5) class MyList(list): '''MyList is interactively defined by MyList.append is a built-in''' def __hash__(self): # XXX: workaround limitation in cloudpickle return hash(self).__hash__() l = MyList() print(Parallel(backend="loky", n_jobs=2)( delayed(l.append)(i) for i in range(3) )) cCs0|d}|tttj|jgdddddS)Nz)joblib_interactive_bound_method_script.pyz\[None, None, None\]Z LokyProcess)r`Z stderr_regexr)rrs9INTERACTIVELY_DEFINED_SUBCLASS_WITH_METHOD_SCRIPT_CONTENTrrrdrtryr,r,r.:test_parallel_with_interactively_defined_bound_method_lokyes   r~cCs"tg}tdd|gksJdS)Nr?rk)iterr")Zexhausted_iteratorr,r,r.%test_parallel_with_exhausted_iteratorosrcCsttddS)z+Helper function to force gc in each worker.rN)rrRrr,r,r,r._cleanup_workertsrcCs"t|tjstdt||S)Nz#Expected np.memmap instance, got %r) isinstancermemmap TypeErrorrXcopyrXr,r,r. check_memmapzs  rcCsdd}tdd|ddd|dD}t||t|D]\}}tj||q:td d|dd d|dD}t||t|D]\}}tj||qdS) Ncss(t|D]}tjdtjd|VqdS)NrZdtype)rcronesZfloat32)rGr_r,r,r.generate_arrayss zBtest_auto_memmap_on_arrays_from_generator..generate_arraysr?rr\ max_nbytesr8css|]}tt|VqdSr:r#rr-rDr,r,r.r`sz.rrr}css|]}tt|VqdSr:rrr,r,r.r`s)r"rrrtestingassert_array_equal)r8rrresultrr,r,r.)test_auto_memmap_on_arrays_from_generatorsrcCs|Sr:r,)argr,r,r.identitysrcs|dj}tj}tj|ddtj|ddg}t||t|ddt ddfdd d D\}t d tj svJd j |ksJtj ||dS) Nz test.mmapuint8rr)Z mmap_moder?rkc3s|]}ttVqdSr:)r#rrrr,r.r`r0z.test_memmap_with_big_offset..)rr)rrtmmapALLOCATIONGRANULARITYrZzerosrr r r"rroffsetrr)rufnamesizeobjrr,rr.test_memmap_with_big_offsets   rcCstjdd.}tdddddtdDWdn1s@0Yt|dksZJ|d}t|jtsrJt|jd ksJdS) NTr{r)r\rcss|]}tt|VqdSr:r]r^r,r,r.r`r0zFtest_warning_about_timeout_not_supported_by_backend..2rzThe backend class 'SequentialBackend' does not support timeout. You have set 'timeout=1' in Parallel but the 'timeout' parameter will not be used.) rrr"rcrrrrr~)rrr,r,r.3test_warning_about_timeout_not_supported_by_backends<rcCs |||<|Sr:r,) input_listrrzr,r,r.set_list_valuesr)rr?r}csVdgdt|dddfddtdD}t|}tddt|DsRJdS) Nrru generatorr2)r\ return_asr8c3s|]}tt||VqdSr:)r#rr^rr,r.r`szPtest_parallel_return_order_with_return_as_generator_parameter..css|]\}}||kVqdSr:r,)r-vrr,r,r.r`r0)r"rcrrr)r\rr,rr.=test_parallel_return_order_with_return_as_generator_parameters  r)rr?rorjcCsvdgdgd}tt2t}t||ddd|DWdn1sP0Yt|}|dksrJdS)NrDrrrr[css|]}ttj|VqdSr:r#rRrr^r,r,r.r`sz%test_abort_backend..)rrrRr")r\r8Zdelayst_startdtr,r,r.test_abort_backends & rcCstjtdtd}d|d<|S)NgArFr)rrintbool)rrr,r,r.get_large_objectsrcCs`t||dd>}|ddtdD}t|t|~tWdn1sR0YdS)Nr)r\r8rcss|]}tt|VqdSr:)r#rr^r,r,r.r`r0z/test_deadlock_with_generator..r)r"rcnextr)r8r\r rr,r,r.test_deadlock_with_generatorsrcCsttddRt||dd}|ddtdD}t}|ddtd D}Wdn1sd0Yt|d ksJd ~tdS) N)This Parallel instance is already runningrrr8rcss|]}ttdVqdSrrrr,r,r.r`r0z/test_multiple_generator_call..rcss|]}tt|VqdSr:rr^r,r,r.r`r0rrr?ZThe error should be raised immediatly when submitting a new task but it took more than 2s.)rrr"rcrRr)r8r\r grZgen2r,r,r.test_multiple_generator_calls4rc Cst||dd}|ddtdD}t}ttdd&|ddtd D}Wdn1sf0Yt|d ksJd Wdn1s0Y~tdS) Nrrcss|]}ttdVqdSrrrr,r,r.r`r0z7test_multiple_generator_call_managed..rrrcss|]}tt|VqdSr:rr^r,r,r.r`r0rrr?r)r"rcrRrrr)r8r\r rrg2r,r,r.$test_multiple_generator_call_managed s4"rcCst||ddddtdD}t||ddddtddD}tddt|tdDsbJtd dt|tddDsJdS) Nrrcss|]}tt|dVqdSr?Nr%r^r,r,r.r`)sz9test_multiple_generator_call_separated..rcss|]}tt|dVqdSrr%r^r,r,r.r`,srcss|]\}}||kVqdSr:r,r-rr_r,r,r.r`0r0css|]\}}||kVqdSr:r,rr,r,r.r`1r0)r"rcrr)r8r\rrr,r,r.&test_multiple_generator_call_separated%s rzbackend, error))r1T)r2F) sequentialFcCsL|dkrtdurtdtd|dd}|ddtdD}t|d d }|r^ttd d nt }|bt }td|ddd dtddD}t t ddt |tddDsJWdn1s0Yt |dksJd}|jr|dkr|d7}t dqt |dks.J|dkrH|jsHJdS)Nr1zRequires multiprocessingr?rrcss|]}ttdVqdSrrr^r,r,r.r`Ar0z.rcSstdS)NzGenerator collected)rr,r,r,r.Br0z;test_multiple_generator_call_separated_gc..z The executor underlying Parallelrcss|]}tt|dVqdSrr%r^r,r,r.r`Msrcss|]\}}||kVqdSr:r,rr,r,r.r`Tr0rurrarg?)rrrar"rcweakreffinalizerrrrRrrraliverr,r)r8errorr rZg_wrr=rretryr,r,r.)test_multiple_generator_call_separated_gc4s0 @ rcCs&|j}tdd||d\}|ddtjdgdDtt|dksNJ|ddtdDWdn1sx0Ytd D]}t|sqtd qt d tdd|d }|d dtjdgdD|ddtdDtd D]}t|sq"td qt d dS)Nr?r)r\rr8Z temp_foldercss|]}tt|VqdSr:rrr,r,r.r`qr0z(test_memmapping_leaks..rrcss|]}ttVqdSr:r#rrr,r,r.r`{r0rrrz/temporary directory of Parallel was not removedrcss|]}tt|VqdSr:rrr,r,r.r`r0css|]}ttVqdSr:rrr,r,r.r`r0) rtr"rrYrr)listdirrcrAssertionError)r8rurrr,r,r.test_memmapping_leakses* 4       r)Nr1r2cCs<td|dddtdD}|ddtdDks8JdS)Nr?r[css|]}tdd|VqdS)cSs|dSr>r,r@r,r,r.rr0z2test_lambda_expression...Nr8r^r,r,r.r`sz)test_lambda_expression..rcSsg|] }|dqS)r?r,r^r,r,r.r/r0z*test_lambda_expression..rb)r8rr,r,r.test_lambda_expressionsrcsd}d}d|td||d}|fddt|D|jj|jjksLJ|jj|jjks`J|fddt|D|jj|jjksJ|jj|jjksJd S) zCTest that a parallel backend correctly resets its batch statistics.r?ig@r)rpr\r8c3s|]}ttjVqdSr:rr^ task_timer,r.r`r0z6test_backend_batch_statistics_reset..c3s|]}ttjVqdSr:rr^rr,r.r`r0N)r"rcrZ_effective_batch_sizeZ_DEFAULT_EFFECTIVE_BATCH_SIZEZ_smoothed_batch_durationZ _DEFAULT_SMOOTHED_BATCH_DURATION)r8r\Zn_inputsrr,rr.#test_backend_batch_statistics_resets$rcCsdD]v}tt|djtks Jt|dd}t|jtks>Jt|dd}t|jtks\Jt|dd}t|jtksJqtdd dd }t|jtksJ|d dd:tdd }t|jtksJ|jdksJWdn1s0Y|d dd@td dd}t|jtks J|jd ks0JWdn1sF0Y|d dd>tdd }t|jtks|J|jdksJWdn1s0Y|d dd@td dd}t|jtksJ|jd ksJWdn1s0YdS)N)rr?rjrkthreadsr\prefer processes sharedmemr\requirer?r1)r\r8rrrarr)rXr"rrrr!r\)rir\rr,r,r.$test_backend_hinting_and_constraintss4    , 0 0 rcCsGdddt}||Ltddd}t|j|ks:Jtddd}t|j|ksXJWdn1sl0YGdd d t}||tddd}t|j|ksJ|\}}|d ksJ|d ksJtddd d }t|jtksJ|\}}d }||ksJ|d ks$JWdn1s:0Yttt|ddWdn1sr0YdS)Nc@s$eZdZdZdZddZddZdS)z[test_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackendTcSsdSr:r,rr,r,r. apply_asyncszgtest_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackend.apply_asynccSs|Sr:r,rr,r,r.r'szltest_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackend.effective_n_jobsNrJrKrLZsupports_sharedmemZ use_threadsrr'r,r,r,r.MyCustomThreadingBackendsrr?rrrrc@s$eZdZdZdZddZddZdS)z\test_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackendFcSsdSr:r,rr,r,r.rszhtest_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackend.apply_asynccSs|Sr:r,rr,r,r.r'szmtest_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackend.effective_n_jobsNrr,r,r,r.MyCustomProcessingBackendsrr r)r\rrpzUsing ThreadingBackend as joblib backend instead of MyCustomProcessingBackend as the latter does not provide shared memory semantics.r8r) r r"rXrZ readouterrrstriprrO)Zcapsysrirrrouterrrr,r,r.9test_backend_hinting_and_constraints_with_custom_backendss*   0      . rcCstttddWdn1s(0YtttddWdn1sZ0YtttdddWdn1s0Ytdur tttdddWdn1s0Ytttd ddWdn1s0YdS) Ninvalidrrrr)rrr1rr+)rrOr"rr,r,r,r.,test_invalid_backend_hinting_and_constraintss ( ( *  * rc stddh}t|jj|jjfg}dkr<|WdS|fddtdD}||dWdS1sx0YdS)zCPerform nested parallel calls and introspect the backend on the wayr?rkrNc3s(|] }ttfddiVqdS)limitrN)r#_recursive_backend_infor^rirr,r.r`7sz*_recursive_backend_info..r)r"rXrrJr;rc)rrirZ this_levelrr,rr.r0s rcCs||ddt}Wdn1s(0YtdkrFd}d}nd}d}|d}|df|dfd|fd|fg}||ksJdS)Nr?rkrrrBackendr)rr(title)rir8backend_types_and_levelsZsecond_level_backend_typeZ max_levelZtop_level_backend_typeZexpected_types_and_levelsr,r,r.test_nested_parallelism_limit<s$  rzThis test requires daskc Cstjddd}tjtdtjd}tdD]\}|dt|d}Wdn1sV0Yt|dkspJt dd |Ds*Jq*|dt}Wdn1s0Yt|dksJt d d |DsJdS) Nr?)Z n_workersZthreads_per_workergcArdask)datar}css|]\}}|dkVqdSZDaskDistributedBackendNr,r-rxrr,r,r.r``sz4test_nested_parallelism_with_dask..css|]\}}|dkVqdSrr,rr,r,r.r`gs) distributedZClientrrrrrcrrr)riclientrr_rr,r,r.!test_nested_parallelism_with_daskTs  ( $rcCstddtdDS)z6A horrible function that does recursive parallel callscss|]}ttVqdSr:)r#_recursive_parallelr^r,r,r.r`mr0z&_recursive_parallel..r?rb)Z nesting_limitr,r,r.rksrc Cs||dd>tt}tWdn1s20YWdn1sP0Y|j}|dkrddlm}t||tfrt dt|t sJdS)Nr?rkr1r)TerminatedWorkerErrorz1Loky worker crash when serializing RecursionError) r BaseExceptionrrzZ&joblib.externals.loky.process_executorrrrrrRecursionError)rir8r excrr,r,r.test_thread_bomb_mitigationps  B  rcCs*i}dD]}tj|||<q|tdfS)N)OMP_NUM_THREADSOPENBLAS_NUM_THREADSMKL_NUM_THREADSZVECLIB_MAXIMUM_THREADSZNUMEXPR_NUM_THREADSZNUMBA_NUM_THREADS ENABLE_IPCrr)r)environrTr)Zenv_varsvarr,r,r._run_parallel_sumsrzNeed OpenMP helper compiledcCstd|dddtdD}ttdd}|D]Z\}}||ksFJ|D]<\}}|drr|t|ksJqN|dks~J|dksNJqNq2dS) Nr?r[css|]}ttVqdSr:)r#rrr,r,r.r`sz-test_parallel_thread_limit..rZ_THREADSr1)r"rcrr(itemsendswithr~)r8rZexpected_num_threadsZworker_env_varsZomp_num_threadsrxrzr,r,r.test_parallel_thread_limits    rz%This test requires dask NOT installedcCs8ttdd|dWdn1s*0YdS)NzPlease install daskrr)rrOrr,r,r.)test_dask_backend_when_dask_not_installedsrc CsGdddt}d}||Ztjt|d,tddddtdDWdn1s`0YWdn1s~0YdS) Nc@s&eZdZddZdddZddZdS) z3test_zero_worker_backend..ZeroWorkerBackendc_sdSrWr,)rCrrir,r,r.rsz=test_zero_worker_backend..ZeroWorkerBackend.configureNcSs tddS)NzNo worker availabler)rCfunccallbackr,r,r.rsz?test_zero_worker_backend..ZeroWorkerBackend.apply_asynccSsdSrWr,rr,r,r.r'szDtest_zero_worker_backend..ZeroWorkerBackend.effective_n_jobs)N)rJrKrLrrr'r,r,r,r.ZeroWorkerBackends rz&ZeroWorkerBackend has no active workerrr?rkcss|]}tt|VqdSr:rr^r,r,r.r`r0z+test_zero_worker_backend..)rrrrr"rc)rirZ expected_msgr,r,r.test_zero_worker_backends   rcsdadddksJtddfddtdD}t|dhksLJdadks^Jtddfd dtdD}t|dhksJdS) Nzoriginal valuecSstSr:)MY_GLOBAL_VARIABLEr,r,r,r. check_globalssz@test_globals_update_at_each_parallel_call..check_globalsr?rkc3s|]}tVqdSr:r8r^rr,r.r`sz.z changed valuec3s|]}tVqdSr:r8r^rr,r.r`s)rr"rcr5)Zworkers_global_variabler,rr.)test_globals_update_at_each_parallel_callsrcCs4ddl}|jdd}|||ddlm}|S)Nrrr)threadpool_info)numpyrYrZrVZ threadpoolctlr)rrDrr,r,r._check_numpy_threadpool_limitss   r cCs8|D] }|d|dkr|dSqtd|dS)Nfilepath num_threadsz,An unexpected module was loaded in child: {})rOrc) child_module parent_info parent_moduler,r,r._parent_max_num_threads_fors rcCs@|D]6}|D],}t||}t|||h}|d|vs Jq qdS)Nr )rmin)Z workers_inforr Zchild_threadpool_infor Zparent_max_num_threadsrr,r,r.check_child_num_threadssr)r?r}rorjcCsdt}t|dkrtjddtd|dddtdD}t|}tt|d }t |||dS) Nr&Need a version of numpy linked to BLASmsgr1)r8r\css|]}ttVqdSr:r#r r^r,r,r.r`sz;test_threadpool_limitation_in_child_loky..r?r r rrrar"rcr'rr(r)r\rworkers_threadpool_infosexpected_child_num_threadsr,r,r.(test_threadpool_limitation_in_child_lokys  rinner_max_num_threads)rr?r}Nr?cCst}t|dkrtjdd|d|d,t|dddtd D}Wdn1s\0Yt|}|durtt|d }n|}t |||dS) Nrrrr1rrkcss|]}ttVqdSr:rr^r,r,r.r`*sz>test_threadpool_limitation_in_child_context..r?rr)rir\rrrrr,r,r.+test_threadpool_limitation_in_child_contexts   &rvar_name)rrrc stddddtj}zdtj<t|dfddtd D}|ddgks`J|d d d 2t|dfd dtd D}Wdn1s0Y|ddgksJW|durtj=q|tj<n|durtj=n |tj<0dS)NTZreusecSs tj|Sr:)r)rrT)rr,r,r._get_envEsz>test_threadpool_limitation_in_child_override.._get_env4rkc3s|]}tVqdSr:r8r^r rr,r.r`Lsz?test_threadpool_limitation_in_child_override..r?r1rrc3s|]}tVqdSr:r8r^r"r,r.r`Qsr)rshutdownr)rrTr"rc)rir\rZoriginal_var_valuerr,r"r.,test_threadpool_limitation_in_child_override7s(   &   r$)r?r}rjcCsJdd}||tdd}tdD]"}||tdd}||ks"Jq"dS)Ncs,tdt|dfddtdDdS)Nrrkc3s|]}ttVqdSr:)r#sumr^r@r,r.r`dr0zAtest_loky_reuse_workers..parallel_call..rvrkr,r@r. parallel_callbsz.test_loky_reuse_workers..parallel_callTrr)rrc)r\r&Zfirst_executorrexecutorr,r,r.test_loky_reuse_workers\s   r()F)rr)ra)N)rMr)rrRrrrr2 tracebackrmathrrpicklerrFrr+r rrqr r r Zjoblib._multiprocessing_helpersrZjoblib.test.commonrrrrrZjoblib.testingrrrrrZjoblib.externals.lokyrrrposix ImportErrorZ _openmp_test_helper.parallel_sumrrZjoblib._parallel_backendsrrrr r!Zjoblib.parallelr"r#r$r%r&r'r(r)r*rZRETURN_GENERATOR_BACKENDSpopsortedkeysZALL_VALID_BACKENDSZPROCESS_BACKENDSZPARALLEL_BACKENDShasattrrr3rr9r=rA ExceptionrBrQrTrVrZrerfrhmarkrnrwrrrrrrrrrrrrrrrrrrrrrrrrr rrrrrrrr extendrcr!r"r&r(r+r-Z_parallel_backendsr.r6r:r>rBrIrKrRrUr\rbrfrnrlZ SQUARE_LOCALZ SQUARE_LAMBDArwrcrorprrrxr{r}r~rrrrrrrrrrrrrrrrrrrrrrrrrrZno_coverrrrrrrr rrrrr$r(r,r,r,r.s                                                D                                    ( ,                ,+ . 6          !