
     h                     

   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dl	Z	d dl
Z
d dlZd dlmZmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZmZmZmZmZ d d	lmZ d d
lmZmZ d dlmZ d dl m!Z" d dl#m$Z$ d dl#m%Z% d dl#m&Z& d dl#m'Z' d dl#m(Z( d dl#m)Z) d dl#m*Z+ d Z,d Z-d Z.d Z/d Z0eed                         Z1e eej        dk    pdd          d                         Z2eed                         Z3ed             Z4ee ed ee"j5        gd!d"g#          d$                                     Z6ee ed ee"j5        gd!d"g#          d%                                     Z7ee ed&d!d"g          d'                                     Z8ee ed&d!d"g          d(                                     Z9ee ed&d!d"g          d)                                     Z:ee ed&d!d"g          d*                                     Z;eed+                         Z<eed,                         Z=ee ed&d!d"g          d-                                     Z>ee ed&d!d"g          d.                                     Z?ee ed&d!d"g          d/                                     Z@ee ed ee"j5        gd!d"g#          d0                                     ZAee ed& ej        d!ejB                            d1          2          d"g          d3                                     ZCee ed ee"j5        gd!d"g#          d4                                     ZDeee ed ee"j5        gd!d"g#          d5                                                 ZEeee ed ee"j5        gd!d"g#          d6                                                 ZFee ed ee"j5        gd!d"g#          d7                                     ZGd8 ZHee ed ee"j5        gd!d"g#          d9                                     ZId: ZJee ed; eK ejL        ee"j5        g eMd<                              d=  ejL        d!d"g eNeO eMd<                              D             #          d>                                     ZPd? ZQe eej        dk    d@           edAdBdCg          dD                                     ZRedE             ZSdF ZTeedG                         ZUdS )H    N)sleep)
with_numpynp)setup_autokill)teardown_autokill)with_multiprocessing)with_dev_shm)raisesparametrizeskipifxfailparam)make_memmap)Paralleldelayed)MemmappingPool)_TestingMemmappingExecutorhas_shareable_memory)ArrayMemmapForwardReducer)_strided_from_memmap)_get_temp_dir)_WeakArrayKeyMap_get_backing_memmapc                  2    t          t          d           d S )Ni,  )timeout)r   __name__     W/var/www/html/Sam_Eipo/venv/lib/python3.11/site-packages/joblib/test/test_memmapping.pysetup_moduler"   "   s    8S))))))r    c                  .    t          t                     d S N)r   r   r   r    r!   teardown_moduler%   &   s    hr    c                 (    t          |           J | S r$   r   )arrays    r!   check_memmap_and_send_backr(   *   s    u%%111Lr    c                 ^    | \  }}}t           j                            ||         |           dS )zDummy helper function to be executed in subprocesses

    Check that the provided array has the expected values in the provided
    range.

    Nr   testingassert_array_equalargsdatapositionexpecteds       r!   check_arrayr2   /   s2      $D(HJ!!$x.(;;;;;r    c                     | \  }}}||         |k    sJ ||xx         dz  cc<   t           j                            ||         d|z             dS )zDummy helper function to be executed in subprocesses


    Check that the input array has the right values in the provided range
    and perform an inplace modification to double the values in the range by
    two.

       Nr*   r-   s       r!   inplace_doubler5   :   sa      $D(H>X%%%%NNNaNNNJ!!$x.!h,?????r    c                    t           j        j        }|                     d          j        }t          j        |t           j        dd          }dt          j        |j        d         |j	                  z  |dd<   |
                                 t          j        |t           j        d	d
dd          }t          j        d                              |j                  |dd<   |ddddddf         }t          j        |          }|j        }t          d| j        dd          fd} ||          }	t          |	          sJ t!          |	t           j                  sJ  ||	|            ||          }
t          |
          sJ  ||
|            ||          }t!          |t           j                  rJ t          |          sJ  |||            ||          }t!          |t           j                  rJ t          |          sJ  |||           |dz  }t          |          rJ  ||          }t          |          rJ t!          |t           j                  rJ  |||dz             t          j        |          }t          |          rJ  ||          }t!          |t           j                  sJ t          |          rJ  |||           dS )z9Check that it is possible to reduce a memmap backed array	test.mmapi  w+dtypeshapemodeg      r   r:   N)         r+Fr@   )r:   r;   r<   orderoffset<      r4   cTc                 *     |           \  }} || S r$   r   xconsr.   reducers      r!   reconstruct_array_or_memmapzEtest_memmap_based_array_reducing.<locals>.reconstruct_array_or_memmapj        WQZZ
dtT{r    r>   )r   r+   r,   joinstrpathmemmapfloat64aranger;   r:   flushreshapeasarrayTr   r   
isinstancendarray)tmpdirr,   filenamebufferabrH   drN   a_reconstructedb_reconstructedc_reconstructedd_reconstructeda3a3_reconstructedb3b3_reconstructedrM   s                    @r!    test_memmap_based_array_reducingri   I   sD    6{{;''/H Yxrz4HHHF 	&,q/FFFFF111I
LLNNN 		("*I3q	2 	2 	2A9R==  ))AaaaD
 	
!B$"ac/A 	
1A	A (fnc4HHG    
 21!44O00000ory11111*** 21!44O00000*** 21!44O/295555500000***11!44O/295555500000*** 
QB#B'''''22266#$455555*BI66666'Q/// 
BB#B'''''22266&
33333#$455555',,,,,r    win32r   z2PermissionError only easily triggerable on Windows)reasonc                    |                      d          j        }d                    |          }t          j        t
          j        d|gt          j        t          j                  }|                                 |	                                \  }}|j
        dk    sJ |dk    sJ d                    |          }||                                v sJ d S )	Nr7   a  if 1:
    import os
    import numpy as np
    import time
    from joblib.externals.loky.backend import resource_tracker
    resource_tracker.VERBOSE = 1

    # Start the resource tracker
    resource_tracker.ensure_running()
    time.sleep(1)

    # Create a file containing numpy data
    memmap = np.memmap(r"{filename}", dtype=np.float64, shape=10, mode='w+')
    memmap[:] = np.arange(10).astype(np.int8).data
    memmap.flush()
    assert os.path.exists(r"{filename}")
    del memmap

    # Create a np.memmap backed by this file
    memmap = np.memmap(r"{filename}", dtype=np.float64, shape=10, mode='w+')
    resource_tracker.register(r"{filename}", "file")

    # Ask the resource_tracker to delete the file backing the np.memmap , this
    # should raise PermissionError that the resource_tracker will log.
    resource_tracker.maybe_unlink(r"{filename}", "file")

    # Wait for the resource_tracker to process the maybe_unlink before cleaning
    # up the memmap
    time.sleep(2)
    )r\   -cstderrstdoutr   r    z'tried to unlink {}, got PermissionError)rP   rQ   format
subprocessPopensys
executablePIPEwaitcommunicate
returncodedecode)r[   r\   cmdpouterrmsgs          r!   2test_resource_tracker_retries_when_permissionerrorr      s     {{;''/H: 	!!; < 	#.$4Z_ *	1 	1 	1AFFHHH}}HC<1#::::
3
:
:8
D
DC#**,,r    c                 n   t           j        j        }|                     d          j        }t          j        |t           j        dd          }t          j        d                              |j	                  |d d <   |dd         }|d d ddf         }|d d d d d d df         }|d	d
d         }t          d | j        dd          fd} ||          }	t          |	          sJ t          |	t           j                  sJ  ||	|            ||          }
t          |
          sJ  ||
|            ||          }t          |          sJ  |||            ||          }t          |          sJ  |||            ||          }t          |          sJ  |||           d S )Nr7   )d      r   r>   r8   r9   i r   
   r?   rF   r>   r@   rH   Tc                 *     |           \  }} || S r$   r   rJ   s      r!   rN   zNtest_high_dimension_memmap_array_reducing.<locals>.reconstruct_array_or_memmap   rO   r    )r   r+   r,   rP   rQ   rR   rS   rT   rV   r;   r   r   rY   )r[   r,   r\   r^   r_   rH   r`   erN   ra   rb   rc   rd   e_reconstructedrM   s                 @r!   )test_high_dimension_memmap_array_reducingr      s#    6{{;''/H 		("*4D	 	 	A9&''//88AaaaD 	
!B$A	!!!QrT'
A	!!!QQQ1*A	!Aa%A (fnc4HHG     21!44O00000ory11111***11!44O00000***11!44O00000***11!44O00000***11!44O00000*****r    c                    |                      d          j        }dt          j        z  }t          j        dz   }t	          j        |d||z             }t          |dd|d|d d d	
	  	        }t          |t          j                  sJ |j        |k    sJ t          |dd|d|dz  fd|d	
	  	        }t          |          j        |k    sJ d S )Nr7   r?   rF   r8   )r<   r;   uint8rCF)r:   r<   rD   rC   r;   stridestotal_buffer_lenunlink_on_gc_collectr4   )r4   )
rP   rQ   mmapALLOCATIONGRANULARITYr   rR   r   rY   rD   r   )r[   fnamesizerD   
memmap_objmemmap_backed_objs         r!   test__strided_from_memmapr      s    KK$$,Et))D'!+F5t4&=AAAJ%e7-33d.2T;@B B BJ j"),,,,,&&&&,W3vSqylD4"  
 0118FBBBBBBr    factorymultiprocessingloky)idsc           	         t           j        j        }|                    d          j        } | dd|          }	 |                    d          j        }t          j        |t           j        dd                              d	           |	                    t          fd
t          j        d                   D                         |dt          j        j                  z             t          j        |t           j        dd          |	                    t          fdt          j        d                   D                        t          j        |          g k    sJ  |dt          j        j                  z              |dt          j        j                  z             t          j        |t           j        ddd          t!          t"                    5  |	                    t$          fdt          j        d                   D                        ddd           n# 1 swxY w Y   t!          t&          t(          f          5  |	                    t          fdt          j        d                   D                        ddd           n# 1 swxY w Y   |                                 ~dS # |                                 ~w xY w)z@Check that subprocess can access and update shared memory memmappoolr   r4   
max_nbytestemp_folderr7   r>   r?   r8   r9         ?c                 X    g | ]&}t          j        d                    D ]	}||fdf
'S rF   r   ranger;   ).0ijr^   s      r!   
<listcomp>z)test_pool_with_memmap.<locals>.<listcomp>  W     ; ; ;#$(-agaj(9(9; ;#$ !"Aq63/ ; ; ; ;r    r   )r?   r>   rH   c                 X    g | ]&}t          j        d                    D ]	}||fdf
'S )rF          @r   )r   r   r   r_   s      r!   r   z)test_pool_with_memmap.<locals>.<listcomp>$  r   r    r   r      )r:   r;   r<   rD   c                     g | ]}|d fS )g      @r   r   r   rH   s     r!   r   z)test_pool_with_memmap.<locals>.<listcomp>5  s    GGGAsGGGr    Nc                     g | ]}|d fS )r   r   r   s     r!   r   z)test_pool_with_memmap.<locals>.<listcomp>:  s    "J"J"J1Aq#;"J"J"Jr    )r   r+   r,   mkdirrQ   rP   rR   float32fillmapr5   r   r;   onesoslistdirr
   AssertionErrorr2   RuntimeError
ValueError	terminate)	r   r[   r,   pool_temp_folderr|   r\   r^   r_   rH   s	         @@@r!   test_pool_with_memmapr     sy    6 ||F++3q.>???A(;;{++3IhbjTJJJ	s	n ; ; ; ;(-agaj(9(9; ; ; 	< 	< 	< 	1a"'!'"2"22333 IhbjSIII	n ; ; ; ;(-agaj(9(9; ; ; 	< 	< 	< z*++r1111 	1a"'!'"2"223331a"'!'"2"22333 IhbjC"$ $ $ N## 	I 	IEE+GGGGU171:5F5FGGGHHH	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I
 \:.// 	L 	LEE."J"J"J"Jagaj8I8I"J"J"JKKK	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	
AA 	
sV   F4K 2A H>2K >IK IK $A J0$K 0J44K 7J48K K)c                 N   t           j        j        }|                    d          j        } | dd|          }	 |                    d          j        }t          j        |t           j        dd                              d	           t          j	                  t          t           j                  rJ t                    sJ |                    t          fd
t          j        d                   D                         |dt          j        j                  z              |dt          j        j                  z             t#          j        |          g k    sJ 	 |                                 ~dS # |                                 ~w xY w)z?Check that subprocess can access and update shared memory arrayr   r   r4   r   r7   r   r8   r9   r   c                 X    g | ]&}t          j        d                    D ]	}||fdf
'S r   r   )r   r   r   r^   a_views      r!   r   z4test_pool_with_memmap_array_view.<locals>.<listcomp>W  sW     ; ; ;#$(-agaj(9(9; ;#$ !'A4 ; ; ; ;r    r   N)r   r+   r,   r   rQ   rP   rR   r   r   rW   rY   r   r   r5   r   r;   r   r   r   r   )r   r[   r,   r   r|   r\   r^   r   s         @@r!    test_pool_with_memmap_array_viewr   A  s    6 ||F++3q.>???A;;{++3IhbjTJJJ	s Afbi00000#F+++++	n ; ; ; ; ;(-agaj(9(9; ; ; 	< 	< 	<
 	1a"'!'"2"2233361rwqw'7'7#7888 z*++r11111 	
AA 	
s   D8F F$backendc                 v   d                     |           }t          j        t          j        d|gt          j        t          j                  }|                                 |                                \  }}|j        dk    s4J |	                                dz   |	                                z               d S )N  if 1:
        import numpy as np
        from joblib import Parallel, delayed


        data = np.random.rand(int(2e6)).reshape((int(1e6), 2))

        # Build a complex cyclic reference that is likely to delay garbage
        # collection of the memmapped array in the worker processes.
        first_list = current_list = [data]
        for i in range(10):
            current_list = [current_list]
        first_list.append(current_list)

        if __name__ == "__main__":
            results = Parallel(n_jobs=2, backend="{b}")(
                delayed(len)(current_list) for i in range(10))
            assert results == [1] * 10
    r_   rm   rn   r   

)
rq   rr   rs   rt   ru   rv   rw   rx   ry   rz   r   r{   r|   r}   r~   s        r!   -test_permission_error_windows_reference_cycler   h  s    $ 	% & 	#.$4Z_ *	1 	1 	1AFFHHH}}HC<1cjjllV3cjjllBr    c                    d                     |           }t          d          D ]}t          j                                        }t          j                            t                    |d<   t          j	        t          j        d|gt          j        t          j        |          }|                                 |                                \  }}|j        dk    s
J |            |dk    sJ t          j        d d         d	vrd
|vsJ d S )Nak  if 1:
        import os
        import time

        import numpy as np

        from joblib import Parallel, delayed
        from testutils import return_slice_of_data

        data = np.ones(int(2e6))

        if __name__ == '__main__':
            # warm-up call to launch the workers and start the resource_tracker
            _ = Parallel(n_jobs=2, verbose=5, backend='{b}')(
                delayed(id)(i) for i in range(20))

            time.sleep(0.5)

            slice_of_data = Parallel(n_jobs=2, verbose=5, backend='{b}')(
                delayed(return_slice_of_data)(data, 0, 20) for _ in range(10))
    r   r>   
PYTHONPATHrm   ro   rp   envr   r    )r>      r   )r>   r   rF      resource_tracker)rq   r   r   environcopypathdirname__file__rr   rs   rt   ru   rv   rw   rx   ry   version_info)r   r{   _r   r|   r}   r~   s          r!   3test_permission_error_windows_memmap_sent_to_parentr     s	   ( 	) , 1XX 2 2jooGOOH55LcndC8$.O$.O> > > 	
==??S|q   #   czzzzBQB'===
 'c11112 2r    c                    t          j        t          d                     t          d| d          fdt	          d          D                       \  } t          d| d          fdt	          d          D                       \  }t
          j                            |          t
          j                            |          k    sJ d S )N      Y@r4   r   n_jobsr   r   c              3   T   K   | ]"} t          t                    d           V  #dS r\   Nr   getattrr   r   r'   s     r!   	<genexpr>z6test_parallel_isolated_temp_folders.<locals>.<genexpr>  Q       F F01
++F F F F F Fr    rF   c              3   T   K   | ]"} t          t                    d           V  #dS r   r   r   s     r!   r   z6test_parallel_isolated_temp_folders.<locals>.<genexpr>  r   r    r   rT   intr   r   r   r   r   )r   
filename_1
filename_2r'   s      @r!   #test_parallel_isolated_temp_foldersr     s     Ic#hhEE81g"EEE F F F F5:1XXF F F  LZ F81g"EEE F F F F5:1XXF F F  LZ 7??:&&"'//**E*EEEEEEEr    c                    t          j        t          d                    t          d| d          5 } |fdt	          d          D                       \  } |fdt	          d          D                       \  }d d d            n# 1 swxY w Y   t
          j                            |          t
          j                            |          k    sJ d S )Nr   r4   r   r   c              3   T   K   | ]"} t          t                    d           V  #dS r   r   r   s     r!   r   z9test_managed_backend_reuse_temp_folder.<locals>.<genexpr>  I       
 
45GGUJ//
 
 
 
 
 
r    rF   c              3   T   K   | ]"} t          t                    d           V  #dS r   r   r   s     r!   r   z9test_managed_backend_reuse_temp_folder.<locals>.<genexpr>  r   r    r   )r   r|   r   r   r'   s       @r!   &test_managed_backend_reuse_temp_folderr     s6   
 Ic#hhE	G	;	;	; 
qq 
 
 
 
9>q
 
 
 
 
 q 
 
 
 
9>q
 
 
 
 
	
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 7??:&&"'//**E*EEEEEEEs   ABBBc                     t          j        t          d                    } t                      }t                      }d }t	          j        || |f          }t	          j        || |f          }|                                 |                                 |                                 |                                 t          |          dk    sJ t          |          dk    sJ ||k    sJ d S )Nr   c                 4    t          ddd          5 }t          d          D ]Z} | fdt          d          D                       \  }|                    t          j                            |                     [	 d d d            d S # 1 swxY w Y   d S )Nr   r4   r   )r   r   r   c              3   T   K   | ]"} t          t                    d           V  #dS r   r   r   s     r!   r   z]test_memmapping_temp_folder_thread_safety.<locals>.concurrent_get_filename.<locals>.<genexpr>  sI        <=$GG$$UJ77     r    rF   )r   r   addr   r   r   )r'   	temp_dirsr|   r   r\   s   `    r!   concurrent_get_filenamezJtest_memmapping_temp_folder_thread_safety.<locals>.concurrent_get_filename  s    fQ2>>> 	9!2YY 9 9Q    AFq    
 bgooh778888	9	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9s   A+BBB)targetr.   rF   )	r   rT   r   set	threadingThreadstartrP   len)r'   temp_dirs_thread_1temp_dirs_thread_2r   t1t2s         r!   )test_memmapping_temp_folder_thread_safetyr     s    Ic#hhE9 9 9 
	&e5G-H
 
 
B 
	&e5G-H
 
 
B HHJJJHHJJJGGIIIGGIII!""a''''!""a''''!3333333r    c            	         d} g d}|D ]\  }}}t          j        t          j        d|                     ||          gt           j        t           j                  }|                                 |                                \  }}|j        |k    sJ |	                                            d|vsJ |	                                            d S )Na>  if 1:
        import os
        import numpy as np
        from joblib import Parallel, delayed
        from joblib.externals.loky.backend import resource_tracker
        from concurrent.futures import ThreadPoolExecutor, wait

        resource_tracker.VERBOSE = 0

        array = np.arange(int(1e2))

        temp_dirs_thread_1 = set()
        temp_dirs_thread_2 = set()


        def raise_error(array):
            raise ValueError


        def parallel_get_filename(array, temp_dirs):
            with Parallel(backend="loky", n_jobs=2, max_nbytes=10) as p:
                for i in range(10):
                    [filename] = p(
                        delayed(getattr)(array, "filename") for _ in range(1)
                    )
                    temp_dirs.add(os.path.dirname(filename))


        def parallel_raise(array, temp_dirs):
            with Parallel(backend="loky", n_jobs=2, max_nbytes=10) as p:
                for i in range(10):
                    [filename] = p(
                        delayed(raise_error)(array) for _ in range(1)
                    )
                    temp_dirs.add(os.path.dirname(filename))


        executor = ThreadPoolExecutor(max_workers=2)

        # both function calls will use the same loky executor, but with a
        # different Parallel object.
        future_1 = executor.submit({f1}, array, temp_dirs_thread_1)
        future_2 = executor.submit({f2}, array, temp_dirs_thread_2)

        # Wait for both threads to terminate their backend
        wait([future_1, future_2])

        future_1.result()
        future_2.result()
    ))parallel_get_filenamer  r   )r  parallel_raiserF   )r  r  rF   rm   )f1f2rn   r   )
rr   rs   rt   ru   rq   rv   rw   rx   ry   rz   )r{   functions_and_returncodesr  r  ry   r|   r}   r~   s           r!   ?test_multithreaded_parallel_termination_resource_tracker_silentr    s    1Cd! ! ! 8 < <B
cndCJJ"J4L4LM$.OJOM M M	==??S|z)))3::<<)))"#---szz||----< <r    c                    d                     |           }t          j                                        }t          j                            t                    |d<   t          j        t          j
        d|gt          j        t          j        |          }|                                 |                                \  }}|j        dk    s
J |            |dk    sJ t          j        d d         d	vrd
|vsJ d S d S )Na  if 1:
        import os
        import time

        import numpy as np

        from joblib import Parallel, delayed
        from testutils import return_slice_of_data

        data = np.ones(100)

        if __name__ == '__main__':
            for i in range(5):
                slice_of_data = Parallel(
                    n_jobs=2, max_nbytes=1, backend='{b}')(
                        delayed(return_slice_of_data)(data, 0, 20)
                        for _ in range(10)
                    )
    r   r   rm   r   r   r    r>   r   r   )rq   r   r   r   r   r   r   rr   rs   rt   ru   rv   rw   rx   ry   r   )r   r{   r   r|   r}   r~   s         r!   'test_many_parallel_calls_on_same_objectr  M  s    $ 	% & *//

C11C	s#		 	 	A FFHHH}}HC<1c#::::
#999
 ##---- :9
 .-r    c                     t          j        t          d                     t          d| d          fdt	          d          D                       \  }t          |          J d S )Ng     @@r4   r   r   c              3   R   K   | ]!} t          t                              V  "d S r$   )r   r(   )r   r   r/   s     r!   r   z8test_memmap_returned_as_regular_array.<locals>.<genexpr>  sX       CE CE67+*++D11CE CE CE CE CE CEr    rF   )r   r   r   r   r   r   )r   resultr/   s     @r!   %test_memmap_returned_as_regular_arrayr  }  s     73s88DBxq'cBBB CE CE CE CE;@88CE CE CE E EHVv&&.....r    c                    | dk    r3t           j                            d          rt          j        d           d                    |           }t          j        t           j        d|gt          j	        t          j	                  }|
                                 |                                \  }}|                                }|                                }|j        dk    sJ |d	z   |z               d
|vs
J |            d S )Nr   winzVThe temporary folder cannot be deleted on Windows in the presence of a reference cycler   r   rm   rn   r   r   resource_tracker)rt   platform
startswithpytestr   rq   rr   rs   ru   rv   rw   rx   rz   ry   r   s        r!   2test_resource_tracker_silent_when_reference_cyclesr    s   " &S\44U;; 	,	
 	
 	

$ 	% & 	#.$4Z_ *	1 	1 	1AFFHHH}}HC
**,,C
**,,C<1cFlS0S(((#(((((r    c                    t          j        |j                  g k    sJ  | dd|j        d          }	 t          j        |j                  g k    sJ t           j                            |j                  rJ t          j        dt          j                  j	        dk    sJ |
                    t          fdt          j        d	                   D                        t          j        |j                  g k    sJ t          j        d
t          j                  j	        dk    sJ |
                    t          fdt          j        d	                   D                        t           j                            |j                  sJ t          j        |j                  }t!          |          dk    sJ t          j        dgd
z  d          }|
                    t$          |g          }|d	         rJ 	 |                                 t          d          D ]7}t)          d           t           j                            |j                  s n#8t+          d                    |                    ~dS # |                                 t          d          D ]7}t)          d           t           j                            |j                  s n#8t+          d                    |                    ~w xY w)z0Check that large arrays are not copied in memoryr>   (   r4   )r   r   verboser?   r=   r   c                     g | ]}|d fS r   r   )r   r   smalls     r!   r   z9test_memmapping_pool_for_large_arrays.<locals>.<listcomp>      KKKUAsOKKKr    r   r      c                     g | ]}|d fS r  r   r   r   larges     r!   r   z9test_memmapping_pool_for_large_arrays.<locals>.<listcomp>  r  r    rF   abcobjectr   皙?z&temporary folder of {} was not deletedN)r   r   rQ   r   exists_temp_folderr   r   r   nbytesr   r2   r   r;   rS   isdirr   r'   r   r   r   r   rq   )	r   r[   r|   dumped_filenamesobjectsresultsr   r  r  s	          @@r!   %test_memmapping_pool_for_large_arraysr)    s    :fn%%++++ 	bfnaHHHA(z&.))R////7>>!.11111,,,|r!!!!	kKKKKU5;q>5J5JKKKLLL z&.))R//// 2:...|s""""	kKKKKU5;q>5J5JKKKLLL w}}Q^,,,,,:an55#$$)))) (E7S=999%%,wi881:~ 	
r 	 	A"III7>>!.11  !8??BB   AA 	
r 	 	A"III7>>!.11  !8??BB   s   GJ B Lz,https://github.com/joblib/joblib/issues/1086)marksc                    d                     |           }t          j                                        }t          j                            t                    |d<   t          j        t          j
        d|gt          j        t          j        |          }|                                 |                                \  }}|                                |                                }}|                    d          d         }|j        dk    sJ |p|            |dk    sJ t          j                            |          rJ d S )	Na  if 1:
        import os
        from pathlib import Path
        from time import sleep

        import numpy as np
        from joblib import Parallel, delayed
        from testutils import print_filename_and_raise

        data = np.random.rand(1000)

        def get_temp_folder(parallel_obj, backend):
            if "{b}" == "loky":
                return Path(p._backend._workers._temp_folder)
            else:
                return Path(p._backend._pool._temp_folder)


        if __name__ == "__main__":
            try:
                with Parallel(n_jobs=2, backend="{b}", max_nbytes=100) as p:
                    temp_folder = get_temp_folder(p, "{b}")
                    p(delayed(print_filename_and_raise)(data)
                              for i in range(1))
            except ValueError as e:
                # the temporary folder should be deleted by the end of this
                # call but apparently on some file systems, this takes
                # some time to be visible.
                #
                # We attempt to write into the temporary folder to test for
                # its existence and we wait for a maximum of 10 seconds.
                for i in range(100):
                    try:
                        with open(temp_folder / "some_file.txt", "w") as f:
                            f.write("some content")
                    except FileNotFoundError:
                        # temp_folder has been deleted, all is fine
                        break

                    # ... else, wait a bit and try again
                    sleep(.1)
                else:
                    raise AssertionError(
                        str(temp_folder) + " was not deleted"
                    ) from e
    r   r   rm   r   
r    )rq   r   r   r   r   r   r   rr   rs   rt   ru   rv   rw   rx   rz   splitry   r"  )r   r{   r   r|   r}   r~   r\   s          r!   &test_child_raises_parent_exits_cleanlyr/    s   :-Z 	[ \ *//

C11C#.$4Z_ *S	: 	: 	:AFFHHH}}HCzz||SZZ\\Cyyq!H<1cjS"9999w~~h'''''''r    c                     | dd|j                   }	 t          j        |j                   g k    sJ t          j        dt          j                  j        dk    sJ |                    t          fdt          j
        d                   D                        t          j        |j                   g k    sJ 	 |                                 ~dS # |                                 ~w xY w)	z2Check that large arrays memmapping can be disabledr>   Nr   r   r=   r  c                     g | ]}|d fS r  r   r  s     r!   r   zBtest_memmapping_pool_for_large_arrays_disabled.<locals>.<listcomp>_  r  r    r   )rQ   r   r   r   r   rS   r$  r   r2   r   r;   r   )r   r[   r|   r  s      @r!   .test_memmapping_pool_for_large_arrays_disabledr2  O  s     	d???A z&.))R//// 2:...|s""""	kKKKKU5;q>5J5JKKKLLL z&.))R///// 	
AA 	
s   B*C C/c                    t           j        }	 t          d          t           _         | dd          }	 |j        }d}|                    |          sJ t
          j                            |          sJ t          j	        dt          j
                  }|j        dk    sJ |                    t          |gdz             t          t          j        |                    d	k    sJ t          j	        dt          j
                  d
z  }|j        dk    sJ |                    t          |gdz             t          t          j        |                    d
k    sJ 	 |                                 ~n# |                                 ~w xY wt#          d          D ]2}t
          j                            |          s nt%          d           3t'          d          	 |t           _        dS # |t           _        w xY w)z1Check that memmapping uses /dev/shm when possibleg    ~Ar>   r   r   z"/dev/shm/joblib_memmapping_folder_r   r=   r  rF   r4   r!  z(temporary folder of pool was not deletedN)jmrSYSTEM_SHARED_MEM_FS_MIN_SIZEr   r#  r  r   r   r"  r   r   rS   r$  r   idr   r   r   r   r   r   )r   	orig_sizer|   r   folder_prefixr^   r_   r   s           r!   'test_memmapping_on_large_enough_dev_shmr:  j  s    1I+6 -0II)GA"%%%	  !~@M#..}=====7>>"233333 2:...A8s????EE"qcBh rz"233449999
 2:...2A8s????EE"qcBhrz"2334499999 KKMMM KKMMMEEEEs 	M 	MA7>>"233 "IIII !KLLL 
 -6)))I)5555s$   &G0 D*E6  G0 6FAG0 0G>c                    t           j        }	 t          d          t           _         | dd          }	 |j        }|                    d          rJ 	 |                                 ~n# |                                 ~w xY wt          j                            |          rJ 	 |t           _        d S # |t           _        w xY w)Ng@6Dr>   r   r4  z/dev/shm)	r5  r6  r   r#  r  r   r   r   r"  )r   r8  r|   r   s       r!   $test_memmapping_on_too_small_dev_shmr<    s     1I6 -0JJ)GA"%%%	  !~'22:>>>>>> KKMMM KKMMMEEEE 7>>"2333333,5)))I)5555s"   &B4 A* B4 *B$B4 4Cc                 v   t           j        j        } | dd|j                  }	 |                    t           j        d          }|                                }t          |          rJ  ||t          j        d                     |                                 ~dS # |                                 ~w xY w)z:Check that large arrays are not copied in memory in returnr>   r   r   )  r.   r>  N)	r   r+   r,   rQ   apply_asyncr   getr   r   )r   r[   r,   r|   resr  s         r!   /test_memmapping_pool_for_large_arrays_in_returnrC    s     6 	bfn===AmmBG'm22		'.....5"'$--000	AA 	
s   A$B! !B8c                 .    t          |           sJ | |z  S )z4Multiplication function to be executed by subprocessr   )r^   n_timess     r!   _worker_multiplyrF    s     """""w;r    c                    t           j        j        } | dd|j                  }	 t          j        t          j        d                              d          d          dddd	ddf         }|                    t          |df
          	                                }t          |          rJ  ||d|z             |                                 ~dS # |                                 ~w xY w)zCheck that memmaps with a bad buffer are returned as regular arrays

    Unary operations and ufuncs on memmap instances return a new memmap
    instance with an in-memory buffer (probably a numpy bug).
    r>   r   r   ip  )r>  r4   r>   rB   )rC   NrF   r?  )r   r+   r,   rQ   rW   rT   rV   r@  rF  rA  r   r   )r   r[   r,   r|   r^   r_   s         r!   6test_workaround_against_bad_memmap_with_copied_buffersrH    s     6bfn===A Jry..|<< " " ""#!!RaR(,
 MM*!QM88<<>>'*****1a!e$$$	AA 	
s   BC C.c                     | S r$   r   )args    r!   identityrK    s    Jr    zfactory,retry_nor>   c                 @    g | ]\  }}d                      ||          S )z{}, {})rq   )r   rK   ys      r!   r   r     s6     	: 	: 	:41aA		 	: 	: 	:r    c                    |                     d          j        }dt          j        z  }t          j        dz   }t	          |d|d|          } | d|j                  }|                    t          |f	                                          }t          |t          j
                  sJ |j        |k    sJ t          j                            ||           |                                 d S )
Nr7   r?   rF   r8   r   )r<   r;   r:   rD   r4   r   r?  )rP   rQ   r   r   r   r@  rK  rA  rY   r   rR   rD   r+   r,   r   )	r   retry_nor[   r   r   rD   objr|   r  s	            r!    test_pool_memmap_with_big_offsetrR    s     KK$$,Et))D'!+F
e$d'#% % %C 	v~...A]]83&]115577Ffbi(((((=F""""J!!#v...KKMMMMMr    c                 &   d}t          || j                  \  }}|du sJ ||                     d          j        k    sJ t          |d           \  }}t          j                            d          r|du sJ |                    |          sJ d S )Nztest.tmpdirFrO  r  )r   rQ   rP   rt   r  r  endswith)r[   pool_folder_namepool_folder
shared_mems       r!   test_pool_get_temp_dirrX    s    $+,<fnMMK&++m44<<<<<+,<$OOOK
|u%% #U"""" 01111111r    z1This test fails with a PermissionError on Windows	mmap_moderA   r8   c                    d d t          d          D             } t          | dd          fdt          |          D                       }t          |          D ]%\  }}t          j                            ||           &d S )Nc                     || d d <   | S r$   r   )arrvalues     r!   funcz4test_numpy_arrays_use_different_memory.<locals>.func*  s    AAA
r    c                 :    g | ]}t          j        d d          S ))r   r   rS   r=   )r   zeros)r   r   s     r!   r   z:test_numpy_arrays_use_different_memory.<locals>.<listcomp>.  s'    EEEabhxy111EEEr    r   r   r4   )rY  r   r   c              3   P   K   | ] \  }} t                    ||          V  !d S r$   r   )r   r   r\  r^  s      r!   r   z9test_numpy_arrays_use_different_memory.<locals>.<genexpr>0  sR       D? D?"(!Sc1D? D? D? D? D? D?r    )r   r   	enumerater   r+   r,   )rY  arraysr(  r   r\  r^  s        @r!   &test_numpy_arrays_use_different_memoryre  %  s    
   FE599EEEFChqCCC D? D? D? D?,5f,=,=D? D? D? ? ?G G$$ . .3

%%c1----. .r    c                  B   dd} t          j        d          }t                                          |d                               |          dk    sJ |}                    |          dk    sJ                     |d                               |          dk    sJ ~t          j                     t          j                  dk    sJ                     |          dk    sJ ~ | j                   t          j        d          }                    |d           t          j                  dk    sJ                     |          dk    sJ t          t                    5                      t          j        d                     d d d            n# 1 swxY w Y   ~ | j                   d t          fd	t          d
          D                       }t          j                    dk    rt          |          dk     sJ d S d S )Nr   c                     t          |          D ]:}t          |           dk    r d S t          j                     t	          d           ;t          |           dk    sJ d S )Nr   r!  )r   r   gccollectr   )	containerretriesr   s      r!   assert_empty_after_gc_collectz>test_weak_array_key_map.<locals>.assert_empty_after_gc_collect:  sd    w 	 	A9~~""JLLL"IIII9~~""""""r    *   r^   r_   rF   rH   c                 ,   t          j        d          }t          t                    5  |                     |           d d d            n# 1 swxY w Y   |                     ||           |                     |          |k    sJ t          |          S )Nrm  )r   r   r
   KeyErrorrA  r   r7  )mr   r^   s      r!   get_set_get_collectz4test_weak_array_key_map.<locals>.get_set_get_collecta  s    GBKKH 	 	EE!HHH	 	 	 	 	 	 	 	 	 	 	 	 	 	 		auuQxx1}}}}!uus   AAAc                 (    g | ]} |          S r   r   )r   r   rq  rp  s     r!   r   z+test_weak_array_key_map.<locals>.<listcomp>i  s'    EEEA))!Q//EEEr    r>  CPython)r   )r   r   r   r   rA  rh  ri  r   _datar
   ro  r   r  python_implementation)rl  r^   r_   rH   
unique_idsrq  rp  s        @@r!   test_weak_array_key_maprw  7  sN   # # # # 	AAEE!SMMM5588s????	A5588s????EE!SMMM5588s????	JLLLqw<<15588s????	!!!'***
AEE!SMMMqw<<15588s????			  	bgbkk               	
!!!'***   EEEEEtEEEFFJ%''944
 :$$$$ 54
 %$s   5(F))F-0F-c                      t                      } t          t          j                  5  t          j        |            d d d            d S # 1 swxY w Y   d S r$   )r   r
   picklePicklingErrordumps)rp  s    r!   #test_weak_array_key_map_no_picklingr|  r  s    A	$	%	%  Q                 s   A

AAc                 ^   t          |                     d                    t          j        dd          }|                               fdd              } t          d          fd	|fD                       } t          d          fd
|fD                       }t          j                            ||           fd t          d          fdt          d          D                       }t          j                            |d         |           d S )Nzarr.datr   r   r=   c                      t                    5 } t          j        |                                 dt          j        d          }d d d            n# 1 swxY w Y   t	          j        dt          j        |d          S )Nr   )accessrD   r   )r:   r]   rD   )openr   filenoACCESS_READr   rZ   r   )fdmmtestfiles     r!   _read_arrayz%test_direct_mmap.<locals>._read_array  s    (^^ 	Nr299;;$2B1MMMB	N 	N 	N 	N 	N 	N 	N 	N 	N 	N 	N 	N 	N 	N 	Nz%rx1EEEEs   5AAAc                     | dz  S )Nr4   r   )rK   s    r!   r^  ztest_direct_mmap.<locals>.func  s    !tr    r4   )r   c              3   H   K   | ]} t                    |          V  d S r$   rb  r   rK   r^  s     r!   r   z#test_direct_mmap.<locals>.<genexpr>  s3      ;;!]WT]]1--;;;;;;r    c              3   H   K   | ]} t                    |          V  d S r$   rb  r  s     r!   r   z#test_direct_mmap.<locals>.<genexpr>  s3       A Aaq!1!1 A A A A A Ar    c                                    S r$   r   )r  s   r!   workerz test_direct_mmap.<locals>.worker  s    {}}r    c              3   F   K   | ]} t                                V  d S r$   rb  )r   r   r  s     r!   r   z#test_direct_mmap.<locals>.<genexpr>  s1       E Eq!2!2 E E E E E Er    rF   r   )	strrP   r   rT   tofiler   r+   r,   r   )	r[   r^   r\  refr(  r  r^  r  r  s	        @@@@r!   test_direct_mmapr  x  sq    6;;y))**H
	"G$$$AHHXF F F F F
   +--C (!


;;;;s;;;
;
;C !ha    A A A AC5 A A AAAGJ!!'3///     !ha    E E E EE!HH E E EEEGJ!!'!*c22222r    )Vr   r   rt   r  rh  ry  	itertoolstimer   rr   r   r  joblib.test.commonr   r   r   r   r   r	   joblib.testingr
   r   r   r   r   joblib.backportsr   joblib.parallelr   r   joblib.poolr   joblib.executorr   TestExecutorjoblib._memmapping_reducerr   r   r   r   r   r   _memmapping_reducerr5  r"   r%   r(   r2   r5   ri   r   r   r   get_memmapping_executorr   r   r   r   r   r   r   r  r  r  r  r)  markr/  r2  r:  r<  rC  rF  rH  rK  listproductr   r   r  rR  rX  re  rw  r|  r  r   r    r!   <module>r     s#
   				  



  				                     - - - - - - - - - - - - - - 0 0 0 0 0 0 3 3 3 3 3 3 + + + + + + D D D D D D D D D D D D D D ( ( ( ( ( ( - - - - - - - - & & & & & & F F F F F F ; ; ; ; ; ; @ @ @ @ @ @ ; ; ; ; ; ; 4 4 4 4 4 4 7 7 7 7 7 7 : : : : : : ( ( ( ( ( (* * *       
< < <@ @ @ I- I-  I-X 	 'RCE E E) )E E )X *+ *+  *+Z C C C* Y)MN#V,. . ./ /. .  /d Y)MN#V,. . .   . .   F Y*F344C C 54  C@ Y*F344,2 ,2 54  ,2^ Y*F344
F 
F 54  
F Y*F344
F 
F 54  
F #4 #4  #4L C< C<  C<L Y*F344*. *. 54  *.Z Y*F344/ / 54  / Y*F3441. 1. 54  1.h Y)MN#V,. . .1 1. .  1h +##E $  	
 	
 	
 	 G( G(   G(T Y)MN#V,. . . . .  . Y)MN#V,. . ..6 .6. .   
.6b Y)MN#V,. . .6 6. .   
60 Y)MN#V,. . . . .  ,   Y)MN#V,. . . . .  2   D			=>a
J 
J K K	: 	:+<9+<	F#SSeeAhh%7%7,9 ,9 	: 	: 	:	; ; ; ; ;  &	2 	2 	2  )% & & &[4,''. . ('& & . 7% 7% 7%t   3 3  3 3 3r    