bpo-39360: Ensure all workers exit when finalizing a multiprocessing … · python/cpython@7f5302f

3 files changed

lines changed

Original file line numberDiff line numberDiff line change

@@ -651,8 +651,6 @@ def close(self):

651651

def terminate(self):

652652

util.debug('terminating pool')

653653

self._state = TERMINATE

654-

self._worker_handler._state = TERMINATE

655-

self._change_notifier.put(None)

656654

self._terminate()

657655
658656

def join(self):

@@ -682,7 +680,12 @@ def _terminate_pool(cls, taskqueue, inqueue, outqueue, pool, change_notifier,

682680

# this is guaranteed to only be called once

683681

util.debug('finalizing pool')

684682
683+

# Notify that the worker_handler state has been changed so the

684+

# _handle_workers loop can be unblocked (and exited) in order to

685+

# send the finalization sentinel all the workers.

685686

worker_handler._state = TERMINATE

687+

change_notifier.put(None)

688+
686689

task_handler._state = TERMINATE

687690
688691

util.debug('helping task handler/workers to finish')

Original file line numberDiff line numberDiff line change

@@ -2778,6 +2778,24 @@ def test_pool_worker_lifetime_early_close(self):

27782778

for (j, res) in enumerate(results):

27792779

self.assertEqual(res.get(), sqr(j))

27802780
2781+

def test_worker_finalization_via_atexit_handler_of_multiprocessing(self):

2782+

# tests cases against bpo-38744 and bpo-39360

2783+

cmd = '''if 1:

2784+

from multiprocessing import Pool

2785+

problem = None

2786+

class A:

2787+

def __init__(self):

2788+

self.pool = Pool(processes=1)

2789+

def test():

2790+

global problem

2791+

problem = A()

2792+

problem.pool.map(float, tuple(range(10)))

2793+

if __name__ == "__main__":

2794+

test()

2795+

'''

2796+

rc, out, err = test.support.script_helper.assert_python_ok('-c', cmd)

2797+

self.assertEqual(rc, 0)

2798+
27812799

#

27822800

# Test of creating a customized manager class

27832801

#

Original file line numberDiff line numberDiff line change

@@ -0,0 +1,4 @@

1+

Ensure all workers exit when finalizing a :class:`multiprocessing.Pool` implicitly via the module finalization

2+

handlers of multiprocessing. This fixes a deadlock situation that can be experienced when the Pool is not

3+

properly finalized via the context manager or a call to ``multiprocessing.Pool.terminate``. Patch by Batuhan Taskaya

4+

and Pablo Galindo.