bpo-26903: Limit ProcessPoolExecutor to 61 workers on Windows (GH-131… · python/cpython@8ea0fd8

4 files changed

lines changed

Original file line numberDiff line numberDiff line change

@@ -216,6 +216,10 @@ to a :class:`ProcessPoolExecutor` will result in deadlock.

216216

given, it will default to the number of processors on the machine.

217217

If *max_workers* is lower or equal to ``0``, then a :exc:`ValueError`

218218

will be raised.

219+

On Windows, *max_workers* must be equal or lower than ``61``. If it is not

220+

then :exc:`ValueError` will be raised. If *max_workers* is ``None``, then

221+

the default chosen will be at most ``61``, even if more processors are

222+

available.

219223

*mp_context* can be a multiprocessing context or None. It will be used to

220224

launch the workers. If *mp_context* is ``None`` or not given, the default

221225

multiprocessing context is used.

Original file line numberDiff line numberDiff line change

@@ -57,6 +57,7 @@

5757

import weakref

5858

from functools import partial

5959

import itertools

60+

import sys

6061

import traceback

6162
6263

# Workers are created as daemon threads and processes. This is done to allow the

@@ -109,6 +110,12 @@ def _python_exit():

109110

EXTRA_QUEUED_CALLS = 1

110111
111112
113+

# On Windows, WaitForMultipleObjects is used to wait for processes to finish.

114+

# It can wait on, at most, 63 objects. There is an overhead of two objects:

115+

# - the result queue reader

116+

# - the thread wakeup reader

117+

_MAX_WINDOWS_WORKERS = 63 - 2

118+
112119

# Hack to embed stringification of remote traceback in local traceback

113120
114121

class _RemoteTraceback(Exception):

@@ -504,9 +511,16 @@ def __init__(self, max_workers=None, mp_context=None,

504511
505512

if max_workers is None:

506513

self._max_workers = os.cpu_count() or 1

514+

if sys.platform == 'win32':

515+

self._max_workers = min(_MAX_WINDOWS_WORKERS,

516+

self._max_workers)

507517

else:

508518

if max_workers <= 0:

509519

raise ValueError("max_workers must be greater than 0")

520+

elif (sys.platform == 'win32' and

521+

max_workers > _MAX_WINDOWS_WORKERS):

522+

raise ValueError(

523+

f"max_workers must be <= {_MAX_WINDOWS_WORKERS}")

510524
511525

self._max_workers = max_workers

512526
Original file line numberDiff line numberDiff line change

@@ -754,6 +754,13 @@ def test_default_workers(self):

754754
755755
756756

class ProcessPoolExecutorTest(ExecutorTest):

757+
758+

@unittest.skipUnless(sys.platform=='win32', 'Windows-only process limit')

759+

def test_max_workers_too_large(self):

760+

with self.assertRaisesRegex(ValueError,

761+

"max_workers must be <= 61"):

762+

futures.ProcessPoolExecutor(max_workers=62)

763+
757764

def test_killed_child(self):

758765

# When a child process is abruptly terminated, the whole pool gets

759766

# "broken".

Original file line numberDiff line numberDiff line change

@@ -0,0 +1 @@

1+

Limit `max_workers` in `ProcessPoolExecutor` to 61 to work around a WaitForMultipleObjects limitation.