[3.7] bpo-26903: Limit ProcessPoolExecutor to 61 workers on Windows (GH-13132) by miss-islington · Pull Request #13206 · python/cpython
Expand Up
@@ -57,6 +57,7 @@
import weakref
from functools import partial
import itertools
import sys
import traceback
# Workers are created as daemon threads and processes. This is done to allow the Expand Down Expand Up @@ -109,6 +110,12 @@ def _python_exit(): EXTRA_QUEUED_CALLS = 1
# On Windows, WaitForMultipleObjects is used to wait for processes to finish. # It can wait on, at most, 63 objects. There is an overhead of two objects: # - the result queue reader # - the thread wakeup reader _MAX_WINDOWS_WORKERS = 63 - 2
# Hack to embed stringification of remote traceback in local traceback
class _RemoteTraceback(Exception): Expand Down Expand Up @@ -504,9 +511,16 @@ def __init__(self, max_workers=None, mp_context=None,
if max_workers is None: self._max_workers = os.cpu_count() or 1 if sys.platform == 'win32': self._max_workers = min(_MAX_WINDOWS_WORKERS, self._max_workers) else: if max_workers <= 0: raise ValueError("max_workers must be greater than 0") elif (sys.platform == 'win32' and max_workers > _MAX_WINDOWS_WORKERS): raise ValueError( f"max_workers must be <= {_MAX_WINDOWS_WORKERS}")
self._max_workers = max_workers
Expand Down
# Workers are created as daemon threads and processes. This is done to allow the Expand Down Expand Up @@ -109,6 +110,12 @@ def _python_exit(): EXTRA_QUEUED_CALLS = 1
# On Windows, WaitForMultipleObjects is used to wait for processes to finish. # It can wait on, at most, 63 objects. There is an overhead of two objects: # - the result queue reader # - the thread wakeup reader _MAX_WINDOWS_WORKERS = 63 - 2
# Hack to embed stringification of remote traceback in local traceback
class _RemoteTraceback(Exception): Expand Down Expand Up @@ -504,9 +511,16 @@ def __init__(self, max_workers=None, mp_context=None,
if max_workers is None: self._max_workers = os.cpu_count() or 1 if sys.platform == 'win32': self._max_workers = min(_MAX_WINDOWS_WORKERS, self._max_workers) else: if max_workers <= 0: raise ValueError("max_workers must be greater than 0") elif (sys.platform == 'win32' and max_workers > _MAX_WINDOWS_WORKERS): raise ValueError( f"max_workers must be <= {_MAX_WINDOWS_WORKERS}")
self._max_workers = max_workers
Expand Down