subprocess's Popen.wait() is now thread safe so that multiple threads · python/cpython@d65ba51
@@ -405,6 +405,10 @@ class STARTUPINFO:
405405import _posixsubprocess
406406import select
407407import selectors
408+try:
409+import threading
410+except ImportError:
411+import dummy_threading as threading
408412409413# When select or poll has indicated that the file is writable,
410414# we can write up to _PIPE_BUF bytes without risk of blocking.
@@ -748,6 +752,12 @@ def __init__(self, args, bufsize=-1, executable=None,
748752pass_fds=()):
749753"""Create new Popen instance."""
750754_cleanup()
755+# Held while anything is calling waitpid before returncode has been
756+# updated to prevent clobbering returncode if wait() or poll() are
757+# called from multiple threads at once. After acquiring the lock,
758+# code must re-check self.returncode to see if another thread just
759+# finished a waitpid() call.
760+self._waitpid_lock = threading.Lock()
751761752762self._input = None
753763self._communication_started = False
@@ -1450,6 +1460,7 @@ def _execute_child(self, args, executable, preexec_fn, close_fds,
14501460def _handle_exitstatus(self, sts, _WIFSIGNALED=os.WIFSIGNALED,
14511461_WTERMSIG=os.WTERMSIG, _WIFEXITED=os.WIFEXITED,
14521462_WEXITSTATUS=os.WEXITSTATUS):
1463+"""All callers to this function MUST hold self._waitpid_lock."""
14531464# This method is called (indirectly) by __del__, so it cannot
14541465# refer to anything outside of its local scope.
14551466if _WIFSIGNALED(sts):
@@ -1471,7 +1482,13 @@ def _internal_poll(self, _deadstate=None, _waitpid=os.waitpid,
1471148214721483 """
14731484if self.returncode is None:
1485+if not self._waitpid_lock.acquire(False):
1486+# Something else is busy calling waitpid. Don't allow two
1487+# at once. We know nothing yet.
1488+return None
14741489try:
1490+if self.returncode is not None:
1491+return self.returncode # Another thread waited.
14751492pid, sts = _waitpid(self.pid, _WNOHANG)
14761493if pid == self.pid:
14771494self._handle_exitstatus(sts)
@@ -1485,10 +1502,13 @@ def _internal_poll(self, _deadstate=None, _waitpid=os.waitpid,
14851502# can't get the status.
14861503# http://bugs.python.org/issue15756
14871504self.returncode = 0
1505+finally:
1506+self._waitpid_lock.release()
14881507return self.returncode
148915081490150914911510def _try_wait(self, wait_flags):
1511+"""All callers to this function MUST hold self._waitpid_lock."""
14921512try:
14931513 (pid, sts) = _eintr_retry_call(os.waitpid, self.pid, wait_flags)
14941514except OSError as e:
@@ -1521,23 +1541,33 @@ def wait(self, timeout=None, endtime=None):
15211541# cribbed from Lib/threading.py in Thread.wait() at r71065.
15221542delay = 0.0005 # 500 us -> initial delay of 1 ms
15231543while True:
1524- (pid, sts) = self._try_wait(os.WNOHANG)
1525-assert pid == self.pid or pid == 0
1526-if pid == self.pid:
1527-self._handle_exitstatus(sts)
1528-break
1544+if self._waitpid_lock.acquire(False):
1545+try:
1546+if self.returncode is not None:
1547+break # Another thread waited.
1548+ (pid, sts) = self._try_wait(os.WNOHANG)
1549+assert pid == self.pid or pid == 0
1550+if pid == self.pid:
1551+self._handle_exitstatus(sts)
1552+break
1553+finally:
1554+self._waitpid_lock.release()
15291555remaining = self._remaining_time(endtime)
15301556if remaining <= 0:
15311557raise TimeoutExpired(self.args, timeout)
15321558delay = min(delay * 2, remaining, .05)
15331559time.sleep(delay)
15341560else:
15351561while self.returncode is None:
1536- (pid, sts) = self._try_wait(0)
1537-# Check the pid and loop as waitpid has been known to return
1538-# 0 even without WNOHANG in odd situations. issue14396.
1539-if pid == self.pid:
1540-self._handle_exitstatus(sts)
1562+with self._waitpid_lock:
1563+if self.returncode is not None:
1564+break # Another thread waited.
1565+ (pid, sts) = self._try_wait(0)
1566+# Check the pid and loop as waitpid has been known to
1567+# return 0 even without WNOHANG in odd situations.
1568+# http://bugs.python.org/issue14396.
1569+if pid == self.pid:
1570+self._handle_exitstatus(sts)
15411571return self.returncode
1542157215431573