subprocess's Popen.wait() is now thread safe so that multiple threads · python/cpython@d65ba51

@@ -405,6 +405,10 @@ class STARTUPINFO:

405405

import _posixsubprocess

406406

import select

407407

import selectors

408+

try:

409+

import threading

410+

except ImportError:

411+

import dummy_threading as threading

408412409413

# When select or poll has indicated that the file is writable,

410414

# we can write up to _PIPE_BUF bytes without risk of blocking.

@@ -748,6 +752,12 @@ def __init__(self, args, bufsize=-1, executable=None,

748752

pass_fds=()):

749753

"""Create new Popen instance."""

750754

_cleanup()

755+

# Held while anything is calling waitpid before returncode has been

756+

# updated to prevent clobbering returncode if wait() or poll() are

757+

# called from multiple threads at once. After acquiring the lock,

758+

# code must re-check self.returncode to see if another thread just

759+

# finished a waitpid() call.

760+

self._waitpid_lock = threading.Lock()

751761752762

self._input = None

753763

self._communication_started = False

@@ -1450,6 +1460,7 @@ def _execute_child(self, args, executable, preexec_fn, close_fds,

14501460

def _handle_exitstatus(self, sts, _WIFSIGNALED=os.WIFSIGNALED,

14511461

_WTERMSIG=os.WTERMSIG, _WIFEXITED=os.WIFEXITED,

14521462

_WEXITSTATUS=os.WEXITSTATUS):

1463+

"""All callers to this function MUST hold self._waitpid_lock."""

14531464

# This method is called (indirectly) by __del__, so it cannot

14541465

# refer to anything outside of its local scope.

14551466

if _WIFSIGNALED(sts):

@@ -1471,7 +1482,13 @@ def _internal_poll(self, _deadstate=None, _waitpid=os.waitpid,

1471148214721483

"""

14731484

if self.returncode is None:

1485+

if not self._waitpid_lock.acquire(False):

1486+

# Something else is busy calling waitpid. Don't allow two

1487+

# at once. We know nothing yet.

1488+

return None

14741489

try:

1490+

if self.returncode is not None:

1491+

return self.returncode # Another thread waited.

14751492

pid, sts = _waitpid(self.pid, _WNOHANG)

14761493

if pid == self.pid:

14771494

self._handle_exitstatus(sts)

@@ -1485,10 +1502,13 @@ def _internal_poll(self, _deadstate=None, _waitpid=os.waitpid,

14851502

# can't get the status.

14861503

# http://bugs.python.org/issue15756

14871504

self.returncode = 0

1505+

finally:

1506+

self._waitpid_lock.release()

14881507

return self.returncode

148915081490150914911510

def _try_wait(self, wait_flags):

1511+

"""All callers to this function MUST hold self._waitpid_lock."""

14921512

try:

14931513

(pid, sts) = _eintr_retry_call(os.waitpid, self.pid, wait_flags)

14941514

except OSError as e:

@@ -1521,23 +1541,33 @@ def wait(self, timeout=None, endtime=None):

15211541

# cribbed from Lib/threading.py in Thread.wait() at r71065.

15221542

delay = 0.0005 # 500 us -> initial delay of 1 ms

15231543

while True:

1524-

(pid, sts) = self._try_wait(os.WNOHANG)

1525-

assert pid == self.pid or pid == 0

1526-

if pid == self.pid:

1527-

self._handle_exitstatus(sts)

1528-

break

1544+

if self._waitpid_lock.acquire(False):

1545+

try:

1546+

if self.returncode is not None:

1547+

break # Another thread waited.

1548+

(pid, sts) = self._try_wait(os.WNOHANG)

1549+

assert pid == self.pid or pid == 0

1550+

if pid == self.pid:

1551+

self._handle_exitstatus(sts)

1552+

break

1553+

finally:

1554+

self._waitpid_lock.release()

15291555

remaining = self._remaining_time(endtime)

15301556

if remaining <= 0:

15311557

raise TimeoutExpired(self.args, timeout)

15321558

delay = min(delay * 2, remaining, .05)

15331559

time.sleep(delay)

15341560

else:

15351561

while self.returncode is None:

1536-

(pid, sts) = self._try_wait(0)

1537-

# Check the pid and loop as waitpid has been known to return

1538-

# 0 even without WNOHANG in odd situations. issue14396.

1539-

if pid == self.pid:

1540-

self._handle_exitstatus(sts)

1562+

with self._waitpid_lock:

1563+

if self.returncode is not None:

1564+

break # Another thread waited.

1565+

(pid, sts) = self._try_wait(0)

1566+

# Check the pid and loop as waitpid has been known to

1567+

# return 0 even without WNOHANG in odd situations.

1568+

# http://bugs.python.org/issue14396.

1569+

if pid == self.pid:

1570+

self._handle_exitstatus(sts)

15411571

return self.returncode

1542157215431573