[3.7] bpo-37531: Enhance regrtest multiprocess timeout (GH-15345) (GH… · python/cpython@494b61a
@@ -126,6 +126,38 @@ def __repr__(self):
126126info.append(f'pid={popen.pid}')
127127return '<%s>' % ' '.join(info)
128128129+def _kill(self):
130+dt = time.monotonic() - self.start_time
131+132+popen = self._popen
133+pid = popen.pid
134+print("Kill worker process %s running for %.1f sec" % (pid, dt),
135+file=sys.stderr, flush=True)
136+137+try:
138+popen.kill()
139+return True
140+except OSError as exc:
141+print("WARNING: Failed to kill worker process %s: %r" % (pid, exc),
142+file=sys.stderr, flush=True)
143+return False
144+145+def _close_wait(self):
146+popen = self._popen
147+148+# stdout and stderr must be closed to ensure that communicate()
149+# does not hang
150+popen.stdout.close()
151+popen.stderr.close()
152+153+try:
154+popen.wait(JOIN_TIMEOUT)
155+except (subprocess.TimeoutExpired, OSError) as exc:
156+print("WARNING: Failed to wait for worker process %s "
157+"completion (timeout=%.1f sec): %r"
158+% (popen.pid, JOIN_TIMEOUT, exc),
159+file=sys.stderr, flush=True)
160+129161def kill(self):
130162"""
131163 Kill the current process (if any).
@@ -135,30 +167,45 @@ def kill(self):
135167 """
136168self._killed = True
137169138-popen = self._popen
139-if popen is None:
170+if self._popen is None:
140171return
141-popen.kill()
142-# stdout and stderr must be closed to ensure that communicate()
143-# does not hang
144-popen.stdout.close()
145-popen.stderr.close()
146-popen.wait()
172+173+if not self._kill():
174+return
175+176+self._close_wait()
147177148178def mp_result_error(self, test_name, error_type, stdout='', stderr='',
149179err_msg=None):
150180test_time = time.monotonic() - self.start_time
151181result = TestResult(test_name, error_type, test_time, None)
152182return MultiprocessResult(result, stdout, stderr, err_msg)
153183184+def _timedout(self, test_name):
185+self._kill()
186+187+stdout = sterr = ''
188+popen = self._popen
189+try:
190+stdout, stderr = popen.communicate(timeout=JOIN_TIMEOUT)
191+except (subprocess.TimeoutExpired, OSError) as exc:
192+print("WARNING: Failed to read worker process %s output "
193+"(timeout=%.1f sec): %r"
194+% (popen.pid, exc, timeout),
195+file=sys.stderr, flush=True)
196+197+self._close_wait()
198+199+return self.mp_result_error(test_name, TIMEOUT, stdout, stderr)
200+154201def _runtest(self, test_name):
155202try:
156203self.start_time = time.monotonic()
157204self.current_test_name = test_name
158205159206self._popen = run_test_in_subprocess(test_name, self.ns)
160207popen = self._popen
161-with popen:
208+try:
162209try:
163210if self._killed:
164211# If kill() has been called before self._popen is set,
@@ -175,12 +222,7 @@ def _runtest(self, test_name):
175222# on reading closed stdout/stderr
176223raise ExitThread
177224178-popen.kill()
179-stdout, stderr = popen.communicate()
180-self.kill()
181-182-return self.mp_result_error(test_name, TIMEOUT,
183-stdout, stderr)
225+return self._timedout(test_name)
184226except OSError:
185227if self._killed:
186228# kill() has been called: communicate() fails
@@ -190,8 +232,10 @@ def _runtest(self, test_name):
190232except:
191233self.kill()
192234raise
235+finally:
236+self._close_wait()
193237194-retcode = popen.wait()
238+retcode = popen.returncode
195239finally:
196240self.current_test_name = None
197241self._popen = None
@@ -286,10 +330,11 @@ def wait_workers(self):
286330if not worker.is_alive():
287331break
288332dt = time.monotonic() - start_time
289-print("Wait for regrtest worker %r for %.1f sec" % (worker, dt))
333+print("Wait for regrtest worker %r for %.1f sec" % (worker, dt),
334+flush=True)
290335if dt > JOIN_TIMEOUT:
291336print("Warning -- failed to join a regrtest worker %s"
292-% worker)
337+% worker, flush=True)
293338break
294339295340def _get_result(self):