Skip to content

Commit 8328f37

Browse files
vstinnervstinner
committed
gh-109276: regrtest: add WORKER_FAILED state
Rename WORKER_ERROR to WORKER_BUG. Add WORKER_FAILED state: it does not stop the manager, whereas WORKER_BUG does. Change also TestResults.display_result() order: display failed tests at the end, the important important information.
1 parent 89966a6 commit 8328f37

File tree

3 files changed

+46
-27
lines changed

3 files changed

+46
-27
lines changed

‎Lib/test/libregrtest/result.py‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@ class State:
1919
ENV_CHANGED = "ENV_CHANGED"
2020
RESOURCE_DENIED = "RESOURCE_DENIED"
2121
INTERRUPTED = "INTERRUPTED"
22-
MULTIPROCESSING_ERROR = "MULTIPROCESSING_ERROR"
22+
WORKER_FAILED = "WORKER_FAILED" # non-zero worker process exit code
23+
WORKER_BUG = "WORKER_BUG" # exception when running a worker
2324
DID_NOT_RUN = "DID_NOT_RUN"
2425
TIMEOUT = "TIMEOUT"
2526

@@ -29,7 +30,8 @@ def is_failed(state):
2930
State.FAILED,
3031
State.UNCAUGHT_EXC,
3132
State.REFLEAK,
32-
State.MULTIPROCESSING_ERROR,
33+
State.WORKER_FAILED,
34+
State.WORKER_BUG,
3335
State.TIMEOUT}
3436

3537
@staticmethod
@@ -42,14 +44,16 @@ def has_meaningful_duration(state):
4244
State.SKIPPED,
4345
State.RESOURCE_DENIED,
4446
State.INTERRUPTED,
45-
State.MULTIPROCESSING_ERROR,
47+
State.WORKER_FAILED,
48+
State.WORKER_BUG,
4649
State.DID_NOT_RUN}
4750

4851
@staticmethod
4952
def must_stop(state):
5053
return state in {
5154
State.INTERRUPTED,
52-
State.MULTIPROCESSING_ERROR}
55+
State.WORKER_BUG,
56+
}
5357

5458

5559
@dataclasses.dataclass(slots=True)
@@ -108,8 +112,10 @@ def __str__(self) -> str:
108112
return f"{self.test_name} skipped (resource denied)"
109113
case State.INTERRUPTED:
110114
return f"{self.test_name} interrupted"
111-
case State.MULTIPROCESSING_ERROR:
112-
return f"{self.test_name} process crashed"
115+
case State.WORKER_FAILED:
116+
return f"{self.test_name} worker non-zero exit code"
117+
case State.WORKER_BUG:
118+
return f"{self.test_name} worker bug"
113119
case State.DID_NOT_RUN:
114120
return f"{self.test_name} ran no tests"
115121
case State.TIMEOUT:

‎Lib/test/libregrtest/results.py‎

Lines changed: 20 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ def __init__(self):
3030
self.rerun_results: list[TestResult] = []
3131

3232
self.interrupted: bool = False
33+
self.worker_bug: bool = False
3334
self.test_times: list[tuple[float, TestName]] = []
3435
self.stats = TestStats()
3536
# used by --junit-xml
@@ -38,7 +39,8 @@ def __init__(self):
3839
def is_all_good(self):
3940
return (not self.bad
4041
and not self.skipped
41-
and not self.interrupted)
42+
and not self.interrupted
43+
and not self.worker_bug)
4244

4345
def get_executed(self):
4446
return (set(self.good) | set(self.bad) | set(self.skipped)
@@ -60,6 +62,8 @@ def get_state(self, fail_env_changed):
6062

6163
if self.interrupted:
6264
state.append("INTERRUPTED")
65+
if self.worker_bug:
66+
state.append("WORKER BUG")
6367
if not state:
6468
state.append("SUCCESS")
6569

@@ -77,6 +81,8 @@ def get_exitcode(self, fail_env_changed, fail_rerun):
7781
exitcode = EXITCODE_NO_TESTS_RAN
7882
elif fail_rerun and self.rerun:
7983
exitcode = EXITCODE_RERUN_FAIL
84+
elif self.worker_bug:
85+
exitcode = EXITCODE_BAD_TEST
8086
return exitcode
8187

8288
def accumulate_result(self, result: TestResult, runtests: RunTests):
@@ -105,6 +111,9 @@ def accumulate_result(self, result: TestResult, runtests: RunTests):
105111
else:
106112
raise ValueError(f"invalid test state: {result.state!r}")
107113

114+
if result.state == State.WORKER_BUG:
115+
self.worker_bug = True
116+
108117
if result.has_meaningful_duration() and not rerun:
109118
self.test_times.append((result.duration, test_name))
110119
if result.stats is not None:
@@ -173,29 +182,28 @@ def write_junit(self, filename: StrPath):
173182
f.write(s)
174183

175184
def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool):
176-
omitted = set(tests) - self.get_executed()
177-
if omitted:
178-
print()
179-
print(count(len(omitted), "test"), "omitted:")
180-
printlist(omitted)
181-
182185
if print_slowest:
183186
self.test_times.sort(reverse=True)
184187
print()
185188
print("10 slowest tests:")
186189
for test_time, test in self.test_times[:10]:
187190
print("- %s: %s" % (test, format_duration(test_time)))
188191

189-
all_tests = [
190-
(self.bad, "test", "{} failed:"),
191-
(self.env_changed, "test", "{} altered the execution environment (env changed):"),
192-
]
192+
all_tests = []
193+
omitted = set(tests) - self.get_executed()
194+
195+
# less important
196+
all_tests.append((omitted, "test", "{} omitted:"))
193197
if not quiet:
194198
all_tests.append((self.skipped, "test", "{} skipped:"))
195199
all_tests.append((self.resource_denied, "test", "{} skipped (resource denied):"))
196-
all_tests.append((self.rerun, "re-run test", "{}:"))
197200
all_tests.append((self.run_no_tests, "test", "{} run no tests:"))
198201

202+
# more important
203+
all_tests.append((self.env_changed, "test", "{} altered the execution environment (env changed):"))
204+
all_tests.append((self.rerun, "re-run test", "{}:"))
205+
all_tests.append((self.bad, "test", "{} failed:"))
206+
199207
for tests_list, count_text, title_format in all_tests:
200208
if tests_list:
201209
print()

‎Lib/test/libregrtest/run_workers.py‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ def __init__(self,
9292
test_name: TestName,
9393
err_msg: str | None,
9494
stdout: str | None,
95-
state: str = State.MULTIPROCESSING_ERROR):
95+
state: str):
9696
result = TestResult(test_name, state=state)
9797
self.mp_result = MultiprocessResult(result, stdout, err_msg)
9898
super().__init__()
@@ -298,7 +298,9 @@ def read_stdout(self, stdout_file: TextIO) -> str:
298298
# gh-101634: Catch UnicodeDecodeError if stdout cannot be
299299
# decoded from encoding
300300
raise WorkerError(self.test_name,
301-
f"Cannot read process stdout: {exc}", None)
301+
f"Cannot read process stdout: {exc}",
302+
stdout=None,
303+
state=State.WORKER_BUG)
302304

303305
def read_json(self, json_file: JsonFile, json_tmpfile: TextIO | None,
304306
stdout: str) -> tuple[TestResult, str]:
@@ -317,10 +319,11 @@ def read_json(self, json_file: JsonFile, json_tmpfile: TextIO | None,
317319
# decoded from encoding
318320
err_msg = f"Failed to read worker process JSON: {exc}"
319321
raise WorkerError(self.test_name, err_msg, stdout,
320-
state=State.MULTIPROCESSING_ERROR)
322+
state=State.WORKER_BUG)
321323

322324
if not worker_json:
323-
raise WorkerError(self.test_name, "empty JSON", stdout)
325+
raise WorkerError(self.test_name, "empty JSON", stdout,
326+
state=State.WORKER_BUG)
324327

325328
try:
326329
result = TestResult.from_json(worker_json)
@@ -329,7 +332,7 @@ def read_json(self, json_file: JsonFile, json_tmpfile: TextIO | None,
329332
# decoded from encoding
330333
err_msg = f"Failed to parse worker process JSON: {exc}"
331334
raise WorkerError(self.test_name, err_msg, stdout,
332-
state=State.MULTIPROCESSING_ERROR)
335+
state=State.WORKER_BUG)
333336

334337
return (result, stdout)
335338

@@ -345,9 +348,11 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult:
345348
stdout = self.read_stdout(stdout_file)
346349

347350
if retcode is None:
348-
raise WorkerError(self.test_name, None, stdout, state=State.TIMEOUT)
351+
raise WorkerError(self.test_name, None, stdout,
352+
state=State.TIMEOUT)
349353
if retcode != 0:
350-
raise WorkerError(self.test_name, f"Exit code {retcode}", stdout)
354+
raise WorkerError(self.test_name, f"Exit code {retcode}", stdout,
355+
state=State.WORKER_FAILED)
351356

352357
result, stdout = self.read_json(json_file, json_tmpfile, stdout)
353358

@@ -527,7 +532,7 @@ def display_result(self, mp_result: MultiprocessResult) -> None:
527532

528533
text = str(result)
529534
if mp_result.err_msg:
530-
# MULTIPROCESSING_ERROR
535+
# WORKER_ERROR
531536
text += ' (%s)' % mp_result.err_msg
532537
elif (result.duration >= PROGRESS_MIN_TIME and not pgo):
533538
text += ' (%s)' % format_duration(result.duration)
@@ -543,7 +548,7 @@ def _process_result(self, item: QueueOutput) -> TestResult:
543548
# Thread got an exception
544549
format_exc = item[1]
545550
print_warning(f"regrtest worker thread failed: {format_exc}")
546-
result = TestResult("<regrtest worker>", state=State.MULTIPROCESSING_ERROR)
551+
result = TestResult("<regrtest worker>", state=State.WORKER_ERROR)
547552
self.results.accumulate_result(result, self.runtests)
548553
return result
549554

0 commit comments

Comments
 (0)