From f9a2fcf53a6c0b3d1056e73bad36a8581770e2d4 Mon Sep 17 00:00:00 2001 From: Zhafron Date: Mon, 9 Nov 2026 11:01:37 +0700 Subject: [PATCH 1/5] fix(vk): report a driver fault in a pipeline compile on Linux Windows wraps vkCreateComputePipelines and vkCreateGraphicsPipelines in __try so a driver that faults on a shader costs one pipeline, not the game. Linux had no equivalent: the call went to the driver as it was, and an access violation inside RADV's compiler ended the process during the first start's warm-up (issues 915 and 920, and the RX 6800 XT report). Arm a thread-local window around the driver call and catch SIGSEGV and SIGBUS inside it, reporting the call's failure with the same values the Windows branch returns. A fault outside the window is passed to whoever held the signal, so the game's own crash report and its slow-path handlers are unchanged. tests/test_linux_driver_fault_guard.py compiles the actual guarded calls from vk_draw.cpp against a fake Vulkan that faults on demand: a fault in the compute or graphics call returns -77 or VK_ERROR_UNKNOWN and the next compile still reaches the driver, a fault outside the window still ends the process, and the unfixed source dies on the same fault. --- gpu/vk/vk_draw.cpp | 54 +++++++++- tests/run_all.py | 2 + tests/test_linux_driver_fault_guard.py | 140 +++++++++++++++++++++++++ 3 files changed, 193 insertions(+), 3 deletions(-) create mode 100644 tests/test_linux_driver_fault_guard.py diff --git a/gpu/vk/vk_draw.cpp b/gpu/vk/vk_draw.cpp index 51b6f17ca..99a0719d4 100644 --- a/gpu/vk/vk_draw.cpp +++ b/gpu/vk/vk_draw.cpp @@ -22,6 +22,8 @@ #include #include #include +#include +#include #include "shader_recompiler/backend/spirv/emit_spirv.h" @@ -737,12 +739,58 @@ __declspec(noinline) VkResult guarded_compute(VkPipelineCache cache, const VkCom } } #else -// (linux: no structured exceptions; the driver is called as it is) +// (linux: no structured exceptions, so the fault arrives as a signal on the compiling thread: an armed window covers the driver +// call alone, and a fault in it is reported as the call's failure, as the Windows guard above reports an access violation. A fault +// outside it goes to whoever had the signal before, so the game's own crash report is unchanged) +static thread_local sigjmp_buf fault_jmp; +static thread_local volatile sig_atomic_t fault_armed; +static struct sigaction fault_previous[2]; +static const int fault_signals[2] = {SIGSEGV, SIGBUS}; + +static void on_driver_fault(int sig, siginfo_t* si, void* ctx) { + if (fault_armed) { + fault_armed = 0; + siglongjmp(fault_jmp, 1); + } + const struct sigaction* p = &fault_previous[sig == SIGSEGV ? 0 : 1]; + if ((p->sa_flags & SA_SIGINFO) && p->sa_sigaction) { + p->sa_sigaction(sig, si, ctx); + } else if (p->sa_handler && p->sa_handler != SIG_IGN && p->sa_handler != SIG_DFL) { + p->sa_handler(sig); + } else { + signal(sig, SIG_DFL); + } +} + +static void install_driver_fault_guard() { + static std::once_flag once; + std::call_once(once, [] { + for (int i = 0; i < 2; i++) { + struct sigaction sa; + memset(&sa, 0, sizeof sa); + sa.sa_sigaction = on_driver_fault; + sa.sa_flags = SA_SIGINFO; + sigemptyset(&sa.sa_mask); + sigaction(fault_signals[i], &sa, &fault_previous[i]); + } + }); +} + VkResult guarded_graphics(VkPipelineCache cache, const VkGraphicsPipelineCreateInfo* ci, VkPipeline* pl) { - return vkCreateGraphicsPipelines(dev.device, cache, 1, ci, nullptr, pl); + install_driver_fault_guard(); + if (sigsetjmp(fault_jmp, 1) != 0) return VK_ERROR_UNKNOWN; + fault_armed = 1; + const VkResult r = vkCreateGraphicsPipelines(dev.device, cache, 1, ci, nullptr, pl); + fault_armed = 0; + return r; } VkResult guarded_compute(VkPipelineCache cache, const VkComputePipelineCreateInfo* ci, VkPipeline* pl) { - return vkCreateComputePipelines(dev.device, cache, 1, ci, nullptr, pl); + install_driver_fault_guard(); + if (sigsetjmp(fault_jmp, 1) != 0) return (VkResult)-77; + fault_armed = 1; + const VkResult r = vkCreateComputePipelines(dev.device, cache, 1, ci, nullptr, pl); + fault_armed = 0; + return r; } #endif VkPipeline build_graphics(const Program& p, const NvnState& s, const Targets& at, VkPrimitiveTopology topo, int part = -1); diff --git a/tests/run_all.py b/tests/run_all.py index 13fe430b3..9f30c6120 100644 --- a/tests/run_all.py +++ b/tests/run_all.py @@ -38,6 +38,7 @@ "tests/test_cpp_preflight.py": ["--cxx", "g++"], "tests/test_invalidate_vcore.py": ["--cxx", "g++"], "tests/test_seed_cs_pointers.py": ["--cxx", "g++"], + "tests/test_linux_driver_fault_guard.py": ["--cxx", "{cxx}"], "tests/test_lower_atomg_cas.py": ["--cxx", "g++"], "nx2/test_cbuf_table_extent.py": ["--cxx", "g++"], "nx2/test_dump_ssbo_read.py": ["--cxx", "g++"], @@ -71,6 +72,7 @@ # (Linux only) "tests/test_hookshot_launch.py": ({"windows"}, "the Linux launcher, a POSIX shell script"), "tests/test_linux_update.py": ({"windows"}, "the Linux builder's update and folder launcher, POSIX files and shell scripts"), + "tests/test_linux_driver_fault_guard.py": ({"windows"}, "POSIX signals: the Linux guard"), } # (spirv-val: SPIRV-Tools comes with the Linux runner's packages, not on Windows) for _name in ("nx2/test_depth_sample_types.py", "nx2/test_default_point_size.py", "nx2/test_draw_parameters.py", "nx2/test_coop_spirv.py", diff --git a/tests/test_linux_driver_fault_guard.py b/tests/test_linux_driver_fault_guard.py new file mode 100644 index 000000000..44888b189 --- /dev/null +++ b/tests/test_linux_driver_fault_guard.py @@ -0,0 +1,140 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +"""Game-free check of the Linux driver-fault guard: the actual guarded pipeline calls from vk_draw.cpp, compiled with a fake +Vulkan whose compute and graphics creation fault on demand. A fault inside the guarded call must come back as the call's +failure and leave the process able to compile again; a fault outside it must still kill the process. + +python -I -B tests/test_linux_driver_fault_guard.py [--cxx clang++] [--expect-fault] +""" +import argparse +import os +from pathlib import Path +import shutil +import signal +import subprocess +import sys +import tempfile + +ROOT = Path(__file__).resolve().parents[1] +START = "return (VkResult)-77; // (a fault in the driver, told apart from what it returns)\n }\n}\n#else\n" +END = "\n#endif\nVkPipeline build_graphics" + +HEAD = r""" +#include +#include +#include +#include +#include +#include +using VkResult = int; +using VkPipeline = uintptr_t; +using VkPipelineCache = uintptr_t; +constexpr VkResult VK_SUCCESS = 0; +constexpr VkResult VK_ERROR_UNKNOWN = -13; +constexpr uintptr_t VK_NULL_HANDLE = 0; +struct VkComputePipelineCreateInfo { int sType = 0; }; +struct VkGraphicsPipelineCreateInfo { int sType = 0; }; +struct { uintptr_t device = 0x1234; } dev; +static int mode; /* (0: compile, 1 and 2: the driver faults) */ +static int calls; +static constexpr VkResult DRIVER_FAULT = -77; +static void fault_now() { *(volatile int *)0x40 = 1; } +VkResult vkCreateComputePipelines(uintptr_t device, VkPipelineCache cache, unsigned count, + const VkComputePipelineCreateInfo *ci, const void *allocator, VkPipeline *out) { + if (device != dev.device || cache != 0 || count != 1 || !ci || allocator || !out) return -1; + ++calls; + if (mode == 1) fault_now(); + *out = 0xcafe; + return VK_SUCCESS; +} +VkResult vkCreateGraphicsPipelines(uintptr_t device, VkPipelineCache cache, unsigned count, + const VkGraphicsPipelineCreateInfo *ci, const void *allocator, VkPipeline *out) { + if (device != dev.device || cache != 0 || count != 1 || !ci || allocator || !out) return -1; + ++calls; + if (mode == 2) fault_now(); + *out = 0xbeef; + return VK_SUCCESS; +} +""" + +MAIN = r""" +int main(int argc, char **argv) { + const char *what = argc > 1 ? argv[1] : "ok"; + if (!std::strcmp(what, "fault-outside")) { fault_now(); std::printf("fail: the fault returned\n"); return 8; } + VkComputePipelineCreateInfo ci; + VkGraphicsPipelineCreateInfo gci; + VkPipeline pl = 0, gpl = 0; + if (!std::strcmp(what, "ok")) { + if (guarded_compute(0, &ci, &pl) != VK_SUCCESS || pl != 0xcafe) { std::printf("fail: compute\n"); return 3; } + if (guarded_graphics(0, &gci, &gpl) != VK_SUCCESS || gpl != 0xbeef) { std::printf("fail: graphics\n"); return 4; } + std::printf("ok: both compiled\n"); + return 0; + } + if (!std::strcmp(what, "fault")) { + mode = 1; + if (guarded_compute(0, &ci, &pl) != DRIVER_FAULT) { std::printf("fail: the fault was not reported\n"); return 5; } + if (pl != VK_NULL_HANDLE) { std::printf("fail: a pipeline was published\n"); return 6; } + } else if (!std::strcmp(what, "fault-graphics")) { + mode = 2; + if (guarded_graphics(0, &gci, &gpl) != VK_ERROR_UNKNOWN) { std::printf("fail: the fault was not reported\n"); return 5; } + if (gpl != VK_NULL_HANDLE) { std::printf("fail: a pipeline was published\n"); return 6; } + } else { + std::printf("fail: unknown case %s\n", what); + return 9; + } + mode = 0; + calls = 0; + pl = gpl = 0; + if (guarded_compute(0, &ci, &pl) != VK_SUCCESS || pl != 0xcafe || calls != 1) { + std::printf("fail: the next compile after the fault did not reach the driver\n"); + return 7; + } + std::printf("ok: the fault was reported and the next compile made\n"); + return 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + parser.add_argument("--source", type=Path, default=ROOT / "gpu/vk/vk_draw.cpp") + parser.add_argument("--expect-fault", action="store_true", + help="unfixed source: a fault in a guarded call must end the process") + args = parser.parse_args() + if os.name == "nt": + raise SystemExit("POSIX signals required") + compiler = shutil.which(args.cxx) + if not compiler: + raise SystemExit("C++ compiler not found: " + args.cxx) + source = args.source.read_text(encoding="utf-8") + start, end = source.index(START) + len(START), source.index(END) + guard = source[start:end] + failures = [] + with tempfile.TemporaryDirectory(prefix="hookshot-fault-guard-") as tmp: + cpp = Path(tmp) / "guard.cpp" + cpp.write_text(HEAD + guard + MAIN, encoding="utf-8") + binary = Path(tmp) / "guard" + build = subprocess.run([compiler, "-std=c++20", "-O1", "-Wall", "-Wextra", "-pthread", "-o", str(binary), str(cpp)], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + if build.returncode: + raise SystemExit("the guard did not compile:\n" + build.stdout) + for case in ("ok", "fault", "fault-graphics", "fault-outside"): + run = subprocess.run([str(binary), case], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + killed = run.returncode == -signal.SIGSEGV + guarded_case = case in ("fault", "fault-graphics") + if case == "fault-outside" or (args.expect_fault and guarded_case): + if not killed: + failures.append(case + ": the fault did not end the process: " + run.stdout.strip()) + elif run.returncode != 0: + failures.append(case + ": " + (run.stdout.strip() or "exit " + str(run.returncode))) + else: + print(case + ": " + run.stdout.strip()) + if failures: + print("\n".join(failures), file=sys.stderr) + raise SystemExit(1) + print("the guard reports a driver fault as the call's failure" if not args.expect_fault + else "the unfixed source dies on a driver fault") + + +if __name__ == "__main__": + main() From e36670c40dd62eb0b259a302868c38bddb852b9c Mon Sep 17 00:00:00 2001 From: Zhafron Date: Mon, 9 Nov 2026 11:04:04 +0700 Subject: [PATCH 2/5] fix(vk): say in the log when the guard drops a pipeline A silent missing pipeline is harder to place in a report than one line naming which kind went. --- gpu/vk/vk_draw.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/gpu/vk/vk_draw.cpp b/gpu/vk/vk_draw.cpp index 99a0719d4..0322807ab 100644 --- a/gpu/vk/vk_draw.cpp +++ b/gpu/vk/vk_draw.cpp @@ -778,7 +778,10 @@ static void install_driver_fault_guard() { VkResult guarded_graphics(VkPipelineCache cache, const VkGraphicsPipelineCreateInfo* ci, VkPipeline* pl) { install_driver_fault_guard(); - if (sigsetjmp(fault_jmp, 1) != 0) return VK_ERROR_UNKNOWN; + if (sigsetjmp(fault_jmp, 1) != 0) { + logf("vk: the driver faulted compiling a graphics pipeline: that one is left out"); + return VK_ERROR_UNKNOWN; + } fault_armed = 1; const VkResult r = vkCreateGraphicsPipelines(dev.device, cache, 1, ci, nullptr, pl); fault_armed = 0; @@ -786,7 +789,10 @@ VkResult guarded_graphics(VkPipelineCache cache, const VkGraphicsPipelineCreateI } VkResult guarded_compute(VkPipelineCache cache, const VkComputePipelineCreateInfo* ci, VkPipeline* pl) { install_driver_fault_guard(); - if (sigsetjmp(fault_jmp, 1) != 0) return (VkResult)-77; + if (sigsetjmp(fault_jmp, 1) != 0) { + logf("vk: the driver faulted compiling a compute pipeline: that one is left out"); + return (VkResult)-77; + } fault_armed = 1; const VkResult r = vkCreateComputePipelines(dev.device, cache, 1, ci, nullptr, pl); fault_armed = 0; From a2f19b6d24261f8b1ea7518d1680247e2d049e85 Mon Sep 17 00:00:00 2001 From: Zhafron Date: Mon, 9 Nov 2026 11:05:57 +0700 Subject: [PATCH 3/5] test(vk): stub the guard log line in the Linux fault test The guarded calls now log the pipeline they drop, so the compiled harness needs that symbol. --- tests/test_linux_driver_fault_guard.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_linux_driver_fault_guard.py b/tests/test_linux_driver_fault_guard.py index 44888b189..e884b42dc 100644 --- a/tests/test_linux_driver_fault_guard.py +++ b/tests/test_linux_driver_fault_guard.py @@ -37,6 +37,8 @@ static int mode; /* (0: compile, 1 and 2: the driver faults) */ static int calls; static constexpr VkResult DRIVER_FAULT = -77; +static int logged; +void logf(const char *, ...) { ++logged; } static void fault_now() { *(volatile int *)0x40 = 1; } VkResult vkCreateComputePipelines(uintptr_t device, VkPipelineCache cache, unsigned count, const VkComputePipelineCreateInfo *ci, const void *allocator, VkPipeline *out) { From 4f98383d1a741e4ef3c9c608e9a224ed8ec3fdd7 Mon Sep 17 00:00:00 2001 From: Zhafron Date: Mon, 9 Nov 2026 12:51:09 +0700 Subject: [PATCH 4/5] fix(vk): catch SIGILL and SIGFPE in the Linux driver-fault guard The Windows branch wraps the pipeline calls in __try and catches every violation. The signal guard took only SIGSEGV and SIGBUS, so a driver that faults with an illegal instruction or an arithmetic trap still ended the game. The host's own handler (host/posix/win32_posix.c) already translates all four, so guard the same four and keep the previous handler per signal. A sigaction that fails used to leave the process unguarded for that signal with nothing in the log. --- gpu/vk/vk_draw.cpp | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/gpu/vk/vk_draw.cpp b/gpu/vk/vk_draw.cpp index 0322807ab..f3bd21ed9 100644 --- a/gpu/vk/vk_draw.cpp +++ b/gpu/vk/vk_draw.cpp @@ -744,15 +744,23 @@ __declspec(noinline) VkResult guarded_compute(VkPipelineCache cache, const VkCom // outside it goes to whoever had the signal before, so the game's own crash report is unchanged) static thread_local sigjmp_buf fault_jmp; static thread_local volatile sig_atomic_t fault_armed; -static struct sigaction fault_previous[2]; -static const int fault_signals[2] = {SIGSEGV, SIGBUS}; +static const int fault_signals[] = {SIGSEGV, SIGBUS, SIGILL, SIGFPE}; +static constexpr int fault_signal_count = (int)(sizeof fault_signals / sizeof fault_signals[0]); +static struct sigaction fault_previous[fault_signal_count]; + +static int fault_slot(int sig) { + for (int i = 0; i < fault_signal_count; i++) { + if (fault_signals[i] == sig) return i; + } + return 0; +} static void on_driver_fault(int sig, siginfo_t* si, void* ctx) { if (fault_armed) { fault_armed = 0; siglongjmp(fault_jmp, 1); } - const struct sigaction* p = &fault_previous[sig == SIGSEGV ? 0 : 1]; + const struct sigaction* p = &fault_previous[fault_slot(sig)]; if ((p->sa_flags & SA_SIGINFO) && p->sa_sigaction) { p->sa_sigaction(sig, si, ctx); } else if (p->sa_handler && p->sa_handler != SIG_IGN && p->sa_handler != SIG_DFL) { @@ -765,13 +773,15 @@ static void on_driver_fault(int sig, siginfo_t* si, void* ctx) { static void install_driver_fault_guard() { static std::once_flag once; std::call_once(once, [] { - for (int i = 0; i < 2; i++) { + for (int i = 0; i < fault_signal_count; i++) { struct sigaction sa; memset(&sa, 0, sizeof sa); sa.sa_sigaction = on_driver_fault; sa.sa_flags = SA_SIGINFO; sigemptyset(&sa.sa_mask); - sigaction(fault_signals[i], &sa, &fault_previous[i]); + if (sigaction(fault_signals[i], &sa, &fault_previous[i]) != 0) { + logf("vk: the driver fault guard could not take signal %d: a fault in it ends the game", fault_signals[i]); + } } }); } From bad04d0531522058ad02f1d7e1c045221167bcaf Mon Sep 17 00:00:00 2001 From: Zhafron Date: Mon, 9 Nov 2026 12:51:09 +0700 Subject: [PATCH 5/5] test(vk): drive the guard's signal chain and the other two signals A fault outside the window has to reach whatever handler held the signal before the guard. A guarded compile now runs first, so that handler is the guard's predecessor rather than the default, and the case fails unless it runs. The fake driver can also raise SIGILL and SIGFPE, so the wider signal set is covered instead of assumed. --- tests/test_linux_driver_fault_guard.py | 47 ++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/tests/test_linux_driver_fault_guard.py b/tests/test_linux_driver_fault_guard.py index e884b42dc..be70470bf 100644 --- a/tests/test_linux_driver_fault_guard.py +++ b/tests/test_linux_driver_fault_guard.py @@ -25,6 +25,7 @@ #include #include #include +#include using VkResult = int; using VkPipeline = uintptr_t; using VkPipelineCache = uintptr_t; @@ -34,17 +35,32 @@ struct VkComputePipelineCreateInfo { int sType = 0; }; struct VkGraphicsPipelineCreateInfo { int sType = 0; }; struct { uintptr_t device = 0x1234; } dev; -static int mode; /* (0: compile, 1 and 2: the driver faults) */ +static int mode; /* (0: compile, 1 to 4: the driver faults) */ static int calls; static constexpr VkResult DRIVER_FAULT = -77; static int logged; void logf(const char *, ...) { ++logged; } static void fault_now() { *(volatile int *)0x40 = 1; } +static void previous_handler(int, siginfo_t *, void *) { + static const char line[] = "previous handler ran\n"; + (void)!::write(1, line, sizeof line - 1); + _exit(42); +} +static void install_previous() { + struct sigaction sa; + memset(&sa, 0, sizeof sa); + sa.sa_sigaction = previous_handler; + sa.sa_flags = SA_SIGINFO; + sigemptyset(&sa.sa_mask); + sigaction(SIGSEGV, &sa, nullptr); +} VkResult vkCreateComputePipelines(uintptr_t device, VkPipelineCache cache, unsigned count, const VkComputePipelineCreateInfo *ci, const void *allocator, VkPipeline *out) { if (device != dev.device || cache != 0 || count != 1 || !ci || allocator || !out) return -1; ++calls; if (mode == 1) fault_now(); + if (mode == 3) raise(SIGILL); + if (mode == 4) raise(SIGFPE); *out = 0xcafe; return VK_SUCCESS; } @@ -61,10 +77,17 @@ MAIN = r""" int main(int argc, char **argv) { const char *what = argc > 1 ? argv[1] : "ok"; - if (!std::strcmp(what, "fault-outside")) { fault_now(); std::printf("fail: the fault returned\n"); return 8; } VkComputePipelineCreateInfo ci; VkGraphicsPipelineCreateInfo gci; VkPipeline pl = 0, gpl = 0; + if (!std::strcmp(what, "fault-outside")) { fault_now(); std::printf("fail: the fault returned\n"); return 8; } + if (!std::strcmp(what, "fault-outside-after-guard")) { + install_previous(); + if (guarded_compute(0, &ci, &pl) != VK_SUCCESS || pl != 0xcafe) { std::printf("fail: the compile before the fault\n"); return 10; } + fault_now(); + std::printf("fail: the fault returned\n"); + return 8; + } if (!std::strcmp(what, "ok")) { if (guarded_compute(0, &ci, &pl) != VK_SUCCESS || pl != 0xcafe) { std::printf("fail: compute\n"); return 3; } if (guarded_graphics(0, &gci, &gpl) != VK_SUCCESS || gpl != 0xbeef) { std::printf("fail: graphics\n"); return 4; } @@ -79,6 +102,14 @@ mode = 2; if (guarded_graphics(0, &gci, &gpl) != VK_ERROR_UNKNOWN) { std::printf("fail: the fault was not reported\n"); return 5; } if (gpl != VK_NULL_HANDLE) { std::printf("fail: a pipeline was published\n"); return 6; } + } else if (!std::strcmp(what, "fault-signals")) { + for (mode = 3; mode <= 4; mode++) { + pl = 0; + if (guarded_compute(0, &ci, &pl) != DRIVER_FAULT || pl != VK_NULL_HANDLE) { + std::printf("fail: the signal that mode %d raised was not reported\n", mode); + return 5; + } + } } else { std::printf("fail: unknown case %s\n", what); return 9; @@ -120,11 +151,15 @@ def main(): stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) if build.returncode: raise SystemExit("the guard did not compile:\n" + build.stdout) - for case in ("ok", "fault", "fault-graphics", "fault-outside"): + for case in ("ok", "fault", "fault-graphics", "fault-signals", "fault-outside", "fault-outside-after-guard"): run = subprocess.run([str(binary), case], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) - killed = run.returncode == -signal.SIGSEGV - guarded_case = case in ("fault", "fault-graphics") - if case == "fault-outside" or (args.expect_fault and guarded_case): + killed = run.returncode in (-signal.SIGSEGV, -signal.SIGBUS, -signal.SIGILL, -signal.SIGFPE) + guarded_case = case in ("fault", "fault-graphics", "fault-signals") + if case == "fault-outside-after-guard": + if run.returncode != 42 or "previous handler ran" not in run.stdout: + failures.append(case + ": the fault outside the window did not reach the handler it found: " + + run.stdout.strip() + " (exit " + str(run.returncode) + ")") + elif case == "fault-outside" or (args.expect_fault and guarded_case): if not killed: failures.append(case + ": the fault did not end the process: " + run.stdout.strip()) elif run.returncode != 0: