CI: Stacktraces of pytest and xdist-workers on timeout

justfile

@@ -53,7 +53,7 @@ test-integration *TEST_ARGS: build-bin build-python
   cd libs/opsqueue_python
   source "./.setup_local_venv.sh"
 
-  timeout {{pytest_timeout_seconds}} pytest --color=yes {{TEST_ARGS}}
+  timeout --signal term --kill-after 1s {{pytest_timeout_seconds}} pytest --color=yes {{TEST_ARGS}}
 
 # Python integration test suite, using artefacts built through Nix. Args are forwarded to pytest
 [group('nix')]
@@ -68,7 +68,7 @@ nix-test-integration *TEST_ARGS: nix-build
   export OPSQUEUE_BIN="${nix_build_bin_dir}/bin/opsqueue"
   export RUST_LOG="opsqueue=debug"
 
-  timeout {{pytest_timeout_seconds}} pytest --color=yes {{TEST_ARGS}}
+  timeout --signal term --kill-after 1s {{pytest_timeout_seconds}} pytest --color=yes {{TEST_ARGS}}
 
 # Run all linters, fast and slow
 [group('lint')]

libs/opsqueue_python/tests/conftest.py

@@ -1,5 +1,9 @@
 import cbor2
 import pickle
+import faulthandler
+import signal
+import sys
+import time
 from contextlib import contextmanager, ExitStack
 from typing import Generator, Callable, Any, Iterable
 import multiprocess  # type: ignore[import-untyped]
@@ -14,9 +18,119 @@
 from opsqueue.consumer import Strategy
 
 
+_previous_sigterm_handler: Any = None
+_sigusr1_dump_file: Any | None = None
+
+
+def _stack_dump_path(pid: int) -> Path:
+    return Path(f"/tmp/opsqueue-pytest-stack-{pid}.log")
+
+
+def _linux_child_pids(parent_pid: int) -> set[int]:
+    children_file = Path(f"/proc/{parent_pid}/task/{parent_pid}/children")
+    try:
+        raw = children_file.read_text(encoding="utf-8").strip()
+    except FileNotFoundError:
+        return set()
+
+    if not raw:
+        return set()
+
+    return {int(pid_text) for pid_text in raw.split()}
+
+
+def _dump_xdist_worker_traces() -> None:
+    worker_pids = _linux_child_pids(os.getpid())
+    if not worker_pids:
+        print(
+            "[opsqueue pytest] No child worker PIDs found at SIGTERM time",
+            file=sys.stderr,
+            flush=True,
+        )
+        return
+
+    print(
+        f"[opsqueue pytest] Requesting stack dump from child workers: {worker_pids}",
+        file=sys.stderr,
+        flush=True,
+    )
+    for pid in worker_pids:
+        try:
+            os.kill(pid, signal.SIGUSR1)
+        except ProcessLookupError:
+            # Worker already exited.
+            continue
+        except Exception as exc:  # pragma: no cover - defensive diagnostics
+            print(
+                f"[opsqueue pytest] Failed sending SIGUSR1 to child pid={pid}: {exc}",
+                file=sys.stderr,
+                flush=True,
+            )
+
+    # Give workers a short moment to flush their dump files before reading them.
+    time.sleep(0.15)
+    for pid in sorted(worker_pids):
+        dump_path = _stack_dump_path(pid)
+        if not dump_path.exists():
+            print(
+                f"[opsqueue pytest] No SIGUSR1 stack dump file found for child pid={pid}: {dump_path}",
+                file=sys.stderr,
+                flush=True,
+            )
+            continue
+
+        print(
+            f"[opsqueue pytest] ===== BEGIN CHILD STACK DUMP pid={pid} ({dump_path}) =====",
+            file=sys.stderr,
+            flush=True,
+        )
+        try:
+            print(dump_path.read_text(encoding="utf-8"), file=sys.stderr, flush=True)
+        except Exception as exc:  # pragma: no cover - defensive diagnostics
+            print(
+                f"[opsqueue pytest] Failed reading dump file for child pid={pid}: {exc}",
+                file=sys.stderr,
+                flush=True,
+            )
+        print(
+            f"[opsqueue pytest] ===== END CHILD STACK DUMP pid={pid} =====",
+            file=sys.stderr,
+            flush=True,
+        )
+
+
+def _handle_sigterm(signum: int, frame: object) -> None:
+    faulthandler.dump_traceback(file=sys.stderr, all_threads=True)
+    _dump_xdist_worker_traces()
+
+    if callable(_previous_sigterm_handler):
+        _previous_sigterm_handler(signum, frame)
+    elif _previous_sigterm_handler == signal.SIG_DFL:
+        signal.signal(signal.SIGTERM, signal.SIG_DFL)
+        os.kill(os.getpid(), signal.SIGTERM)
+    # If previous handler was SIG_IGN, return and keep process alive.
+    # Timeout will eventually kill the process if it doesn't exit on its own.
+
+
 @pytest.hookimpl(tryfirst=True)
 def pytest_configure(config: pytest.Config) -> None:
-    print("A")
+    global _previous_sigterm_handler
+    global _sigusr1_dump_file
+
+    _sigusr1_dump_file = _stack_dump_path(os.getpid()).open("w", encoding="utf-8")
+
+    # Make SIGUSR1 print a traceback in any process (controller or worker).
+    faulthandler.register(
+        signal.SIGUSR1,
+        file=_sigusr1_dump_file,
+        all_threads=True,
+        chain=True,
+    )
+
+    # When `timeout` sends SIGTERM to the controller, request worker dumps too.
+    faulthandler.enable(all_threads=True)
+    _previous_sigterm_handler = signal.getsignal(signal.SIGTERM)
+    signal.signal(signal.SIGTERM, _handle_sigterm)
     multiprocess.set_start_method("forkserver")