diff --git a/justfile b/justfile index 0cba2f3..b1027d8 100644 --- a/justfile +++ b/justfile @@ -53,7 +53,7 @@ test-integration *TEST_ARGS: build-bin build-python cd libs/opsqueue_python source "./.setup_local_venv.sh" - timeout {{pytest_timeout_seconds}} pytest --color=yes {{TEST_ARGS}} + timeout --signal term --kill-after 1s {{pytest_timeout_seconds}} pytest --color=yes {{TEST_ARGS}} # Python integration test suite, using artefacts built through Nix. Args are forwarded to pytest [group('nix')] @@ -68,7 +68,7 @@ nix-test-integration *TEST_ARGS: nix-build export OPSQUEUE_BIN="${nix_build_bin_dir}/bin/opsqueue" export RUST_LOG="opsqueue=debug" - timeout {{pytest_timeout_seconds}} pytest --color=yes {{TEST_ARGS}} + timeout --signal term --kill-after 1s {{pytest_timeout_seconds}} pytest --color=yes {{TEST_ARGS}} # Run all linters, fast and slow [group('lint')] diff --git a/libs/opsqueue_python/tests/conftest.py b/libs/opsqueue_python/tests/conftest.py index 0052bca..c8778a7 100644 --- a/libs/opsqueue_python/tests/conftest.py +++ b/libs/opsqueue_python/tests/conftest.py @@ -1,5 +1,9 @@ import cbor2 import pickle +import faulthandler +import signal +import sys +import time from contextlib import contextmanager, ExitStack from typing import Generator, Callable, Any, Iterable import multiprocess # type: ignore[import-untyped] @@ -14,9 +18,119 @@ from opsqueue.consumer import Strategy +_previous_sigterm_handler: Any = None +_sigusr1_dump_file: Any | None = None + + +def _stack_dump_path(pid: int) -> Path: + return Path(f"/tmp/opsqueue-pytest-stack-{pid}.log") + + +def _linux_child_pids(parent_pid: int) -> set[int]: + children_file = Path(f"/proc/{parent_pid}/task/{parent_pid}/children") + try: + raw = children_file.read_text(encoding="utf-8").strip() + except FileNotFoundError: + return set() + + if not raw: + return set() + + return {int(pid_text) for pid_text in raw.split()} + + +def _dump_xdist_worker_traces() -> None: + worker_pids = _linux_child_pids(os.getpid()) + if not worker_pids: + print( + "[opsqueue pytest] No child worker PIDs found at SIGTERM time", + file=sys.stderr, + flush=True, + ) + return + + print( + f"[opsqueue pytest] Requesting stack dump from child workers: {worker_pids}", + file=sys.stderr, + flush=True, + ) + for pid in worker_pids: + try: + os.kill(pid, signal.SIGUSR1) + except ProcessLookupError: + # Worker already exited. + continue + except Exception as exc: # pragma: no cover - defensive diagnostics + print( + f"[opsqueue pytest] Failed sending SIGUSR1 to child pid={pid}: {exc}", + file=sys.stderr, + flush=True, + ) + + # Give workers a short moment to flush their dump files before reading them. + time.sleep(0.15) + for pid in sorted(worker_pids): + dump_path = _stack_dump_path(pid) + if not dump_path.exists(): + print( + f"[opsqueue pytest] No SIGUSR1 stack dump file found for child pid={pid}: {dump_path}", + file=sys.stderr, + flush=True, + ) + continue + + print( + f"[opsqueue pytest] ===== BEGIN CHILD STACK DUMP pid={pid} ({dump_path}) =====", + file=sys.stderr, + flush=True, + ) + try: + print(dump_path.read_text(encoding="utf-8"), file=sys.stderr, flush=True) + except Exception as exc: # pragma: no cover - defensive diagnostics + print( + f"[opsqueue pytest] Failed reading dump file for child pid={pid}: {exc}", + file=sys.stderr, + flush=True, + ) + print( + f"[opsqueue pytest] ===== END CHILD STACK DUMP pid={pid} =====", + file=sys.stderr, + flush=True, + ) + + +def _handle_sigterm(signum: int, frame: object) -> None: + faulthandler.dump_traceback(file=sys.stderr, all_threads=True) + _dump_xdist_worker_traces() + + if callable(_previous_sigterm_handler): + _previous_sigterm_handler(signum, frame) + elif _previous_sigterm_handler == signal.SIG_DFL: + signal.signal(signal.SIGTERM, signal.SIG_DFL) + os.kill(os.getpid(), signal.SIGTERM) + # If previous handler was SIG_IGN, return and keep process alive. + # Timeout will eventually kill the process if it doesn't exit on its own. + + @pytest.hookimpl(tryfirst=True) def pytest_configure(config: pytest.Config) -> None: - print("A") + global _previous_sigterm_handler + global _sigusr1_dump_file + + _sigusr1_dump_file = _stack_dump_path(os.getpid()).open("w", encoding="utf-8") + + # Make SIGUSR1 print a traceback in any process (controller or worker). + faulthandler.register( + signal.SIGUSR1, + file=_sigusr1_dump_file, + all_threads=True, + chain=True, + ) + + # When `timeout` sends SIGTERM to the controller, request worker dumps too. + faulthandler.enable(all_threads=True) + _previous_sigterm_handler = signal.getsignal(signal.SIGTERM) + signal.signal(signal.SIGTERM, _handle_sigterm) multiprocess.set_start_method("forkserver")