From 8699ae4f421bb1d049b937e735e40a4797a4b719 Mon Sep 17 00:00:00 2001 From: cesnimda Date: Wed, 21 Oct 2026 07:13:04 +0200 Subject: [PATCH] ci: retain native parser crash diagnostics --- .gitea/workflows/ci-deploy.yml | 3 +- .../2026-10-21-parser-native-exit.md | 50 +++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 docs/infrastructure/2026-10-21-parser-native-exit.md diff --git a/.gitea/workflows/ci-deploy.yml b/.gitea/workflows/ci-deploy.yml index cbab806..a6fa4b8 100644 --- a/.gitea/workflows/ci-deploy.yml +++ b/.gitea/workflows/ci-deploy.yml @@ -111,7 +111,8 @@ jobs: done test "$parser_install_succeeded" = true PIP_CACHE_DIR="$parser_pip_cache" python3 /tmp/pip.pyz install --upgrade --target .python-ci pytest==8.3.5 httpx2==2.12.0 - PYTHONPATH="$PWD/.python-ci" AI_SERVICE_SKIP_MODEL_LOAD=1 python3 -m pytest -q + python3 --version + PYTHONFAULTHANDLER=1 PYTHONPATH="$PWD/.python-ci" AI_SERVICE_SKIP_MODEL_LOAD=1 python3 -m pytest -q - name: Restore backend # The runner/proxy has occasionally supplied checksum-invalid NuGet cache entries (NU3008). diff --git a/docs/infrastructure/2026-10-21-parser-native-exit.md b/docs/infrastructure/2026-10-21-parser-native-exit.md new file mode 100644 index 0000000..10d9cd4 --- /dev/null +++ b/docs/infrastructure/2026-10-21-parser-native-exit.md @@ -0,0 +1,50 @@ +# Parser CI native exit investigation, 2026-10-21 + +## Verified evidence + +CI run 834, API test job 1721, failed in `Test document parser boundary`. +The fetched log is `.artifacts/phase7/p7-09/instrumented-release-ci/20261021T050601Z/failed-job-logs.log`; +the adjacent `api-observation.json` records the GET of job 1721's logs. +The runner's internal task ID 1689 is separate from the API job ID. + +At 2026-10-20 20:36:38 UTC, pytest reported `56 passed, 1 warning in 5.36s`. +The shell subsequently reported a segmentation fault and exit 139. Passing assertions +did not constitute successful process completion. Later browser steps did not run; +deployment remained blocked. + +Read-only host inspection with +`coredumpctl --no-pager info 3804681` confirmed: + +- Signal: 11 (SIGSEGV). +- Command: `python3 -m pytest -q`. +- Executable: `/usr/bin/python3.11`. +- Timestamp: 2026-10-20 22:36:38 CEST (20:36:38 UTC). +- Docker scope: `040c0b99c92e0a5a48c00994e10a264884ca8172f5eac94d49e3d7aa1d92fd27`, + matching the failed job's container. +- Stored core: `/var/lib/systemd/coredump/core.python3.0.7aab2b376ce94d92bf796bbeb62063a4.3804681.1792528598000000.zst`. + +The core was inaccessible to SSH user `pi`; `sudo -n` required a password. +No native backtrace was obtained. Kernel journal error entries for +20:30-20:40 UTC were empty; that does not establish host health. + +## Interpretation and bounded diagnostic + +The timing is consistent with interpreter shutdown or cleanup, but the root cause +is unproven. `AI_SERVICE_SKIP_MODEL_LOAD=1` prevents model weight loading; +`app.py` still imports Torch and Transformers. Native extension cleanup is therefore +a candidate, not an identified cause. Previous download corruption and browser +crashes do not prove the cause of this Python crash. + +The workflow now prints the Python version and enables `PYTHONFAULTHANDLER=1` +for the unchanged pytest command. It preserves the existing dependency installs, +all tests, and process exit status, without adding retries or suppressing signals. +The existing pip log already records installed distribution versions. In this run, +the test-runtime install upgraded locked AnyIO 4.14.2 to 4.15.1 and IDNA 3.19 to +3.20; this is a reproducibility observation, not a demonstrated cause, and no +dependency change is included here. + +Next evidence: inspect the retained core with authorized access, or capture the +faulthandler output from a subsequent normal CI run before choosing a fix. +The diagnostic change has only received workflow text and whitespace checks; +it has not been exercised in CI. No production mutation, model qualification, +or successful release is claimed.