cesnimda
19c5251612
Run untrusted document decoders in a secret-free, resource-bounded child process and terminate its process tree on deadline. Harden the production container and enforce parser and lint gates in CI.
289 lines
14 KiB
YAML
289 lines
14 KiB
YAML
name: CI and Deploy
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- main
|
|
pull_request:
|
|
|
|
jobs:
|
|
test:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Setup .NET (resilient)
|
|
shell: bash
|
|
# actions/setup-dotnet on this single self-hosted runner intermittently
|
|
# leaves a partial extraction in the shared tool-cache ("tar: Cannot open:
|
|
# File exists") or corrupts the SDK download. Install into a clean private
|
|
# dir via dotnet-install.sh and retry once on failure, mirroring the
|
|
# npm ci / NuGet retries elsewhere in this workflow.
|
|
run: |
|
|
export DOTNET_ROOT="$HOME/.dotnet"
|
|
export PATH="$DOTNET_ROOT:$PATH"
|
|
|
|
validate_sdk() {
|
|
[ -x "$DOTNET_ROOT/dotnet" ] \
|
|
&& "$DOTNET_ROOT/dotnet" --list-sdks | grep -q '^9\.' \
|
|
&& "$DOTNET_ROOT/dotnet" --info >/dev/null
|
|
}
|
|
|
|
if validate_sdk; then
|
|
echo "Reusing valid runner-local .NET 9 SDK."
|
|
else
|
|
install() {
|
|
local feed="${1:-}"
|
|
local -a feed_args=()
|
|
if [ -n "$feed" ]; then
|
|
feed_args=(--azure-feed "$feed")
|
|
fi
|
|
curl -fsSL https://dot.net/v1/dotnet-install.sh -o /tmp/dotnet-install.sh
|
|
rm -rf "$DOTNET_ROOT"
|
|
bash /tmp/dotnet-install.sh --channel 9.0 --install-dir "$DOTNET_ROOT" "${feed_args[@]}"
|
|
validate_sdk
|
|
}
|
|
install || ( echo "dotnet install failed ($?) — retrying from alternate Microsoft CDN..." \
|
|
&& install https://dotnetcli.azureedge.net/dotnet )
|
|
fi
|
|
echo "$DOTNET_ROOT" >> "$GITHUB_PATH"
|
|
"$DOTNET_ROOT/dotnet" --info
|
|
|
|
- name: Setup Node
|
|
uses: actions/setup-node@v4
|
|
with:
|
|
node-version: '20'
|
|
|
|
- name: Test repository safety scripts
|
|
# Standard-library only and plan-only: this validates the synthetic benchmark harness
|
|
# without contacting Ollama, pulling a model, or requiring package installation.
|
|
run: python3 scripts/test-ollama-evaluation.py
|
|
|
|
- name: Test document parser boundary
|
|
working-directory: tools/summarizer
|
|
run: |
|
|
python3 -m venv .venv-ci
|
|
. .venv-ci/bin/activate
|
|
python -m pip install --upgrade pip
|
|
python -m pip install --require-hashes \
|
|
--extra-index-url https://download.pytorch.org/whl/cpu \
|
|
-r requirements-linux.lock
|
|
python -m pip install pytest==8.3.5 httpx2==2.12.0
|
|
AI_SERVICE_SKIP_MODEL_LOAD=1 python -m pytest -q
|
|
|
|
- name: Restore backend
|
|
# The runner/proxy has occasionally supplied checksum-invalid NuGet cache entries (NU3008).
|
|
# Retry from clean HTTP/global caches; signature verification remains enabled.
|
|
run: |
|
|
restore() { dotnet restore JobTrackerApi/JobTrackerApi.csproj; }
|
|
restore || ( echo "Backend restore failed ($?) — clearing runner NuGet caches and retrying once..." \
|
|
&& dotnet nuget locals http-cache --clear \
|
|
&& dotnet nuget locals global-packages --clear \
|
|
&& dotnet restore JobTrackerApi/JobTrackerApi.csproj --no-cache )
|
|
|
|
- name: Build backend
|
|
# Roslyn has intermittently exited 139 on this resource-constrained runner. Retry once
|
|
# without compiler-server or parallel build processes; restore failures are handled above.
|
|
run: |
|
|
dotnet build JobTrackerApi/JobTrackerApi.csproj --configuration Release --no-restore \
|
|
|| ( echo "Backend compiler crashed ($?) — retrying single-process..." \
|
|
&& dotnet build JobTrackerApi/JobTrackerApi.csproj --configuration Release \
|
|
--no-restore --disable-build-servers --maxcpucount:1 \
|
|
/p:UseSharedCompilation=false /p:BuildInParallel=false )
|
|
|
|
# The "Build backend" step above only builds JobTrackerApi, so the test project needs its own
|
|
# restore + build. These are separate steps on purpose: this runner is flaky (see the SDK and
|
|
# npm retries in this file) and the job log is not readable from the API, so distinct steps make
|
|
# the failing phase identifiable from step boundaries alone. Restore retries once, mirroring the
|
|
# npm ci pattern below. The suite itself is never weakened or filtered.
|
|
- name: Restore backend tests
|
|
run: |
|
|
restore() { dotnet restore JobTrackerApi.Tests/JobTrackerApi.Tests.csproj; }
|
|
restore || ( echo "NuGet restore failed ($?) — clearing runner caches and retrying once..." \
|
|
&& dotnet nuget locals http-cache --clear \
|
|
&& dotnet nuget locals global-packages --clear \
|
|
&& dotnet restore JobTrackerApi.Tests/JobTrackerApi.Tests.csproj --no-cache )
|
|
|
|
- name: Build backend tests
|
|
run: |
|
|
dotnet build JobTrackerApi.Tests/JobTrackerApi.Tests.csproj --configuration Release --no-restore \
|
|
|| ( echo "Test compiler crashed ($?) — retrying single-process..." \
|
|
&& dotnet build JobTrackerApi.Tests/JobTrackerApi.Tests.csproj --configuration Release \
|
|
--no-restore --disable-build-servers --maxcpucount:1 \
|
|
/p:UseSharedCompilation=false /p:BuildInParallel=false )
|
|
|
|
# Restore and build both pass on this runner, but the test run dies after ~3s -- far too fast to
|
|
# have executed 306 tests. The suite passes on Windows, in a clean Linux container, under a 1GB
|
|
# memory cap, and with the SDK installed to a custom dir without DOTNET_ROOT, so the cause is
|
|
# specific to this runner. This one-test smoke separates "the test host cannot start at all"
|
|
# from "something in the suite takes the host down"; the log is not readable via the API, so the
|
|
# step boundary is the signal.
|
|
- name: Audit backend dependencies
|
|
run: dotnet list JobTrackerApi/JobTrackerApi.csproj package --vulnerable --include-transitive
|
|
|
|
- name: Test backend (host smoke)
|
|
run: dotnet test JobTrackerApi.Tests/JobTrackerApi.Tests.csproj --configuration Release --no-build --filter "FullyQualifiedName~CvBuilderTests.Every_catalog_theme_renders_valid_html"
|
|
|
|
# This runner fails at a DIFFERENT stage on different runs with no readable diagnostics
|
|
# (test host at 3s; on another run `dotnet restore` at 0s). See
|
|
# docs/infrastructure/runner-investigation.md. Collection parallelism stays off for
|
|
# determinism, the same reason the frontend runs --runInBand. Every test runs; nothing skipped.
|
|
- name: Test backend
|
|
run: dotnet test JobTrackerApi.Tests/JobTrackerApi.Tests.csproj --configuration Release --no-build -- xUnit.parallelizeTestCollections=false xUnit.maxParallelThreads=1
|
|
|
|
- name: Install frontend deps
|
|
working-directory: job-tracker-ui
|
|
env:
|
|
npm_config_audit: 'false'
|
|
npm_config_fund: 'false'
|
|
run: |
|
|
node -v
|
|
npm -v
|
|
# The runner's shared npm cache has returned checksum-invalid tarballs and
|
|
# injected control bytes into installed JS. Give each attempt an isolated
|
|
# cache so a poisoned archive cannot survive into this checkout or retry.
|
|
npm_cache="$(mktemp -d)"
|
|
npm ci --no-audit --no-fund --cache "$npm_cache" \
|
|
|| ( echo "npm ci failed ($?) — cleaning node_modules and retrying once..." \
|
|
&& rm -rf node_modules \
|
|
&& npm_cache="$(mktemp -d)" \
|
|
&& npm ci --no-audit --no-fund --cache "$npm_cache" )
|
|
|
|
# Moderate React Router advisories remain documented and mitigated; high/critical findings
|
|
# in either production or test/browser tooling block the build.
|
|
- name: Audit frontend dependencies
|
|
working-directory: job-tracker-ui
|
|
run: npm audit --audit-level=high
|
|
|
|
- name: Lint frontend
|
|
working-directory: job-tracker-ui
|
|
run: npm run lint
|
|
|
|
- name: Test frontend
|
|
working-directory: job-tracker-ui
|
|
# Run the WHOLE suite. Never whitelist test files here again: the previous
|
|
# whitelist silently skipped new suites and let two regressions reach main.
|
|
# The self-hosted runner can also terminate Node with SIGSEGV (139) without a
|
|
# Jest assertion or diagnostic. Retry only that infrastructure signature once;
|
|
# ordinary assertion/configuration failures still fail immediately.
|
|
run: |
|
|
set +e
|
|
npm test -- --watchAll=false --runInBand
|
|
test_status=$?
|
|
set -e
|
|
|
|
if [ "$test_status" -eq 0 ]; then
|
|
exit 0
|
|
fi
|
|
if [ "$test_status" -ne 139 ]; then
|
|
exit "$test_status"
|
|
fi
|
|
|
|
echo "Frontend test process crashed with SIGSEGV — retrying the complete suite once..."
|
|
npm test -- --watchAll=false --runInBand
|
|
|
|
- name: Install browser smoke runtime
|
|
working-directory: job-tracker-ui
|
|
run: npx playwright install --with-deps chromium
|
|
|
|
- name: Test browser smoke flows
|
|
working-directory: job-tracker-ui
|
|
env:
|
|
# Exercise the deployable frontend and avoid Turbopack's persistence crashes on
|
|
# this runner's slow Docker volume.
|
|
PLAYWRIGHT_STATIC_EXPORT: 'true'
|
|
# This self-hosted runner has twice corrupted Playwright's installed JS/browser
|
|
# payload (invalid control bytes followed by SIGSEGV/139). Retry the complete
|
|
# suite once from a verified, clean install. The retry is intentionally bounded;
|
|
# a genuine or repeated browser-test failure still blocks deployment.
|
|
run: |
|
|
if npm run test:e2e; then
|
|
exit 0
|
|
else
|
|
smoke_status=$?
|
|
fi
|
|
|
|
echo "Browser smoke failed ($smoke_status) — rebuilding the browser runtime and retrying once..."
|
|
rm -rf node_modules
|
|
recovery_cache="$(mktemp -d)"
|
|
npm ci --no-audit --no-fund --cache "$recovery_cache"
|
|
npx playwright install --with-deps --force chromium
|
|
npm run test:e2e
|
|
|
|
- name: Build frontend
|
|
working-directory: job-tracker-ui
|
|
env:
|
|
CI: 'false'
|
|
GENERATE_SOURCEMAP: 'false'
|
|
NODE_OPTIONS: --max-old-space-size=4096
|
|
# CRA's build (Terser minify + fork-ts-checker workers) has repeatedly died silently on
|
|
# this runner with no error output (OOM/SIGSEGV signature — same resource-starved-runner
|
|
# class as the npm ci and dotnet-install flakes elsewhere in this workflow). Retry once.
|
|
run: |
|
|
npm run build \
|
|
|| ( echo "Frontend build failed ($?) — retrying once..." && npm run build )
|
|
|
|
deploy:
|
|
needs: test
|
|
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Run remote deploy
|
|
uses: appleboy/ssh-action@v1.0.3
|
|
with:
|
|
host: ${{ secrets.PROD_HOST }}
|
|
username: ${{ secrets.PROD_USER }}
|
|
key: ${{ secrets.PROD_SSH_KEY }}
|
|
command_timeout: 40m
|
|
script: |
|
|
set -euo pipefail
|
|
if [ ! -d /opt/job-tracker/app/.git ]; then
|
|
echo "Expected git checkout at /opt/job-tracker/app but .git was not found."
|
|
exit 1
|
|
fi
|
|
cd /opt/job-tracker/app
|
|
if ! git fetch --all --prune; then
|
|
echo "git fetch failed on server. Check remote auth/URL for /opt/job-tracker/app."
|
|
exit 1
|
|
fi
|
|
if ! git rev-parse --verify --quiet ${{ github.sha }} >/dev/null; then
|
|
echo "Commit ${{ github.sha }} is not available in the server checkout after fetch."
|
|
exit 1
|
|
fi
|
|
git reset --hard ${{ github.sha }}
|
|
git clean -fd
|
|
chmod +x deploy/deploy.sh
|
|
DEPLOY_BUILD_AI_SERVICE=true \
|
|
APP_VERSION=${{ github.run_number }} \
|
|
APP_COMMIT_SHA=${{ github.sha }} \
|
|
APP_BUILD_STAMP="$(date -u +'%Y-%m-%d %H:%M UTC')" \
|
|
./deploy/deploy.sh
|
|
docker compose -f docker-compose.yml ps
|
|
AI_CONTAINER_ID="$(docker compose -f docker-compose.yml ps -q ai-service)"
|
|
if [ -z "$AI_CONTAINER_ID" ]; then
|
|
echo "AI service container id could not be resolved after deploy. Continuing because AI is not a deploy gate for the core app."
|
|
else
|
|
ATTEMPTS=90
|
|
SLEEP_SECS=2
|
|
i=1
|
|
while [ "$i" -le "$ATTEMPTS" ]; do
|
|
HEALTH_STATUS="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$AI_CONTAINER_ID" 2>/dev/null || echo unknown)"
|
|
if [ "$HEALTH_STATUS" = "healthy" ]; then
|
|
break
|
|
fi
|
|
if [ "$HEALTH_STATUS" = "unhealthy" ]; then
|
|
echo "AI service became unhealthy during deploy readiness wait. Continuing because AI is not a deploy gate for the core app."
|
|
docker compose -f docker-compose.yml logs --tail=200 ai-service || true
|
|
break
|
|
fi
|
|
sleep "$SLEEP_SECS"
|
|
i=$((i + 1))
|
|
done
|
|
if [ "${HEALTH_STATUS:-unknown}" != "healthy" ]; then
|
|
echo "AI service did not become healthy within $((ATTEMPTS * SLEEP_SECS)) seconds. Final status: ${HEALTH_STATUS:-unknown}. Continuing because AI is not a deploy gate for the core app."
|
|
docker compose -f docker-compose.yml ps
|
|
docker compose -f docker-compose.yml logs --tail=200 ai-service || true
|
|
fi
|
|
fi
|