cesnimda
328 lines
15 KiB
YAML
328 lines
15 KiB
YAML
name: CI and Deploy
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- main
|
|
pull_request:
|
|
|
|
jobs:
|
|
test:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Setup .NET (resilient)
|
|
shell: bash
|
|
# actions/setup-dotnet on this single self-hosted runner intermittently
|
|
# leaves a partial extraction in the shared tool-cache ("tar: Cannot open:
|
|
# File exists") or corrupts the SDK download. Install into a clean private
|
|
# dir via dotnet-install.sh and retry once on failure, mirroring the
|
|
# npm ci / NuGet retries elsewhere in this workflow.
|
|
run: |
|
|
export DOTNET_ROOT="$HOME/.dotnet"
|
|
export PATH="$DOTNET_ROOT:$PATH"
|
|
|
|
validate_sdk() {
|
|
[ -x "$DOTNET_ROOT/dotnet" ] \
|
|
&& "$DOTNET_ROOT/dotnet" --list-sdks | grep -q '^9\.0\.317 ' \
|
|
&& "$DOTNET_ROOT/dotnet" --info >/dev/null
|
|
}
|
|
|
|
if validate_sdk; then
|
|
echo "Reusing valid runner-local .NET 9 SDK."
|
|
else
|
|
install() {
|
|
local feed="${1:-}"
|
|
local -a feed_args=()
|
|
if [ -n "$feed" ]; then
|
|
feed_args=(--azure-feed "$feed")
|
|
fi
|
|
curl -fsSL https://dot.net/v1/dotnet-install.sh -o /tmp/dotnet-install.sh
|
|
rm -rf "$DOTNET_ROOT"
|
|
bash /tmp/dotnet-install.sh --version 9.0.317 --install-dir "$DOTNET_ROOT" "${feed_args[@]}"
|
|
validate_sdk
|
|
}
|
|
install || ( echo "dotnet install failed ($?) — retrying from alternate Microsoft CDN..." \
|
|
&& install https://dotnetcli.azureedge.net/dotnet )
|
|
fi
|
|
echo "$DOTNET_ROOT" >> "$GITHUB_PATH"
|
|
"$DOTNET_ROOT/dotnet" --info
|
|
|
|
- name: Setup Node
|
|
uses: actions/setup-node@v4
|
|
with:
|
|
node-version: '20'
|
|
|
|
- name: Test repository safety scripts
|
|
# Standard-library only and plan-only: this validates the synthetic benchmark harness
|
|
# without contacting Ollama, pulling a model, or requiring package installation.
|
|
run: |
|
|
python3 scripts/test-ollama-evaluation.py
|
|
python3 scripts/test-supply-chain.py
|
|
|
|
- name: Test browser extension
|
|
run: node --test browser-extension/tests/*.test.mjs
|
|
|
|
- name: Scan tracked files and generate dependency SBOM
|
|
run: |
|
|
python3 scripts/supply-chain.py scan-secrets
|
|
python3 scripts/supply-chain.py generate-sbom --output .artifacts/jobjakt.cdx.json
|
|
|
|
- name: Test document parser boundary
|
|
working-directory: tools/summarizer
|
|
run: |
|
|
# The Debian runner omits pip and python3-venv. Use pip's official
|
|
# zipapp and install into the checkout so CI remains isolated without
|
|
# sudo or a mutable runner-global Python environment.
|
|
curl -fsSLo /tmp/pip.pyz https://bootstrap.pypa.io/pip/pip.pyz
|
|
install_parser_dependencies() {
|
|
rm -rf .python-ci
|
|
parser_pip_cache="$(mktemp -d)"
|
|
PIP_CACHE_DIR="$parser_pip_cache" python3 /tmp/pip.pyz install --target .python-ci --require-hashes \
|
|
--extra-index-url https://download.pytorch.org/whl/cpu \
|
|
-r requirements-linux.lock
|
|
}
|
|
install_parser_dependencies \
|
|
|| ( echo "Parser dependency install failed ($?) — retrying with a fresh isolated cache..." \
|
|
&& install_parser_dependencies )
|
|
PIP_CACHE_DIR="$parser_pip_cache" python3 /tmp/pip.pyz install --upgrade --target .python-ci pytest==8.3.5 httpx2==2.12.0
|
|
PYTHONPATH="$PWD/.python-ci" AI_SERVICE_SKIP_MODEL_LOAD=1 python3 -m pytest -q
|
|
|
|
- name: Restore backend
|
|
# The runner/proxy has occasionally supplied checksum-invalid NuGet cache entries (NU3008).
|
|
# Retry from clean HTTP/global caches; signature verification remains enabled.
|
|
run: |
|
|
restore() { dotnet restore JobTrackerApi/JobTrackerApi.csproj --locked-mode; }
|
|
restore || ( echo "Backend restore failed ($?) — clearing runner NuGet caches and retrying once..." \
|
|
&& dotnet nuget locals http-cache --clear \
|
|
&& dotnet nuget locals global-packages --clear \
|
|
&& dotnet restore JobTrackerApi/JobTrackerApi.csproj --no-cache --locked-mode )
|
|
|
|
- name: Build backend
|
|
# Roslyn has intermittently exited 139 on this resource-constrained runner. Retry once
|
|
# without compiler-server or parallel build processes; restore failures are handled above.
|
|
run: |
|
|
dotnet build JobTrackerApi/JobTrackerApi.csproj --configuration Release --no-restore \
|
|
|| ( echo "Backend compiler crashed ($?) — retrying single-process..." \
|
|
&& dotnet build JobTrackerApi/JobTrackerApi.csproj --configuration Release \
|
|
--no-restore --disable-build-servers --maxcpucount:1 \
|
|
/p:UseSharedCompilation=false /p:BuildInParallel=false )
|
|
|
|
# The "Build backend" step above only builds JobTrackerApi, so the test project needs its own
|
|
# restore + build. These are separate steps on purpose: this runner is flaky (see the SDK and
|
|
# npm retries in this file) and the job log is not readable from the API, so distinct steps make
|
|
# the failing phase identifiable from step boundaries alone. Restore retries once, mirroring the
|
|
# npm ci pattern below. The suite itself is never weakened or filtered.
|
|
- name: Restore backend tests
|
|
run: |
|
|
restore() { dotnet restore JobTrackerApi.Tests/JobTrackerApi.Tests.csproj --locked-mode; }
|
|
restore || ( echo "NuGet restore failed ($?) — clearing runner caches and retrying once..." \
|
|
&& dotnet nuget locals http-cache --clear \
|
|
&& dotnet nuget locals global-packages --clear \
|
|
&& dotnet restore JobTrackerApi.Tests/JobTrackerApi.Tests.csproj --no-cache --locked-mode )
|
|
|
|
- name: Build backend tests
|
|
run: |
|
|
dotnet build JobTrackerApi.Tests/JobTrackerApi.Tests.csproj --configuration Release --no-restore \
|
|
|| ( echo "Test compiler crashed ($?) — retrying single-process..." \
|
|
&& dotnet build JobTrackerApi.Tests/JobTrackerApi.Tests.csproj --configuration Release \
|
|
--no-restore --disable-build-servers --maxcpucount:1 \
|
|
/p:UseSharedCompilation=false /p:BuildInParallel=false )
|
|
|
|
# Restore and build both pass on this runner, but the test run dies after ~3s -- far too fast to
|
|
# have executed 306 tests. The suite passes on Windows, in a clean Linux container, under a 1GB
|
|
# memory cap, and with the SDK installed to a custom dir without DOTNET_ROOT, so the cause is
|
|
# specific to this runner. This one-test smoke separates "the test host cannot start at all"
|
|
# from "something in the suite takes the host down"; the log is not readable via the API, so the
|
|
# step boundary is the signal.
|
|
- name: Audit backend dependencies
|
|
run: dotnet list JobTrackerApi/JobTrackerApi.csproj package --vulnerable --include-transitive
|
|
|
|
- name: Test backend (host smoke)
|
|
run: dotnet test JobTrackerApi.Tests/JobTrackerApi.Tests.csproj --configuration Release --no-build --filter "FullyQualifiedName~CvBuilderTests.Every_catalog_theme_renders_valid_html"
|
|
|
|
# This runner fails at a DIFFERENT stage on different runs with no readable diagnostics
|
|
# (test host at 3s; on another run `dotnet restore` at 0s). See
|
|
# docs/infrastructure/runner-investigation.md. Collection parallelism stays off for
|
|
# determinism, the same reason the frontend runs --runInBand. Every test runs; nothing skipped.
|
|
- name: Test backend
|
|
run: dotnet test JobTrackerApi.Tests/JobTrackerApi.Tests.csproj --configuration Release --no-build -- xUnit.parallelizeTestCollections=false xUnit.maxParallelThreads=1
|
|
|
|
- name: Install frontend deps
|
|
working-directory: job-tracker-ui
|
|
env:
|
|
npm_config_audit: 'false'
|
|
npm_config_fund: 'false'
|
|
run: |
|
|
node -v
|
|
npm -v
|
|
# The runner's shared npm cache has returned checksum-invalid tarballs and
|
|
# injected control bytes into installed JS. Give each attempt an isolated
|
|
# cache so a poisoned archive cannot survive into this checkout or retry.
|
|
npm_cache="$(mktemp -d)"
|
|
npm ci --no-audit --no-fund --cache "$npm_cache" \
|
|
|| ( echo "npm ci failed ($?) — cleaning node_modules and retrying once..." \
|
|
&& rm -rf node_modules \
|
|
&& npm_cache="$(mktemp -d)" \
|
|
&& npm ci --no-audit --no-fund --cache "$npm_cache" )
|
|
|
|
# Moderate React Router advisories remain documented and mitigated; high/critical findings
|
|
# in either production or test/browser tooling block the build.
|
|
- name: Audit frontend dependencies
|
|
working-directory: job-tracker-ui
|
|
run: npm audit --audit-level=high
|
|
|
|
- name: Lint frontend
|
|
working-directory: job-tracker-ui
|
|
run: npm run lint
|
|
|
|
- name: Test frontend
|
|
working-directory: job-tracker-ui
|
|
# Run the WHOLE suite. Never whitelist test files here again: the previous
|
|
# whitelist silently skipped new suites and let two regressions reach main.
|
|
# The self-hosted runner can also terminate Node with SIGSEGV (139) without a
|
|
# Jest assertion or diagnostic. Retry only that infrastructure signature once;
|
|
# ordinary assertion/configuration failures still fail immediately.
|
|
run: |
|
|
set +e
|
|
npm test -- --watchAll=false --runInBand
|
|
test_status=$?
|
|
set -e
|
|
|
|
if [ "$test_status" -eq 0 ]; then
|
|
exit 0
|
|
fi
|
|
if [ "$test_status" -ne 139 ]; then
|
|
exit "$test_status"
|
|
fi
|
|
|
|
echo "Frontend test process crashed with SIGSEGV — retrying the complete suite once..."
|
|
npm test -- --watchAll=false --runInBand
|
|
|
|
- name: Build frontend
|
|
working-directory: job-tracker-ui
|
|
env:
|
|
CI: 'false'
|
|
GENERATE_SOURCEMAP: 'false'
|
|
NODE_OPTIONS: --max-old-space-size=4096
|
|
# Build once, then make Playwright serve this exact export. A SIGSEGV-only retry preserves
|
|
# the runner workaround without hiding an ordinary compiler or type-check failure.
|
|
run: |
|
|
set +e
|
|
npm run build
|
|
build_status=$?
|
|
set -e
|
|
|
|
if [ "$build_status" -eq 0 ]; then
|
|
exit 0
|
|
fi
|
|
if [ "$build_status" -ne 139 ]; then
|
|
exit "$build_status"
|
|
fi
|
|
|
|
echo "Frontend build process crashed with SIGSEGV — retrying once..."
|
|
npm run build
|
|
|
|
- name: Install browser smoke runtime
|
|
working-directory: job-tracker-ui
|
|
env:
|
|
PLAYWRIGHT_DOWNLOAD_CONNECTION_TIMEOUT: '120000'
|
|
run: |
|
|
# Invoke the lockfile-installed CLI directly. `npx` may create a second
|
|
# temporary dependency tree when its cache is damaged, which made this
|
|
# runner's intermittent truncated downloads substantially worse.
|
|
node ./node_modules/@playwright/test/cli.js install --with-deps chromium
|
|
|
|
- name: Test browser smoke flows
|
|
working-directory: job-tracker-ui
|
|
env:
|
|
# Exercise the deployable frontend and avoid Turbopack's persistence crashes on
|
|
# this runner's slow Docker volume.
|
|
PLAYWRIGHT_STATIC_EXPORT: 'true'
|
|
PLAYWRIGHT_PREBUILT_EXPORT: 'true'
|
|
# The runner occasionally terminates an otherwise healthy Node process with
|
|
# SIGSEGV/139. Retry only that infrastructure signature. Do not reinstall npm
|
|
# or browser payloads here: doing so used to redownload hundreds of megabytes
|
|
# through the same unreliable transport and made a transient crash persistent.
|
|
# Assertion/configuration failures remain terminal and always block deployment.
|
|
run: |
|
|
set +e
|
|
npm run test:e2e
|
|
smoke_status=$?
|
|
set -e
|
|
|
|
if [ "$smoke_status" -eq 0 ]; then
|
|
exit 0
|
|
fi
|
|
if [ "$smoke_status" -ne 139 ]; then
|
|
exit "$smoke_status"
|
|
fi
|
|
|
|
echo "Browser smoke process crashed with SIGSEGV — retrying the unchanged suite once..."
|
|
npm run test:e2e
|
|
|
|
deploy:
|
|
needs: test
|
|
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Run remote deploy
|
|
uses: appleboy/ssh-action@v1.0.3
|
|
with:
|
|
host: ${{ secrets.PROD_HOST }}
|
|
username: ${{ secrets.PROD_USER }}
|
|
key: ${{ secrets.PROD_SSH_KEY }}
|
|
command_timeout: 40m
|
|
script: |
|
|
set -euo pipefail
|
|
if [ ! -d /opt/job-tracker/app/.git ]; then
|
|
echo "Expected git checkout at /opt/job-tracker/app but .git was not found."
|
|
exit 1
|
|
fi
|
|
cd /opt/job-tracker/app
|
|
if ! git fetch --all --prune; then
|
|
echo "git fetch failed on server. Check remote auth/URL for /opt/job-tracker/app."
|
|
exit 1
|
|
fi
|
|
if ! git rev-parse --verify --quiet ${{ github.sha }} >/dev/null; then
|
|
echo "Commit ${{ github.sha }} is not available in the server checkout after fetch."
|
|
exit 1
|
|
fi
|
|
git reset --hard ${{ github.sha }}
|
|
git clean -fd
|
|
chmod +x deploy/deploy.sh
|
|
DEPLOY_BUILD_AI_SERVICE=true \
|
|
APP_VERSION=${{ github.run_number }} \
|
|
APP_COMMIT_SHA=${{ github.sha }} \
|
|
APP_BUILD_STAMP="$(date -u +'%Y-%m-%d %H:%M UTC')" \
|
|
./deploy/deploy.sh
|
|
docker compose -f docker-compose.yml ps
|
|
AI_CONTAINER_ID="$(docker compose -f docker-compose.yml ps -q ai-service)"
|
|
if [ -z "$AI_CONTAINER_ID" ]; then
|
|
echo "AI service container id could not be resolved after deploy. Continuing because AI is not a deploy gate for the core app."
|
|
else
|
|
ATTEMPTS=90
|
|
SLEEP_SECS=2
|
|
i=1
|
|
while [ "$i" -le "$ATTEMPTS" ]; do
|
|
HEALTH_STATUS="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$AI_CONTAINER_ID" 2>/dev/null || echo unknown)"
|
|
if [ "$HEALTH_STATUS" = "healthy" ]; then
|
|
break
|
|
fi
|
|
if [ "$HEALTH_STATUS" = "unhealthy" ]; then
|
|
echo "AI service became unhealthy during deploy readiness wait. Continuing because AI is not a deploy gate for the core app."
|
|
docker compose -f docker-compose.yml logs --tail=200 ai-service || true
|
|
break
|
|
fi
|
|
sleep "$SLEEP_SECS"
|
|
i=$((i + 1))
|
|
done
|
|
if [ "${HEALTH_STATUS:-unknown}" != "healthy" ]; then
|
|
echo "AI service did not become healthy within $((ATTEMPTS * SLEEP_SECS)) seconds. Final status: ${HEALTH_STATUS:-unknown}. Continuing because AI is not a deploy gate for the core app."
|
|
docker compose -f docker-compose.yml ps
|
|
docker compose -f docker-compose.yml logs --tail=200 ai-service || true
|
|
fi
|
|
fi
|