cesnimda
416 lines
19 KiB
YAML
416 lines
19 KiB
YAML
name: CI and Deploy
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- main
|
|
pull_request:
|
|
|
|
jobs:
|
|
test:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Setup .NET (resilient)
|
|
shell: bash
|
|
# actions/setup-dotnet on this single self-hosted runner intermittently
|
|
# leaves a partial extraction in the shared tool-cache ("tar: Cannot open:
|
|
# File exists") or corrupts the SDK download. Install into a clean private
|
|
# dir via dotnet-install.sh and retry once on failure, mirroring the
|
|
# npm ci / NuGet retries elsewhere in this workflow.
|
|
run: |
|
|
export DOTNET_ROOT="$HOME/.dotnet"
|
|
export PATH="$DOTNET_ROOT:$PATH"
|
|
|
|
validate_sdk() {
|
|
[ -x "$DOTNET_ROOT/dotnet" ] \
|
|
&& "$DOTNET_ROOT/dotnet" --list-sdks | grep -q '^9\.0\.317 ' \
|
|
&& "$DOTNET_ROOT/dotnet" --info >/dev/null
|
|
}
|
|
|
|
if validate_sdk; then
|
|
echo "Reusing valid runner-local .NET 9 SDK."
|
|
else
|
|
install() {
|
|
local feed="${1:-}"
|
|
local -a feed_args=()
|
|
if [ -n "$feed" ]; then
|
|
feed_args=(--azure-feed "$feed")
|
|
fi
|
|
curl -fsSL https://dot.net/v1/dotnet-install.sh -o /tmp/dotnet-install.sh
|
|
rm -rf "$DOTNET_ROOT"
|
|
bash /tmp/dotnet-install.sh --version 9.0.317 --install-dir "$DOTNET_ROOT" "${feed_args[@]}"
|
|
validate_sdk
|
|
}
|
|
install || ( echo "dotnet install failed ($?) — retrying from alternate Microsoft CDN..." \
|
|
&& install https://dotnetcli.azureedge.net/dotnet )
|
|
fi
|
|
echo "$DOTNET_ROOT" >> "$GITHUB_PATH"
|
|
"$DOTNET_ROOT/dotnet" --info
|
|
|
|
- name: Setup Node
|
|
uses: actions/setup-node@v4
|
|
with:
|
|
node-version: '20'
|
|
|
|
- name: Setup Compose for configuration checks
|
|
# No Docker daemon or GitHub token is required for config rendering.
|
|
# Pin the official Linux x86_64 release and verify before execution.
|
|
shell: bash
|
|
run: |
|
|
test "$(uname -m)" = x86_64
|
|
compose_dir="$(mktemp -d)"
|
|
curl --fail --silent --show-error --location --retry 3 \
|
|
https://github.com/docker/compose/releases/download/v5.3.1/docker-compose-linux-x86_64 \
|
|
-o "$compose_dir/docker-compose"
|
|
echo "f9ebc6ebdb19d769b793c245a736caaeb198c62587f13b25c660c13b4987f959 $compose_dir/docker-compose" | sha256sum --check --strict
|
|
chmod +x "$compose_dir/docker-compose"
|
|
"$compose_dir/docker-compose" version
|
|
echo "$compose_dir" >> "$GITHUB_PATH"
|
|
|
|
- name: Test repository safety scripts
|
|
# Python standard library and Compose config rendering only; no services start.
|
|
run: |
|
|
python3 scripts/test-ollama-evaluation.py
|
|
python3 scripts/test-supply-chain.py
|
|
python3 scripts/test-ingress.py
|
|
|
|
- name: Test browser extension
|
|
run: node --test browser-extension/tests/*.test.mjs
|
|
|
|
- name: Scan tracked files and generate dependency SBOM
|
|
run: |
|
|
python3 scripts/supply-chain.py scan-secrets
|
|
python3 scripts/supply-chain.py generate-sbom --output .artifacts/jobjakt.cdx.json
|
|
|
|
- name: Test document parser boundary
|
|
working-directory: tools/summarizer
|
|
run: |
|
|
# The Debian runner omits pip and python3-venv. Use pip's official
|
|
# zipapp and install into the checkout so CI remains isolated without
|
|
# sudo or a mutable runner-global Python environment.
|
|
curl -fsSLo /tmp/pip.pyz https://bootstrap.pypa.io/pip/pip.pyz
|
|
install_parser_dependencies() {
|
|
rm -rf .python-ci
|
|
parser_pip_cache="$(mktemp -d)"
|
|
PIP_CACHE_DIR="$parser_pip_cache" python3 /tmp/pip.pyz install --target .python-ci --require-hashes \
|
|
--extra-index-url https://download.pytorch.org/whl/cpu \
|
|
-r requirements-linux.lock
|
|
}
|
|
parser_install_succeeded=false
|
|
for attempt in 1 2 3; do
|
|
if install_parser_dependencies; then
|
|
parser_install_succeeded=true
|
|
break
|
|
fi
|
|
echo "Parser dependency install attempt $attempt failed — retrying with a fresh isolated cache..."
|
|
done
|
|
test "$parser_install_succeeded" = true
|
|
PIP_CACHE_DIR="$parser_pip_cache" python3 /tmp/pip.pyz install --upgrade --target .python-ci pytest==8.3.5 httpx2==2.12.0
|
|
PYTHONPATH="$PWD/.python-ci" AI_SERVICE_SKIP_MODEL_LOAD=1 python3 -m pytest -q
|
|
|
|
- name: Restore backend
|
|
# The runner/proxy has occasionally supplied checksum-invalid NuGet cache entries (NU3008).
|
|
# Retry from clean HTTP/global caches; signature verification remains enabled.
|
|
run: |
|
|
restore() { dotnet restore JobTrackerApi/JobTrackerApi.csproj --locked-mode; }
|
|
restore || ( echo "Backend restore failed ($?) — clearing runner NuGet caches and retrying once..." \
|
|
&& dotnet nuget locals http-cache --clear \
|
|
&& dotnet nuget locals global-packages --clear \
|
|
&& dotnet restore JobTrackerApi/JobTrackerApi.csproj --no-cache --locked-mode )
|
|
|
|
- name: Build backend
|
|
# Roslyn has intermittently exited 139 on this resource-constrained runner. Retry once
|
|
# without compiler-server or parallel build processes; restore failures are handled above.
|
|
run: |
|
|
dotnet build JobTrackerApi/JobTrackerApi.csproj --configuration Release --no-restore \
|
|
|| ( echo "Backend compiler crashed ($?) — retrying single-process..." \
|
|
&& dotnet build JobTrackerApi/JobTrackerApi.csproj --configuration Release \
|
|
--no-restore --disable-build-servers --maxcpucount:1 \
|
|
/p:UseSharedCompilation=false /p:BuildInParallel=false )
|
|
|
|
# The "Build backend" step above only builds JobTrackerApi, so the test project needs its own
|
|
# restore + build. These are separate steps on purpose: this runner is flaky (see the SDK and
|
|
# npm retries in this file) and the job log is not readable from the API, so distinct steps make
|
|
# the failing phase identifiable from step boundaries alone. Restore retries once, mirroring the
|
|
# npm ci pattern below. The suite itself is never weakened or filtered.
|
|
- name: Restore backend tests
|
|
run: |
|
|
restore() { dotnet restore JobTrackerApi.Tests/JobTrackerApi.Tests.csproj --locked-mode; }
|
|
restore || ( echo "NuGet restore failed ($?) — clearing runner caches and retrying once..." \
|
|
&& dotnet nuget locals http-cache --clear \
|
|
&& dotnet nuget locals global-packages --clear \
|
|
&& dotnet restore JobTrackerApi.Tests/JobTrackerApi.Tests.csproj --no-cache --locked-mode )
|
|
|
|
- name: Build backend tests
|
|
run: |
|
|
dotnet build JobTrackerApi.Tests/JobTrackerApi.Tests.csproj --configuration Release --no-restore \
|
|
|| ( echo "Test compiler crashed ($?) — retrying single-process..." \
|
|
&& dotnet build JobTrackerApi.Tests/JobTrackerApi.Tests.csproj --configuration Release \
|
|
--no-restore --disable-build-servers --maxcpucount:1 \
|
|
/p:UseSharedCompilation=false /p:BuildInParallel=false )
|
|
|
|
# Restore and build both pass on this runner, but the test run dies after ~3s -- far too fast to
|
|
# have executed 306 tests. The suite passes on Windows, in a clean Linux container, under a 1GB
|
|
# memory cap, and with the SDK installed to a custom dir without DOTNET_ROOT, so the cause is
|
|
# specific to this runner. This one-test smoke separates "the test host cannot start at all"
|
|
# from "something in the suite takes the host down"; the log is not readable via the API, so the
|
|
# step boundary is the signal.
|
|
- name: Audit backend dependencies
|
|
run: dotnet list JobTrackerApi/JobTrackerApi.csproj package --vulnerable --include-transitive
|
|
|
|
- name: Test backend (host smoke)
|
|
run: dotnet test JobTrackerApi.Tests/JobTrackerApi.Tests.csproj --configuration Release --no-build --filter "FullyQualifiedName~CvBuilderTests.Every_catalog_theme_renders_valid_html"
|
|
|
|
# This runner fails at a DIFFERENT stage on different runs with no readable diagnostics
|
|
# (test host at 3s; on another run `dotnet restore` at 0s). See
|
|
# docs/infrastructure/runner-investigation.md. Collection parallelism stays off for
|
|
# determinism, the same reason the frontend runs --runInBand. Every test runs; nothing skipped.
|
|
- name: Test backend
|
|
run: dotnet test JobTrackerApi.Tests/JobTrackerApi.Tests.csproj --configuration Release --no-build -- xUnit.parallelizeTestCollections=false xUnit.maxParallelThreads=1
|
|
|
|
- name: Install frontend deps
|
|
working-directory: job-tracker-ui
|
|
env:
|
|
npm_config_audit: 'false'
|
|
npm_config_fund: 'false'
|
|
run: |
|
|
node -v
|
|
npm -v
|
|
# The runner's shared npm cache has returned checksum-invalid tarballs and
|
|
# injected control bytes into installed JS. Give each attempt an isolated
|
|
# cache so a poisoned archive cannot survive into this checkout or retry.
|
|
npm_cache="$(mktemp -d)"
|
|
npm ci --no-audit --no-fund --cache "$npm_cache" \
|
|
|| ( echo "npm ci failed ($?) — cleaning node_modules and retrying once..." \
|
|
&& rm -rf node_modules \
|
|
&& npm_cache="$(mktemp -d)" \
|
|
&& npm ci --no-audit --no-fund --cache "$npm_cache" )
|
|
|
|
# Moderate React Router advisories remain documented and mitigated; high/critical findings
|
|
# in either production or test/browser tooling block the build.
|
|
- name: Audit frontend dependencies
|
|
working-directory: job-tracker-ui
|
|
run: npm audit --audit-level=high
|
|
|
|
- name: Lint frontend
|
|
working-directory: job-tracker-ui
|
|
# Match the runner resilience used by tests/builds without masking real lint errors.
|
|
# Exit 139 is the self-hosted runner killing Node before ESLint can report a result.
|
|
run: |
|
|
set +e
|
|
npm run lint
|
|
lint_status=$?
|
|
set -e
|
|
|
|
if [ "$lint_status" -eq 0 ]; then
|
|
exit 0
|
|
fi
|
|
if [ "$lint_status" -ne 139 ]; then
|
|
exit "$lint_status"
|
|
fi
|
|
|
|
echo "Frontend lint process crashed with SIGSEGV — retrying once..."
|
|
npm run lint
|
|
|
|
- name: Test frontend
|
|
working-directory: job-tracker-ui
|
|
# Run the WHOLE suite. Never whitelist test files here again: the previous
|
|
# whitelist silently skipped new suites and let two regressions reach main.
|
|
# The self-hosted runner can also terminate Node with SIGSEGV (139) without a
|
|
# Jest assertion or diagnostic. Retry only that infrastructure signature once;
|
|
# ordinary assertion/configuration failures still fail immediately.
|
|
run: |
|
|
set +e
|
|
npm test -- --watchAll=false --runInBand
|
|
test_status=$?
|
|
set -e
|
|
|
|
if [ "$test_status" -eq 0 ]; then
|
|
exit 0
|
|
fi
|
|
if [ "$test_status" -ne 139 ]; then
|
|
exit "$test_status"
|
|
fi
|
|
|
|
echo "Frontend test process crashed with SIGSEGV — retrying the complete suite once..."
|
|
npm test -- --watchAll=false --runInBand
|
|
|
|
- name: Build frontend
|
|
working-directory: job-tracker-ui
|
|
env:
|
|
CI: 'false'
|
|
GENERATE_SOURCEMAP: 'false'
|
|
NODE_OPTIONS: --max-old-space-size=4096
|
|
# NEXT_PUBLIC values are embedded during `next build`. The browser smoke
|
|
# suite serves this prebuilt export, so it must target its isolated API
|
|
# rather than the normal localhost development port.
|
|
NEXT_PUBLIC_API_BASE_URL: http://localhost:5302/api
|
|
# Build once, then make Playwright serve this exact export. A SIGSEGV-only retry preserves
|
|
# the runner workaround without hiding an ordinary compiler or type-check failure.
|
|
run: |
|
|
set +e
|
|
npm run build
|
|
build_status=$?
|
|
set -e
|
|
|
|
if [ "$build_status" -eq 0 ]; then
|
|
exit 0
|
|
fi
|
|
if [ "$build_status" -ne 139 ]; then
|
|
exit "$build_status"
|
|
fi
|
|
|
|
echo "Frontend build process crashed with SIGSEGV — retrying once..."
|
|
npm run build
|
|
|
|
- name: Install browser smoke runtime
|
|
working-directory: job-tracker-ui
|
|
env:
|
|
PLAYWRIGHT_DOWNLOAD_CONNECTION_TIMEOUT: '120000'
|
|
run: |
|
|
# Invoke the lockfile-installed CLI directly. `npx` may create a second
|
|
# temporary dependency tree when its cache is damaged, which made this
|
|
# runner's intermittent truncated downloads substantially worse.
|
|
node ./node_modules/@playwright/test/cli.js install --with-deps chromium
|
|
|
|
- name: Test browser smoke flows
|
|
working-directory: job-tracker-ui
|
|
env:
|
|
# Exercise the deployable frontend and avoid Turbopack's persistence crashes on
|
|
# this runner's slow Docker volume.
|
|
PLAYWRIGHT_STATIC_EXPORT: 'true'
|
|
PLAYWRIGHT_PREBUILT_EXPORT: 'true'
|
|
# The runner occasionally terminates an otherwise healthy Node process with
|
|
# SIGSEGV/139. Retry only that infrastructure signature. Do not reinstall npm
|
|
# or browser payloads here: doing so used to redownload hundreds of megabytes
|
|
# through the same unreliable transport and made a transient crash persistent.
|
|
# Assertion/configuration failures remain terminal and always block deployment.
|
|
run: |
|
|
smoke_log="$(mktemp)"
|
|
set +e
|
|
npm run test:e2e 2>&1 | tee "$smoke_log"
|
|
smoke_status=${PIPESTATUS[0]}
|
|
set -e
|
|
|
|
if [ "$smoke_status" -eq 0 ]; then
|
|
exit 0
|
|
fi
|
|
|
|
# This self-hosted runner has occasionally mutated Axe's installed JavaScript after a
|
|
# checksum-valid npm download (`true` became `tr<garbage>e`, or a stray `break` appeared).
|
|
# Retry only that unmistakable source-corruption signature from a new isolated cache.
|
|
# Ordinary Axe violations and application assertions remain terminal.
|
|
if grep -q "AxeBuilder.analyze" "$smoke_log" \
|
|
&& grep -Eq "page\.evaluate: (ReferenceError|SyntaxError)" "$smoke_log"; then
|
|
echo "Axe runtime source appears corrupted — reinstalling the locked dependencies from a fresh cache and retrying the complete suite once..."
|
|
rm -rf node_modules
|
|
npm_cache="$(mktemp -d)"
|
|
npm ci --no-audit --no-fund --cache "$npm_cache"
|
|
npm run test:e2e
|
|
exit $?
|
|
fi
|
|
|
|
if [ "$smoke_status" -ne 139 ]; then
|
|
exit "$smoke_status"
|
|
fi
|
|
|
|
echo "Browser smoke process crashed with SIGSEGV — retrying the unchanged suite once..."
|
|
npm run test:e2e
|
|
|
|
deploy:
|
|
needs: test
|
|
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Run remote deploy
|
|
uses: appleboy/ssh-action@v1.0.3
|
|
env:
|
|
MAINTENANCE_GITEA_TOKEN: ${{ secrets.MAINTENANCE_GITEA_TOKEN }}
|
|
with:
|
|
host: ${{ secrets.PROD_HOST }}
|
|
username: ${{ secrets.PROD_USER }}
|
|
key: ${{ secrets.PROD_SSH_KEY }}
|
|
envs: MAINTENANCE_GITEA_TOKEN
|
|
command_timeout: 40m
|
|
script: |
|
|
set -euo pipefail
|
|
if [ ! -d /opt/job-tracker/app/.git ]; then
|
|
echo "Expected git checkout at /opt/job-tracker/app but .git was not found."
|
|
exit 1
|
|
fi
|
|
cd /opt/job-tracker/app
|
|
if [ -z "${MAINTENANCE_GITEA_TOKEN:-}" ]; then
|
|
echo "MAINTENANCE_GITEA_TOKEN is not configured as a repository Actions secret."
|
|
exit 1
|
|
fi
|
|
case "$MAINTENANCE_GITEA_TOKEN" in
|
|
*[!A-Za-z0-9_-]*)
|
|
echo "MAINTENANCE_GITEA_TOKEN must be a single token value, not a pasted dotenv block."
|
|
exit 1
|
|
;;
|
|
esac
|
|
shared_env=/opt/job-tracker/shared/.env
|
|
if [ ! -f "$shared_env" ]; then
|
|
echo "Expected deployment environment at $shared_env."
|
|
exit 1
|
|
fi
|
|
env_tmp="$(mktemp)"
|
|
trap 'rm -f "$env_tmp"' EXIT
|
|
grep -Ev '^MAINTENANCE_GITEA_TOKEN=|^```(dotenv)?[[:space:]]*$' "$shared_env" > "$env_tmp"
|
|
printf 'MAINTENANCE_GITEA_TOKEN=%s\n' "$MAINTENANCE_GITEA_TOKEN" >> "$env_tmp"
|
|
chmod --reference="$shared_env" "$env_tmp"
|
|
if ! docker compose --env-file "$env_tmp" -f docker-compose.yml config --quiet; then
|
|
echo "Refusing to replace the shared deployment environment because it is invalid."
|
|
exit 1
|
|
fi
|
|
cat "$env_tmp" > "$shared_env"
|
|
rm -f "$env_tmp"
|
|
trap - EXIT
|
|
unset MAINTENANCE_GITEA_TOKEN
|
|
if ! git fetch --all --prune; then
|
|
echo "git fetch failed on server. Check remote auth/URL for /opt/job-tracker/app."
|
|
exit 1
|
|
fi
|
|
if ! git rev-parse --verify --quiet ${{ github.sha }} >/dev/null; then
|
|
echo "Commit ${{ github.sha }} is not available in the server checkout after fetch."
|
|
exit 1
|
|
fi
|
|
git reset --hard ${{ github.sha }}
|
|
git clean -fd
|
|
chmod +x deploy/deploy.sh
|
|
DEPLOY_BUILD_AI_SERVICE=true \
|
|
APP_VERSION=${{ github.run_number }} \
|
|
APP_COMMIT_SHA=${{ github.sha }} \
|
|
APP_BUILD_STAMP="$(date -u +'%Y-%m-%d %H:%M UTC')" \
|
|
./deploy/deploy.sh
|
|
docker compose -f docker-compose.yml ps
|
|
AI_CONTAINER_ID="$(docker compose -f docker-compose.yml ps -q ai-service)"
|
|
if [ -z "$AI_CONTAINER_ID" ]; then
|
|
echo "AI service container id could not be resolved after deploy. Continuing because AI is not a deploy gate for the core app."
|
|
else
|
|
ATTEMPTS=90
|
|
SLEEP_SECS=2
|
|
i=1
|
|
while [ "$i" -le "$ATTEMPTS" ]; do
|
|
HEALTH_STATUS="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$AI_CONTAINER_ID" 2>/dev/null || echo unknown)"
|
|
if [ "$HEALTH_STATUS" = "healthy" ]; then
|
|
break
|
|
fi
|
|
if [ "$HEALTH_STATUS" = "unhealthy" ]; then
|
|
echo "AI service became unhealthy during deploy readiness wait. Continuing because AI is not a deploy gate for the core app."
|
|
docker compose -f docker-compose.yml logs --tail=200 ai-service || true
|
|
break
|
|
fi
|
|
sleep "$SLEEP_SECS"
|
|
i=$((i + 1))
|
|
done
|
|
if [ "${HEALTH_STATUS:-unknown}" != "healthy" ]; then
|
|
echo "AI service did not become healthy within $((ATTEMPTS * SLEEP_SECS)) seconds. Final status: ${HEALTH_STATUS:-unknown}. Continuing because AI is not a deploy gate for the core app."
|
|
docker compose -f docker-compose.yml ps
|
|
docker compose -f docker-compose.yml logs --tail=200 ai-service || true
|
|
fi
|
|
fi
|