feat(ops): automate backups and private CV benchmarks

.gitea/workflows/backup.yml

@@ -0,0 +1,32 @@
+name: Production backup
+
+on:
+  schedule:
+    # One of these UTC hours is 02:00 in Europe/Oslo; the guard skips the other.
+    - cron: '0 0,1 * * *'
+  workflow_dispatch:
+
+jobs:
+  backup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Create verified production recovery set
+        uses: appleboy/ssh-action@v1.0.3
+        with:
+          host: ${{ secrets.PROD_HOST }}
+          username: ${{ secrets.PROD_USER }}
+          key: ${{ secrets.PROD_SSH_KEY }}
+          command_timeout: 20m
+          script: |
+            set -euo pipefail
+            if [ "${{ github.event_name }}" = "schedule" ] \
+                && [ "$(TZ=Europe/Oslo date +%H)" != "02" ]; then
+              echo "Outside the 02:00 Europe/Oslo backup window."
+              exit 0
+            fi
+            cd /opt/job-tracker/app
+            if [ ! -x deploy/deploy.sh ]; then
+              echo "The deployed backup script is unavailable."
+              exit 1
+            fi
+            DEPLOY_BACKUP_ONLY=true deploy/deploy.sh

.gitea/workflows/maintenance.yml

@@ -15,28 +15,48 @@ jobs:
     if: inputs.job == 'cv-benchmark'
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Setup .NET
-        uses: actions/setup-dotnet@v4
+      - name: Run private corpus benchmark on production
+        uses: appleboy/ssh-action@v1.0.3
         with:
-          dotnet-version: 9.0.317
+          host: ${{ secrets.PROD_HOST }}
+          username: ${{ secrets.PROD_USER }}
+          key: ${{ secrets.PROD_SSH_KEY }}
+          command_timeout: 30m
+          script: |
+            set -euo pipefail
+            cd /opt/job-tracker/app
+            corpus_dir=/home/pi/cvs
+            approved_dir="$corpus_dir/approved-jsons"
+            output_dir="$(mktemp -d)/latest"
+            cleanup() { rm -rf "$(dirname "$output_dir")"; }
+            trap cleanup EXIT
 
-      - name: Run CV corpus benchmark
-        env:
-          CV_BENCHMARK_OUTPUT_DIR: ${{ github.workspace }}/tmp/cv-benchmarks/latest
-          CV_BENCHMARK_APPROVED_DIR: ${{ github.workspace }}/tmp/cv-benchmarks/approved-fixtures
-        run: |
-          chmod +x scripts/run-cv-benchmark.sh
-          scripts/run-cv-benchmark.sh
-          test -s "$CV_BENCHMARK_OUTPUT_DIR/index.json"
-          test -s "$CV_BENCHMARK_OUTPUT_DIR/report.md"
+            AI_SERVICE_TOKEN="$(sed -n 's/^AI_SERVICE_TOKEN=//p' /opt/job-tracker/shared/.env | tail -n 1)"
+            if [ -z "$AI_SERVICE_TOKEN" ]; then
+              echo "AI_SERVICE_TOKEN is not configured." >&2
+              exit 1
+            fi
+            ai_id="$(docker compose -f docker-compose.yml ps -q ai-service)"
+            ai_ip="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$ai_id")"
+            if [ -z "$ai_ip" ]; then
+              echo "The AI extraction service is not running." >&2
+              exit 1
+            fi
 
-      - name: Retain benchmark report and fixture candidates
-        uses: actions/upload-artifact@v4
-        with:
-          name: cv-benchmark-${{ github.run_number }}
-          path: tmp/cv-benchmarks
-          if-no-files-found: error
-          retention-days: 30
+            chmod +x scripts/run-cv-benchmark.sh
+            CV_BENCHMARK_CORPUS_DIR="$corpus_dir" \
+            CV_BENCHMARK_OUTPUT_DIR="$output_dir" \
+            CV_BENCHMARK_APPROVED_DIR="$approved_dir" \
+            CV_AI_BASE_URL="http://$ai_ip:8001" \
+            CV_AI_SERVICE_TOKEN="$AI_SERVICE_TOKEN" \
+              scripts/run-cv-benchmark.sh
+
+            backend_id="$(docker compose -f docker-compose.yml ps -q backend)"
+            if [ -z "$backend_id" ]; then
+              echo "The backend container is not running." >&2
+              exit 1
+            fi
+            docker exec "$backend_id" sh -c 'mkdir -p /data/CvBenchmarks'
+            docker cp "$output_dir/." "$backend_id:/data/CvBenchmarks/"
+            docker exec "$backend_id" test -s /data/CvBenchmarks/index.json
+            docker exec "$backend_id" test -s /data/CvBenchmarks/report.md