feat(ops): protect complete pre-deploy recovery set

.gitea/workflows/ci-deploy.yml

@@ -312,10 +312,13 @@ jobs:
     steps:
       - name: Run remote deploy
         uses: appleboy/ssh-action@v1.0.3
+        env:
+          MAINTENANCE_GITEA_TOKEN: ${{ secrets.MAINTENANCE_GITEA_TOKEN }}
         with:
           host: ${{ secrets.PROD_HOST }}
           username: ${{ secrets.PROD_USER }}
           key: ${{ secrets.PROD_SSH_KEY }}
+          envs: MAINTENANCE_GITEA_TOKEN
           command_timeout: 40m
           script: |
             set -euo pipefail
@@ -324,6 +327,24 @@ jobs:
               exit 1
             fi
             cd /opt/job-tracker/app
+            if [ -z "${MAINTENANCE_GITEA_TOKEN:-}" ]; then
+              echo "MAINTENANCE_GITEA_TOKEN is not configured as a repository Actions secret."
+              exit 1
+            fi
+            shared_env=/opt/job-tracker/shared/.env
+            if [ ! -f "$shared_env" ]; then
+              echo "Expected deployment environment at $shared_env."
+              exit 1
+            fi
+            env_tmp="$(mktemp)"
+            trap 'rm -f "$env_tmp"' EXIT
+            grep -v '^MAINTENANCE_GITEA_TOKEN=' "$shared_env" > "$env_tmp"
+            printf 'MAINTENANCE_GITEA_TOKEN=%s\n' "$MAINTENANCE_GITEA_TOKEN" >> "$env_tmp"
+            chmod --reference="$shared_env" "$env_tmp"
+            cat "$env_tmp" > "$shared_env"
+            rm -f "$env_tmp"
+            trap - EXIT
+            unset MAINTENANCE_GITEA_TOKEN
             if ! git fetch --all --prune; then
               echo "git fetch failed on server. Check remote auth/URL for /opt/job-tracker/app."
               exit 1

deploy/README.md

@@ -193,7 +193,11 @@ deploy if the backup fails**. Nothing else in the deploy runs without a restore
 - **Selection:** driven solely by `DATABASE_PROVIDER`, which must be set.
   - `mariadb` / `mysql` → SQL dump, `jobtracker-<database>-<UTC timestamp>.sql.gz`, e.g.
     `jobtracker-jobtracker-20260719T153759Z.sql.gz`
+  - MariaDB deployments also archive `jobtracker_data`, which contains attachments, imported CVs,
+    generated exports and Data Protection keys.
   - `sqlite` → data volume archive, `jobtracker-sqlite-<UTC timestamp>.tar.gz`
+  - every deployment archives the separate account-deletion tombstone volume when it exists;
+    restore it independently and never replace it with an older application-data snapshot
   - anything else → the deploy stops
 - **The filename tells you which path ran.** If you expect a MariaDB deploy and find a
   `jobtracker-sqlite-*.tar.gz`, the environment is wrong — that is the exact failure this check exists
@@ -215,6 +219,7 @@ like a restore point.
 |---|---|
 | MariaDB | non-empty; valid gzip; contains `CREATE TABLE`; contains the `Dump completed` trailer that `mariadb-dump` writes last, so a dump that died partway through is rejected |
 | SQLite | non-empty; valid gzip; the archive actually contains `jobtracker.db` |
+| Recovery volumes | non-empty valid gzip archives; a missing required application-data volume aborts the deploy |
 
 A failed check deletes the file rather than leaving something that looks like a backup.
 

deploy/deploy.sh

@@ -265,6 +265,7 @@ validate_deploy_config() {
 # exactly when a restore point matters. See deploy/README.md.
 # ---------------------------------------------------------------------------
 BACKUP_DIR="${BACKUP_DIR:-/opt/job-tracker/backups}"
+BACKUP_STAMP="$(date -u +%Y%m%dT%H%M%SZ)"
 
 backup_database() {
   if [ "${DEPLOY_SKIP_DB_BACKUP:-false}" = "true" ]; then
@@ -276,8 +277,6 @@ backup_database() {
   local provider
   provider="$(printf '%s' "$DATABASE_PROVIDER" | tr '[:upper:]' '[:lower:]')"
 
-  local stamp
-  stamp="$(date -u +%Y%m%dT%H%M%SZ)"
   mkdir -p "$BACKUP_DIR"
   if [ ! -w "$BACKUP_DIR" ]; then
     echo "Backup directory ${BACKUP_DIR} is not writable; repairing ownership via Docker."
@@ -304,7 +303,7 @@ backup_database() {
       return 1
     fi
 
-    local target="$BACKUP_DIR/jobtracker-sqlite-${stamp}.tar.gz"
+    local target="$BACKUP_DIR/jobtracker-sqlite-${BACKUP_STAMP}.tar.gz"
     echo "Backing up SQLite data volume '${volume}' to ${target}"
     # The archive path is built inside the container: passing /backup/... as an argument
     # gets rewritten by MSYS path translation when the script is run from Git Bash.
@@ -341,7 +340,7 @@ backup_database() {
     return 1
   fi
 
-  local target="$BACKUP_DIR/jobtracker-${db_name}-${stamp}.sql.gz"
+  local target="$BACKUP_DIR/jobtracker-${db_name}-${BACKUP_STAMP}.sql.gz"
   echo "Backing up MariaDB database '${db_name}' on ${db_host}:${db_port} to ${target}"
 
   # The password goes via MYSQL_PWD, never on the command line, so it cannot leak
@@ -469,6 +468,51 @@ verify_volume_backup() {
   report_backup "$target"
 }
 
+backup_named_volume() {
+  local volume="$1" label="$2" required="$3"
+  if ! docker volume inspect "$volume" >/dev/null 2>&1; then
+    if [ "$required" = "true" ]; then
+      echo "Required recovery volume '${volume}' does not exist. Aborting deploy."
+      return 1
+    fi
+    echo "Recovery volume '${volume}' does not exist yet; nothing to preserve."
+    return 0
+  fi
+
+  local target="$BACKUP_DIR/jobtracker-${label}-${BACKUP_STAMP}.tar.gz"
+  echo "Backing up recovery volume '${volume}' to ${target}"
+  if ! docker run --rm \
+      -v "$volume":/data:ro \
+      -v "$BACKUP_DIR":/backup \
+      -e ARCHIVE_NAME="$(basename "$target")" \
+      alpine:3 sh -c 'tar czf "/backup/$ARCHIVE_NAME" -C /data .'; then
+    echo "Recovery volume backup FAILED. Aborting deploy."
+    rm -f "$target"
+    return 1
+  fi
+
+  verify_backup_file_shape "$target" "archive" || return 1
+  report_backup "$target"
+}
+
+backup_recovery_volumes() {
+  local project_name="${COMPOSE_PROJECT_NAME:-$(basename "$PWD")}" provider
+  provider="$(printf '%s' "$DATABASE_PROVIDER" | tr '[:upper:]' '[:lower:]')"
+
+  # With MariaDB, the database dump does not include attachments, imported CVs,
+  # exports or Data Protection keys stored in /data. SQLite already archives this
+  # exact volume in backup_database, so avoid writing the same archive twice.
+  if [ "$provider" != "sqlite" ]; then
+    backup_named_volume "${project_name}_jobtracker_data" "files-${project_name}" true || return 1
+  fi
+
+  # Tombstones are restored independently from application data so an older data
+  # restore cannot resurrect an erased identity. The volume may be absent on the
+  # first deployment of the deletion lifecycle.
+  backup_named_volume "${project_name}_jobtracker_deletion_tombstones" \
+    "deletion-tombstones-${project_name}" false
+}
+
 resolve_existing_web_proxy_subnet
 
 if ! validate_deploy_config; then
@@ -479,6 +523,10 @@ if ! backup_database; then
   exit 1
 fi
 
+if ! backup_recovery_volumes; then
+  exit 1
+fi
+
 previous_backend_ref=""
 previous_frontend_ref=""
 capture_previous_core_image() {

docs/deployment/backup-restore.md

@@ -86,9 +86,10 @@ MYSQL_PWD='<pw>' mariadb --host=<host> --user=<user> --default-character-set=utf
 ## Production access
 
 A sanitized read-only host inventory was completed on 2026-08-15. It verified the integrity of the
-existing compressed database dumps without reading private rows or restoring data. The newest
-observed dump was dated 2026-08-02, and the set is database-only: it does not prove recovery of owned
-files, data-protection keys, configuration, or account-deletion tombstones.
+then-existing compressed database dumps without reading private rows or restoring data. The newest
+observed dump was dated 2026-08-02, and that set was database-only. Current deployments now create a
+verified database backup plus application-data and deletion-tombstone volume archives before any
+build or replacement. A scratch restore is still required before recovery is considered proven.
 
 **Authorized production steps still required:**
 
@@ -98,10 +99,11 @@ files, data-protection keys, configuration, or account-deletion tombstones.
    - table count (~42) and row counts for `AspNetUsers`, `JobApplications`, `Companies`,
      `CareerProfiles` match production;
    - a real record containing `æ`/`ø`/`å` reads back correctly (the check above).
-3. Build and restore a complete recovery bundle covering the `jobtracker_data` volume and deployment
-   data-protection keys as well as MariaDB. Preserve the separately mounted
-   `jobtracker_deletion_tombstones` volume across application-data restores; never overwrite it with
-   an older backup capable of resurrecting a deleted identity.
+3. Restore the matching `jobtracker_data` archive alongside the database dump. It contains owner
+   files and deployment Data Protection keys.
+4. Verify the separately archived `jobtracker_deletion_tombstones` ledger, but preserve the newest
+   ledger across application-data restores; never overwrite it with an older backup capable of
+   resurrecting a deleted identity.
 
 Until that is done, backup/restore is proven **on the mechanism and on synthetic Norwegian data**, not
 on the production dataset.

docs/operations/stripe-activation.md

@@ -1,5 +1,8 @@
 # Stripe activation
 
+For a dashboard-by-dashboard operator walkthrough, use
+[`stripe-billing-setup-walkthrough.md`](stripe-billing-setup-walkthrough.md).
+
 The application code is complete. Activate it in Stripe test mode first; test and live mode have
 different API keys, price ids, webhook endpoints/signing secrets, and customer data.
 

docs/operations/stripe-billing-setup-walkthrough.md

@@ -0,0 +1,133 @@
+# Stripe billing setup walkthrough
+
+This closes production blocker 2. Complete the test-mode checklist first, prove the full lifecycle,
+then repeat it in live mode. Stripe test and live objects are separate: keys, prices, webhook secrets,
+customers and subscriptions cannot be mixed.
+
+The current production misconfiguration is specific: `STRIPE_PRICE_PREMIUM` contains a Product ID
+(`prod_...`). JobTracker needs the recurring Price ID attached to that product (`price_...`). The API
+deliberately treats the Product ID as disabled instead of sending an invalid Checkout request.
+
+## What you need
+
+- Access to the Stripe account that will receive JobTracker subscriptions.
+- Access to `/opt/job-tracker/shared/.env` on the production host.
+- One disposable JobTracker user for the test purchase.
+- A decision on the monthly Pro price and currency. Start with one monthly price; annual billing can
+  be added after the monthly lifecycle is proven.
+
+Never paste Stripe keys or webhook signing secrets into Git, chat, screenshots, tickets or CI logs.
+
+## Part 1 — create the test subscription price
+
+1. Sign in to the Stripe Dashboard and enable **Test mode**.
+2. Open **Product catalog** and either open the existing JobTracker Pro product or create one named
+   `JobTracker Pro`.
+3. Add a price with:
+   - pricing model: standard/flat rate;
+   - billing: recurring;
+   - interval: monthly;
+   - currency and amount: your chosen commercial terms.
+4. Save the price.
+5. Open the new price and copy its identifier. It must start with `price_`, not `prod_`.
+
+The product describes what is sold. The price describes how it is billed. Checkout needs the latter.
+
+## Part 2 — configure the customer portal
+
+1. In Stripe, open the customer-portal settings.
+2. Enable the portal in test mode.
+3. Allow customers to update payment details and cancel subscriptions.
+4. Decide whether cancellation is immediate or at the end of the billing period. The application
+   follows Stripe's resulting subscription status; Stripe remains authoritative.
+5. Save the portal configuration.
+
+## Part 3 — create the test webhook
+
+1. Open Stripe's webhook/event-destination area in test mode.
+2. Add this public HTTPS endpoint:
+
+   ```text
+   https://jobs.cesnimda.uk/api/billing/webhook
+   ```
+
+3. Subscribe it to exactly these events:
+
+   ```text
+   customer.subscription.created
+   customer.subscription.updated
+   customer.subscription.deleted
+   ```
+
+4. Save the endpoint.
+5. Reveal and copy that endpoint's signing secret. It starts with `whsec_`.
+
+The endpoint signing secret belongs to this specific webhook endpoint and mode. It is not the API
+secret key shown on Stripe's API keys page.
+
+## Part 4 — install the three test values
+
+On the production host, edit `/opt/job-tracker/shared/.env` and set:
+
+```text
+STRIPE_SECRET_KEY=<test secret key beginning sk_test_>
+STRIPE_PRICE_PREMIUM=<test recurring price id beginning price_>
+STRIPE_WEBHOOK_SECRET=<test endpoint secret beginning whsec_>
+```
+
+Before saving, verify all three values came from test mode. Do not mix a live key with a test price or
+test webhook secret. Redeploy after saving so the backend receives the new environment.
+
+## Part 5 — prove the test lifecycle
+
+Use a disposable authenticated JobTracker account:
+
+1. Open account/settings and confirm billing is available.
+2. Select **Upgrade**. Stripe-hosted Checkout must open for the expected monthly amount and currency.
+3. Complete Checkout using a Stripe test card.
+4. Confirm Stripe redirects to `/settings?billing=success`.
+5. In Stripe's webhook delivery log, confirm the subscription event received a successful `2xx`
+   response.
+6. Refresh JobTracker and confirm the user has Pro/Premium entitlements.
+7. Open **Manage billing** and confirm the Stripe customer portal opens.
+8. Cancel the test subscription using the portal.
+9. Confirm the updated/deleted subscription webhook succeeds and JobTracker removes Premium after
+   the subscription is no longer active or trialling.
+10. Replay one already-successful event from Stripe. JobTracker's stored state and role must remain
+    correct; replay must not duplicate or corrupt billing state.
+
+Do not continue to live mode if any webhook delivery is red, Checkout shows the wrong amount, or the
+role does not follow Stripe's subscription state.
+
+## Part 6 — live-mode cutover
+
+1. Disable Test mode in Stripe.
+2. Create or open the live JobTracker Pro product and create a live recurring monthly price.
+3. Configure the live customer portal.
+4. Create the live webhook endpoint with the same URL and three subscription events.
+5. Replace all three production environment values together:
+   - live `sk_live_...` secret key;
+   - live `price_...` recurring Price ID;
+   - live endpoint-specific `whsec_...` signing secret.
+6. Redeploy.
+7. Complete one low-risk real subscription, portal visit and cancellation.
+8. Confirm every live webhook delivery is successful and the JobTracker role follows the live
+   subscription state.
+
+## Troubleshooting
+
+| Symptom | Check |
+| --- | --- |
+| Billing appears unavailable | `STRIPE_PRICE_PREMIUM` must begin `price_`; all three settings must be present. |
+| Checkout says the price does not exist | The API key and Price ID probably come from different Stripe modes or accounts. |
+| Webhook signature fails | Use the endpoint's `whsec_...` secret, not an API key or a Stripe CLI temporary secret. |
+| Checkout succeeds but Premium is absent | Inspect the webhook delivery response; JobTracker provisions from signed subscription events. |
+| Portal does not open | Enable and save the customer portal in the same Stripe mode as the customer. |
+| Cancellation does not remove Premium | Check the resulting Stripe status and replay the authoritative subscription event. |
+
+## Safe rollback
+
+Removing all three Stripe environment values and redeploying disables new Checkout and portal
+sessions without deleting billing history. Do not manually remove Premium while Stripe still reports
+an active subscription. Repair webhook delivery and replay the authoritative event instead.
+