diff --git a/.gitea/workflows/ci-deploy.yml b/.gitea/workflows/ci-deploy.yml index 67f8fb2..ba4d4d0 100644 --- a/.gitea/workflows/ci-deploy.yml +++ b/.gitea/workflows/ci-deploy.yml @@ -312,10 +312,13 @@ jobs: steps: - name: Run remote deploy uses: appleboy/ssh-action@v1.0.3 + env: + MAINTENANCE_GITEA_TOKEN: ${{ secrets.MAINTENANCE_GITEA_TOKEN }} with: host: ${{ secrets.PROD_HOST }} username: ${{ secrets.PROD_USER }} key: ${{ secrets.PROD_SSH_KEY }} + envs: MAINTENANCE_GITEA_TOKEN command_timeout: 40m script: | set -euo pipefail @@ -324,6 +327,24 @@ jobs: exit 1 fi cd /opt/job-tracker/app + if [ -z "${MAINTENANCE_GITEA_TOKEN:-}" ]; then + echo "MAINTENANCE_GITEA_TOKEN is not configured as a repository Actions secret." + exit 1 + fi + shared_env=/opt/job-tracker/shared/.env + if [ ! -f "$shared_env" ]; then + echo "Expected deployment environment at $shared_env." + exit 1 + fi + env_tmp="$(mktemp)" + trap 'rm -f "$env_tmp"' EXIT + grep -v '^MAINTENANCE_GITEA_TOKEN=' "$shared_env" > "$env_tmp" + printf 'MAINTENANCE_GITEA_TOKEN=%s\n' "$MAINTENANCE_GITEA_TOKEN" >> "$env_tmp" + chmod --reference="$shared_env" "$env_tmp" + cat "$env_tmp" > "$shared_env" + rm -f "$env_tmp" + trap - EXIT + unset MAINTENANCE_GITEA_TOKEN if ! git fetch --all --prune; then echo "git fetch failed on server. Check remote auth/URL for /opt/job-tracker/app." exit 1 diff --git a/deploy/README.md b/deploy/README.md index 7f28c10..65a28a5 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -193,7 +193,11 @@ deploy if the backup fails**. Nothing else in the deploy runs without a restore - **Selection:** driven solely by `DATABASE_PROVIDER`, which must be set. - `mariadb` / `mysql` → SQL dump, `jobtracker--.sql.gz`, e.g. `jobtracker-jobtracker-20260719T153759Z.sql.gz` + - MariaDB deployments also archive `jobtracker_data`, which contains attachments, imported CVs, + generated exports and Data Protection keys. - `sqlite` → data volume archive, `jobtracker-sqlite-.tar.gz` + - every deployment archives the separate account-deletion tombstone volume when it exists; + restore it independently and never replace it with an older application-data snapshot - anything else → the deploy stops - **The filename tells you which path ran.** If you expect a MariaDB deploy and find a `jobtracker-sqlite-*.tar.gz`, the environment is wrong — that is the exact failure this check exists @@ -215,6 +219,7 @@ like a restore point. |---|---| | MariaDB | non-empty; valid gzip; contains `CREATE TABLE`; contains the `Dump completed` trailer that `mariadb-dump` writes last, so a dump that died partway through is rejected | | SQLite | non-empty; valid gzip; the archive actually contains `jobtracker.db` | +| Recovery volumes | non-empty valid gzip archives; a missing required application-data volume aborts the deploy | A failed check deletes the file rather than leaving something that looks like a backup. diff --git a/deploy/deploy.sh b/deploy/deploy.sh index c37641c..2bc35d6 100644 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -265,6 +265,7 @@ validate_deploy_config() { # exactly when a restore point matters. See deploy/README.md. # --------------------------------------------------------------------------- BACKUP_DIR="${BACKUP_DIR:-/opt/job-tracker/backups}" +BACKUP_STAMP="$(date -u +%Y%m%dT%H%M%SZ)" backup_database() { if [ "${DEPLOY_SKIP_DB_BACKUP:-false}" = "true" ]; then @@ -276,8 +277,6 @@ backup_database() { local provider provider="$(printf '%s' "$DATABASE_PROVIDER" | tr '[:upper:]' '[:lower:]')" - local stamp - stamp="$(date -u +%Y%m%dT%H%M%SZ)" mkdir -p "$BACKUP_DIR" if [ ! -w "$BACKUP_DIR" ]; then echo "Backup directory ${BACKUP_DIR} is not writable; repairing ownership via Docker." @@ -304,7 +303,7 @@ backup_database() { return 1 fi - local target="$BACKUP_DIR/jobtracker-sqlite-${stamp}.tar.gz" + local target="$BACKUP_DIR/jobtracker-sqlite-${BACKUP_STAMP}.tar.gz" echo "Backing up SQLite data volume '${volume}' to ${target}" # The archive path is built inside the container: passing /backup/... as an argument # gets rewritten by MSYS path translation when the script is run from Git Bash. @@ -341,7 +340,7 @@ backup_database() { return 1 fi - local target="$BACKUP_DIR/jobtracker-${db_name}-${stamp}.sql.gz" + local target="$BACKUP_DIR/jobtracker-${db_name}-${BACKUP_STAMP}.sql.gz" echo "Backing up MariaDB database '${db_name}' on ${db_host}:${db_port} to ${target}" # The password goes via MYSQL_PWD, never on the command line, so it cannot leak @@ -469,6 +468,51 @@ verify_volume_backup() { report_backup "$target" } +backup_named_volume() { + local volume="$1" label="$2" required="$3" + if ! docker volume inspect "$volume" >/dev/null 2>&1; then + if [ "$required" = "true" ]; then + echo "Required recovery volume '${volume}' does not exist. Aborting deploy." + return 1 + fi + echo "Recovery volume '${volume}' does not exist yet; nothing to preserve." + return 0 + fi + + local target="$BACKUP_DIR/jobtracker-${label}-${BACKUP_STAMP}.tar.gz" + echo "Backing up recovery volume '${volume}' to ${target}" + if ! docker run --rm \ + -v "$volume":/data:ro \ + -v "$BACKUP_DIR":/backup \ + -e ARCHIVE_NAME="$(basename "$target")" \ + alpine:3 sh -c 'tar czf "/backup/$ARCHIVE_NAME" -C /data .'; then + echo "Recovery volume backup FAILED. Aborting deploy." + rm -f "$target" + return 1 + fi + + verify_backup_file_shape "$target" "archive" || return 1 + report_backup "$target" +} + +backup_recovery_volumes() { + local project_name="${COMPOSE_PROJECT_NAME:-$(basename "$PWD")}" provider + provider="$(printf '%s' "$DATABASE_PROVIDER" | tr '[:upper:]' '[:lower:]')" + + # With MariaDB, the database dump does not include attachments, imported CVs, + # exports or Data Protection keys stored in /data. SQLite already archives this + # exact volume in backup_database, so avoid writing the same archive twice. + if [ "$provider" != "sqlite" ]; then + backup_named_volume "${project_name}_jobtracker_data" "files-${project_name}" true || return 1 + fi + + # Tombstones are restored independently from application data so an older data + # restore cannot resurrect an erased identity. The volume may be absent on the + # first deployment of the deletion lifecycle. + backup_named_volume "${project_name}_jobtracker_deletion_tombstones" \ + "deletion-tombstones-${project_name}" false +} + resolve_existing_web_proxy_subnet if ! validate_deploy_config; then @@ -479,6 +523,10 @@ if ! backup_database; then exit 1 fi +if ! backup_recovery_volumes; then + exit 1 +fi + previous_backend_ref="" previous_frontend_ref="" capture_previous_core_image() { diff --git a/docs/deployment/backup-restore.md b/docs/deployment/backup-restore.md index 3b2bde8..21d5e07 100644 --- a/docs/deployment/backup-restore.md +++ b/docs/deployment/backup-restore.md @@ -86,9 +86,10 @@ MYSQL_PWD='' mariadb --host= --user= --default-character-set=utf ## Production access A sanitized read-only host inventory was completed on 2026-08-15. It verified the integrity of the -existing compressed database dumps without reading private rows or restoring data. The newest -observed dump was dated 2026-08-02, and the set is database-only: it does not prove recovery of owned -files, data-protection keys, configuration, or account-deletion tombstones. +then-existing compressed database dumps without reading private rows or restoring data. The newest +observed dump was dated 2026-08-02, and that set was database-only. Current deployments now create a +verified database backup plus application-data and deletion-tombstone volume archives before any +build or replacement. A scratch restore is still required before recovery is considered proven. **Authorized production steps still required:** @@ -98,10 +99,11 @@ files, data-protection keys, configuration, or account-deletion tombstones. - table count (~42) and row counts for `AspNetUsers`, `JobApplications`, `Companies`, `CareerProfiles` match production; - a real record containing `æ`/`ø`/`å` reads back correctly (the check above). -3. Build and restore a complete recovery bundle covering the `jobtracker_data` volume and deployment - data-protection keys as well as MariaDB. Preserve the separately mounted - `jobtracker_deletion_tombstones` volume across application-data restores; never overwrite it with - an older backup capable of resurrecting a deleted identity. +3. Restore the matching `jobtracker_data` archive alongside the database dump. It contains owner + files and deployment Data Protection keys. +4. Verify the separately archived `jobtracker_deletion_tombstones` ledger, but preserve the newest + ledger across application-data restores; never overwrite it with an older backup capable of + resurrecting a deleted identity. Until that is done, backup/restore is proven **on the mechanism and on synthetic Norwegian data**, not on the production dataset. diff --git a/docs/operations/stripe-activation.md b/docs/operations/stripe-activation.md index cf01633..6698691 100644 --- a/docs/operations/stripe-activation.md +++ b/docs/operations/stripe-activation.md @@ -1,5 +1,8 @@ # Stripe activation +For a dashboard-by-dashboard operator walkthrough, use +[`stripe-billing-setup-walkthrough.md`](stripe-billing-setup-walkthrough.md). + The application code is complete. Activate it in Stripe test mode first; test and live mode have different API keys, price ids, webhook endpoints/signing secrets, and customer data. diff --git a/docs/operations/stripe-billing-setup-walkthrough.md b/docs/operations/stripe-billing-setup-walkthrough.md new file mode 100644 index 0000000..a8771d1 --- /dev/null +++ b/docs/operations/stripe-billing-setup-walkthrough.md @@ -0,0 +1,133 @@ +# Stripe billing setup walkthrough + +This closes production blocker 2. Complete the test-mode checklist first, prove the full lifecycle, +then repeat it in live mode. Stripe test and live objects are separate: keys, prices, webhook secrets, +customers and subscriptions cannot be mixed. + +The current production misconfiguration is specific: `STRIPE_PRICE_PREMIUM` contains a Product ID +(`prod_...`). JobTracker needs the recurring Price ID attached to that product (`price_...`). The API +deliberately treats the Product ID as disabled instead of sending an invalid Checkout request. + +## What you need + +- Access to the Stripe account that will receive JobTracker subscriptions. +- Access to `/opt/job-tracker/shared/.env` on the production host. +- One disposable JobTracker user for the test purchase. +- A decision on the monthly Pro price and currency. Start with one monthly price; annual billing can + be added after the monthly lifecycle is proven. + +Never paste Stripe keys or webhook signing secrets into Git, chat, screenshots, tickets or CI logs. + +## Part 1 — create the test subscription price + +1. Sign in to the Stripe Dashboard and enable **Test mode**. +2. Open **Product catalog** and either open the existing JobTracker Pro product or create one named + `JobTracker Pro`. +3. Add a price with: + - pricing model: standard/flat rate; + - billing: recurring; + - interval: monthly; + - currency and amount: your chosen commercial terms. +4. Save the price. +5. Open the new price and copy its identifier. It must start with `price_`, not `prod_`. + +The product describes what is sold. The price describes how it is billed. Checkout needs the latter. + +## Part 2 — configure the customer portal + +1. In Stripe, open the customer-portal settings. +2. Enable the portal in test mode. +3. Allow customers to update payment details and cancel subscriptions. +4. Decide whether cancellation is immediate or at the end of the billing period. The application + follows Stripe's resulting subscription status; Stripe remains authoritative. +5. Save the portal configuration. + +## Part 3 — create the test webhook + +1. Open Stripe's webhook/event-destination area in test mode. +2. Add this public HTTPS endpoint: + + ```text + https://jobs.cesnimda.uk/api/billing/webhook + ``` + +3. Subscribe it to exactly these events: + + ```text + customer.subscription.created + customer.subscription.updated + customer.subscription.deleted + ``` + +4. Save the endpoint. +5. Reveal and copy that endpoint's signing secret. It starts with `whsec_`. + +The endpoint signing secret belongs to this specific webhook endpoint and mode. It is not the API +secret key shown on Stripe's API keys page. + +## Part 4 — install the three test values + +On the production host, edit `/opt/job-tracker/shared/.env` and set: + +```text +STRIPE_SECRET_KEY= +STRIPE_PRICE_PREMIUM= +STRIPE_WEBHOOK_SECRET= +``` + +Before saving, verify all three values came from test mode. Do not mix a live key with a test price or +test webhook secret. Redeploy after saving so the backend receives the new environment. + +## Part 5 — prove the test lifecycle + +Use a disposable authenticated JobTracker account: + +1. Open account/settings and confirm billing is available. +2. Select **Upgrade**. Stripe-hosted Checkout must open for the expected monthly amount and currency. +3. Complete Checkout using a Stripe test card. +4. Confirm Stripe redirects to `/settings?billing=success`. +5. In Stripe's webhook delivery log, confirm the subscription event received a successful `2xx` + response. +6. Refresh JobTracker and confirm the user has Pro/Premium entitlements. +7. Open **Manage billing** and confirm the Stripe customer portal opens. +8. Cancel the test subscription using the portal. +9. Confirm the updated/deleted subscription webhook succeeds and JobTracker removes Premium after + the subscription is no longer active or trialling. +10. Replay one already-successful event from Stripe. JobTracker's stored state and role must remain + correct; replay must not duplicate or corrupt billing state. + +Do not continue to live mode if any webhook delivery is red, Checkout shows the wrong amount, or the +role does not follow Stripe's subscription state. + +## Part 6 — live-mode cutover + +1. Disable Test mode in Stripe. +2. Create or open the live JobTracker Pro product and create a live recurring monthly price. +3. Configure the live customer portal. +4. Create the live webhook endpoint with the same URL and three subscription events. +5. Replace all three production environment values together: + - live `sk_live_...` secret key; + - live `price_...` recurring Price ID; + - live endpoint-specific `whsec_...` signing secret. +6. Redeploy. +7. Complete one low-risk real subscription, portal visit and cancellation. +8. Confirm every live webhook delivery is successful and the JobTracker role follows the live + subscription state. + +## Troubleshooting + +| Symptom | Check | +| --- | --- | +| Billing appears unavailable | `STRIPE_PRICE_PREMIUM` must begin `price_`; all three settings must be present. | +| Checkout says the price does not exist | The API key and Price ID probably come from different Stripe modes or accounts. | +| Webhook signature fails | Use the endpoint's `whsec_...` secret, not an API key or a Stripe CLI temporary secret. | +| Checkout succeeds but Premium is absent | Inspect the webhook delivery response; JobTracker provisions from signed subscription events. | +| Portal does not open | Enable and save the customer portal in the same Stripe mode as the customer. | +| Cancellation does not remove Premium | Check the resulting Stripe status and replay the authoritative subscription event. | + +## Safe rollback + +Removing all three Stripe environment values and redeploying disables new Checkout and portal +sessions without deleting billing history. Do not manually remove Premium while Stripe still reports +an active subscription. Repair webhook delivery and replay the authoritative event instead. +