docs(release): scope legacy rollback guidance to its migration baseline

docs(release): record offline rollback health verification
fix(release): verify core health after automatic image rollback
2026-11-11 07:32:04 +01:00 · 2026-11-11 07:20:46 +01:00 · 2026-11-11 07:19:45 +01:00 · 2026-11-10 13:26:37 +01:00 · 2026-11-10 13:24:30 +01:00 · 2026-11-10 13:21:56 +01:00
15 changed files with 667 additions and 33 deletions
@@ -3,3 +3,7 @@
 The deployment script and Gitea workflow live in this directory and `.gitea/workflows/`. The current deployment, health/readiness, rollback, and secret-handling guidance is in the repository [infrastructure guide](../docs/infrastructure/README.md), especially [deployment](../docs/infrastructure/deployment.md) and [disaster recovery](../docs/infrastructure/disaster-recovery.md).

 Keep the server `.env` outside the repository. Do not run production deployment, restore, migration, or credential rotation as part of local documentation work.
+
+MariaDB backup connection parsing uses `scripts/parse-backup-connection.py` (Python 3, already required by deployment preflight). Keys are case-insensitive and support the documented host/user/password/database aliases. Single/double-quoted values may contain semicolons and doubled quote characters. The dump contract requires one TCP host, a database and user, with port 1-65535 (default 3306); an empty password is retained. Duplicate aliases, control characters, host lists/socket paths and database path separators are rejected before dumping. This deliberately bounded parser is not the full MySqlConnector configuration engine. Additional connection options do not change existing dump-client TLS behavior; that qualification remains separate.
+
+Run `python scripts/test-deploy-backup.py` for the offline synthetic dump-boundary regression. It extracts only the backup function and never executes the deployment entry point, contacts a database or reads the production environment.
@@ -326,19 +326,19 @@ backup_database() {
    return 1
  fi

-  # Parse the ADO.NET connection string. Keys are case-insensitive in .NET, so match that.
-  local db_host db_port db_name db_user db_pass
-  db_host="$(sed -n 's/.*[Ss]erver=\([^;]*\).*/\1/p' <<<"$cs")"
-  db_port="$(sed -n 's/.*[Pp]ort=\([^;]*\).*/\1/p' <<<"$cs")"
-  db_name="$(sed -n 's/.*[Dd]atabase=\([^;]*\).*/\1/p' <<<"$cs")"
-  db_user="$(sed -n 's/.*[Uu]ser[ ]*[Ii]*[Dd]*=\([^;]*\).*/\1/p' <<<"$cs")"
-  db_pass="$(sed -n 's/.*[Pp]assword=\([^;]*\).*/\1/p' <<<"$cs")"
-  db_port="${db_port:-3306}"
-
-  if [ -z "$db_host" ] || [ -z "$db_name" ] || [ -z "$db_user" ]; then
-    echo "Could not parse host/database/user from JOBTRACKER_CONNECTION_STRING. Aborting deploy."
+  # Keep secrets off argv; parse aliases/quoted separators without evaluating shell text.
+  local db_host db_port db_name db_user db_pass parsed_connection
+  local -a connection_fields=()
+  if ! parsed_connection="$(printf '%s' "$cs" | python3 scripts/parse-backup-connection.py)"; then
+    echo "Could not parse JOBTRACKER_CONNECTION_STRING for backup. Aborting deploy."
    return 1
  fi
+  mapfile -t connection_fields <<< "$parsed_connection"
+  db_host="${connection_fields[0]}"
+  db_port="${connection_fields[1]}"
+  db_name="${connection_fields[2]}"
+  db_user="${connection_fields[3]}"
+  db_pass="${connection_fields[4]}"

  local target="$BACKUP_DIR/jobtracker-${db_name}-${BACKUP_STAMP}.sql.gz"
  echo "Backing up MariaDB database '${db_name}' on ${db_host}:${db_port} to ${target}"
@@ -525,12 +525,14 @@ apply_backup_retention() {
    return 1
  fi

-  local month monthly_root monthly_dir
+  local month monthly_root monthly_dir complete_tmp
  month="$(TZ="$timezone" date +%Y-%m)"
  monthly_root="$BACKUP_DIR/monthly"
  monthly_dir="$monthly_root/$month"
-  mkdir -p "$monthly_root"
-  chmod 700 "$monthly_root"
+  if ! mkdir -p "$monthly_root" || ! chmod 700 "$monthly_root"; then
+    echo "Could not prepare monthly backup directory. Aborting deploy."
+    return 1
+  fi

  local current_files=("$BACKUP_DIR"/jobtracker-*"-$BACKUP_STAMP".*)
  if [ ! -e "${current_files[0]}" ]; then
@@ -539,11 +541,21 @@ apply_backup_retention() {
  fi

  if [ ! -f "$monthly_dir/.complete" ]; then
-    mkdir -p "$monthly_dir"
-    chmod 700 "$monthly_dir"
-    cp --preserve=mode,timestamps -- "${current_files[@]}" "$monthly_dir/"
-    printf '%s\n' "$BACKUP_STAMP" > "$monthly_dir/.complete"
-    chmod 600 "$monthly_dir/.complete"
+    if ! mkdir -p "$monthly_dir" || ! chmod 700 "$monthly_dir" ||
+        ! cp --preserve=mode,timestamps -- "${current_files[@]}" "$monthly_dir/"; then
+      echo "Could not retain monthly recovery files. Aborting deploy."
+      return 1
+    fi
+    if ! complete_tmp="$(mktemp "$monthly_dir/.complete.XXXXXX")"; then
+      echo "Could not create monthly recovery marker. Aborting deploy."
+      return 1
+    fi
+    if ! printf '%s\n' "$BACKUP_STAMP" > "$complete_tmp" ||
+        ! mv -T -- "$complete_tmp" "$monthly_dir/.complete"; then
+      rm -f -- "$complete_tmp"
+      echo "Could not publish monthly recovery marker. Aborting deploy."
+      return 1
+    fi
    echo "Monthly recovery set retained: ${monthly_dir}"
  fi

@@ -574,14 +586,20 @@ upload_current_backup_set() {
  remote="${remote%/}"
  local current_files=("$BACKUP_DIR"/jobtracker-*"-$BACKUP_STAMP".*) file
  for file in "${current_files[@]}"; do
-    rclone copyto "$file" "$remote/daily/$(basename "$file")" --immutable
+    if ! rclone copyto "$file" "$remote/daily/$(basename "$file")" --immutable; then
+      echo "Daily off-host backup upload failed. Aborting deploy."
+      return 1
+    fi
  done

  local month="$(TZ="${BACKUP_TIMEZONE:-Europe/Oslo}" date +%Y-%m)"
  if [ -d "$BACKUP_DIR/monthly/$month" ]; then
-    rclone copy "$BACKUP_DIR/monthly/$month" "$remote/monthly/$month" --immutable
+    if ! rclone copy "$BACKUP_DIR/monthly/$month" "$remote/monthly/$month" --immutable; then
+      echo "Monthly off-host backup upload failed. Aborting deploy."
+      return 1
+    fi
  fi
-  echo "Current recovery set copied to configured encrypted off-host remote."
+  echo "Current recovery set copied to configured off-host remote."
 }

 resolve_existing_web_proxy_subnet
@@ -642,10 +660,18 @@ finish_deploy() {

  if [ "$status" -ne 0 ] && [ "$replacement_started" = true ]; then
    if [ -n "$previous_backend_ref" ] && [ -n "$previous_frontend_ref" ]; then
-      echo "Deployment failed after core replacement; restoring previous healthy images."
+      echo "Deployment failed after core replacement; restarting previous core images."
      if BACKEND_IMAGE="$previous_backend_ref" FRONTEND_IMAGE="$previous_frontend_ref" \
          compose up -d --force-recreate --remove-orphans backend frontend; then
-        echo "Previous core release restored. Deployment remains failed; inspect the logs above."
+        if curl -fsS --connect-timeout 5 --max-time 5 --retry 5 --retry-delay 2 --retry-connrefused --retry-max-time 30 "${APP_PUBLIC_BASE_URL%/}/" >/dev/null &&
+            curl -fsS --connect-timeout 5 --max-time 5 --retry 5 --retry-delay 2 --retry-connrefused --retry-max-time 30 "${APP_PUBLIC_BASE_URL%/}/health" >/dev/null &&
+            curl -fsS --connect-timeout 5 --max-time 5 --retry 5 --retry-delay 2 --retry-connrefused --retry-max-time 30 "${APP_PUBLIC_BASE_URL%/}/ready" >/dev/null; then
+          echo "Previous core images restarted and public core checks passed. Deployment remains failed; inspect the logs above."
+        else
+          echo "Previous core images restarted, but rollback health verification failed. Manual recovery is required."
+          compose ps || true
+          compose logs --tail=200 backend frontend || true
+        fi
      else
        echo "Automatic core rollback failed. Manual recovery is required."
        compose ps || true
@@ -1,5 +1,11 @@
 # First production deployment runbook

+> Historical July 2026 baseline. Current release preparation follows
+> [the release qualification matrix](../docs/release-readiness/14_RELEASE_ACCEPTANCE_MATRIX.md) and
+> [recovery specification](../docs/release-readiness/11_BACKUP_RESTORE_ROLLBACK.md).
+> Reconcile the examples below with the exact candidate, migration history and isolated rehearsal
+> before operational use. Current rollback health checks are source-tested offline.
+
 > Written 2026-07-19 against the actual implementation, not against the other docs. Every command and
 > failure mode below was exercised locally against MariaDB 11 containers. No production data was used.
 >
@@ -220,9 +226,11 @@ everything written since the dump — including anything users did during and af
 **Restore the database only if the data itself is wrong.** If the backend simply will not start, the
 code rollback is almost certainly the whole fix.

-Why a code rollback is safe here: every Phase 4/5 migration is a no-op — the reconciler owns those
-tables — so reverting code never leaves migration history ahead of the schema. The reconciler is
-additive and never drops a table holding rows, so the older code simply ignores the newer tables.
+The July 2026 baseline used no-op Phase 4/5 migrations around reconciler-owned tables. Current
+migrations also perform schema changes, including `AddJobRevision` adding the application Version
+column and `AddUserOperationEvents` creating its table and indexes. Qualify the exact old/new image
+pair against the current migration history and representative post-upgrade writes. The recovery
+specification above includes ownership, revision, document and restore checks for that rehearsal.

 ## Procedure

@@ -0,0 +1,23 @@
+# Release discovery · endpoint and environment census
+
+Source baseline `f19e7ebf4aeae350b43fa84cf115c393030280f0` on `codex/release-backup-parser`. Read-only source census. The 41 unique controller route prefixes are grouped below. Controller and action attributes, fallback authorisation, proxy policy, and runtime responses are separate checks. Repeat endpoint-level runtime verification on the accepted release candidate.
+
+| Surface | Route owners under `/api` | Qualification owner and required failure cases |
+| --- | --- | --- |
+| Identity and accounts | `auth`, `auth/2fa`, `auth/sessions`, `account-lifecycle`, `export` | Auth/session, account lifecycle and export tests. Cover invalid/expired/revoked session, CSRF for cookie writes, rate limits, recent sign-in, owner isolation and final deletion policy. |
+| Job tracking and discovery | `jobapplications`, `jobapplications/{jobId:int}` and its `checklist`, `focus-plan`, `workspace`, `ai` paths, `calendar`, `companies`, `jobimport`, `job-discovery` | Ownership, RD21, application/workspace, calendar and import/provider tests. Cover foreign/missing IDs, stale writes, parser/source denial and documented provider rights. |
+| Career and files | `career/evidence`, `career/profile`, `profile-cv`, `cv`, `documents`, `attachments`, `public-cv`, `backup` | CV revision, import, document, public CV and backup tests. Cover frozen revisions, public opt-in and withdrawal, private downloads, upload limits, physical renderer qualification and scoped restoration. |
+| Mail | `correspondence`, `email`, `email/drafts`, `gmail`, `imap`, `microsoft-graph` | Email ownership, Gmail, provider and draft tests. Verify OAuth state, consent/expiry, mailbox ownership, idempotent send, uncertain outcome recovery and real delivery separately. |
+| Personal processing | `ai/settings`, `ai/usage`, `notification-settings`, `notifications`, `operations`, `rules` | Tenant/worker/AI tests. Verify entitlement and account isolation, no model start by default, queue and lease recovery, and redacted failures. |
+| Administration and billing | `admin/audit`, `admin/processing`, `admin/system`, `users`, `billing`, `client-errors` | AdminProcessing, Users, AuthAndSystem, Billing and ClientErrors tests. Cover role denial, minimisation, signed webhooks, safe input and rate/size controls. Client error intake relies on the configured fallback auth policy, so verify both supported configurations. |
+
+Outside the controllers, `GET /health` reports process/build liveness and `GET /ready` checks database connectivity with a terse 503 on failure. They cannot stand in for authenticated application smoke. OpenAPI is mapped only in Development. Explicit anonymous controller paths include auth configuration and entry/recovery flows, the 2FA challenge, provider callbacks, Stripe webhook and opted-in public CV HTML/PDF. They still need route-specific proof of token, signature, consent or data scope as applicable. Public CV HTML carries a noindex header and the PDF action has a named rate limit. Source inspection does not establish actual ingress controls.
+
+| Environment | Source contract | Evidence required at the gate |
+| --- | --- | --- |
+| Local dev | Development compose has backend, frontend and optional local AI service. Development MariaDB uses a persistent dev volume. | Confirm isolated config, version and selected service flags. Keep developer data and credentials out of shared artifacts. |
+| Disposable integration | Database compose pins MariaDB 12.2.2 by digest and declares a separate test service. | Prove ephemeral storage, port and project isolation, migrations and cleanup. Keep the hosted isolation gate until operator evidence exists. |
+| Staging and recovery rehearsal | The release procedure requires an isolated, approved target. No running target or staging identity was inspected here. | Freeze image/config/migration IDs, scope credentials and network, use synthetic or approved sanitised data, and prove rollback/restore and alert behaviour. |
+| Production | Production compose separates frontend, backend, AI service and Ollama. Exact deployed versions, ingress, mounted data and service activation are external evidence. | Verify proxy and TLS, only intended exposed routes, build identity, readiness, provider/worker boundaries, and the operator's approved release decision. |
+
+No API, hosted CI, staging, production, Docker, provider or model operation was executed for this census. No configuration secret values were read. Stage 1 has a source-bound surface map. Exact endpoint/runtime evidence, candidate identity and recovery qualification remain open.
@@ -11,7 +11,7 @@
 | `deploy/deploy.sh:backup_database` | Requires explicit provider; SQL dump for MariaDB, volume archive for SQLite; rejects missing/empty/malformed/trailerless dumps | Format checks do not restore or compare records; emergency skip exists and is not approved here |
 | `backup_recovery_volumes` | MariaDB additionally archives required application data volume; independently archives tombstones when present | Consistency between dump/files/tombstones and current key availability still needs isolated rehearsal |
 | Retention/off-host copy | Daily/monthly retention and optional rclone copy | Source message calls remote encrypted but source does not establish actual remote encryption configuration; off-host configuration and recovery remain unverified |
-| `finish_deploy` | On failure after replacement, restarts previous backend/frontend image references when both exist; retains failed exit | No database reversal, no previous AI image restoration, and no post-rollback health check in this handler |
+| `finish_deploy` | On failure after replacement, restarts previous backend/frontend references, checks public frontend/liveness/readiness with bounded retries and retains failed exit | Offline stubs verify control flow. Actual recovery, image identity, database reversal and previous AI image restoration remain unqualified |
 | Tombstone store and replay | Validates ledger records; Program invokes `StageRestoredAccountsAsync` at startup; deleted identities are staged again | Test restored data with latest independent tombstones; corrupt ledger must fail closed. Existing AccountDeletionTests include restore replay, not a complete operational restore |

 Build retry paths prune Docker build/system cache and deployment may warm an Ollama model. Therefore even a proposed "local deploy smoke" must not run this script against the shared workstation. No such action was taken.
@@ -34,3 +34,48 @@ Proposed future remediation: parse with the same supported connection-string sem
 6. Measure recovery duration and data-loss window against owner-approved RTO/RPO. No target or measurement exists in this checkpoint. Preserve cleanup and evidence identities; no production execution or release approval implied.

 Fresh checks: Git Bash `bash -n deploy/deploy.sh` PASSED (syntax only); `python scripts/test-mariadb-workflow.py` PASSED 4/4 (source quarantine guards only). These are separate from the earlier 5/5 offline runner tests. Restore, failure injection, hosted isolation, live encryption and rollback health remain NOT TESTED.
+
+## RR-12 offline failure injection
+
+Tested the unchanged `finish_deploy` function and EXIT trap from baseline `47c6834cee192a5f1f054ddcd500b4d45c48c985`, `deploy/deploy.sh` blob `8341110de5a827698efc6eeb6951f296770562da`. The test extracts only the handler and trap and supplies synthetic image refs and mock Compose and cleanup functions. The full deploy script was never sourced or executed.
+
+| Case | Observed behavior |
+| --- | --- |
+| Replacement started, both previous core image references exist | Recreates backend and frontend with those refs only. The failed deployment status is preserved. No health probe follows the successful Compose call. |
+| Compose rejects rollback | Attempts status and backend/frontend log collection. The original failure status is preserved, even when the rollback command returns a different status. |
+| Either or both previous references are missing | Skips automatic recreation and reports that no complete prior core release is available. |
+| Deployment fails before replacement, or succeeds | Skips automatic recreation. |
+
+Both auth-config temporary-file cleanup calls occur before the rollback decision. The test stubs deletion and checks the exact call sequence.
+
+- `python scripts/test-deploy-rollback.py -v` **PASSED 5/5**, including three missing-reference combinations.
+- `python scripts/test-deploy-backup.py -v` **PASSED 5/5** and `python scripts/test-mariadb-workflow.py -v` **PASSED 4/4**. Git Bash `bash -n deploy/deploy.sh` passed syntax validation.
+
+RR-12 remains open for an authorised isolated rehearsal. After the prior images are recreated, verify exact image identity, `/health`, `/ready`, and authenticated reads and writes against representative migrated and post-write data. A successful Compose call does not establish service health or recover schema, data, or the AI image. RR-13 still needs authorised off-host retrieval/decryption and recovery-set consistency verification. No Docker, network service, database, remote storage, deployment or restore was invoked by these checks.
+
+## RR-13 off-host upload failure handling
+
+Reproduced two transfer failures in the deployment gate using the real `upload_current_backup_set` function, synthetic files and a stub `rclone`. The caller uses `! upload_current_backup_set`, so Bash suppresses errexit inside that function. When either daily `copyto` or monthly `copy` returned 23, the old code still printed success and the gate returned 0. The healthy control selected only current-stamp daily files, then the monthly directory.
+
+The bounded fix explicitly returns failure on either copy error. A daily error stops before subsequent files and the monthly copy. A monthly error stops before the deployment continues. The successful path still uses immutable transfers and no longer claims the configured remote is encrypted. A failed upload may leave some remote files already transferred.
+
+- Red run: **2 failures and 1 passing control**. Green run: **8/8 backup tests** including the three upload cases. The independent rollback handler and hosted quarantine guard checks passed **5/5** and **4/4**. Bash syntax validation passed.
+- Encrypted off-host retrieval, decryption, completeness and consistent restore of database/files/keys/tombstones remain required. No actual transfer, restore, production configuration, keys or backup files were accessed.
+
+## RR-13 monthly completion marker
+
+The same conditional Bash behavior applies to `apply_backup_retention`. With a stubbed partial file copy returning 23, the old function wrote `.complete`, reported the month retained and continued through the mocked pruning, upload and deployment boundary. The successful control retained byte-identical synthetic files and the expected stamp.
+
+The fix explicitly checks preparation and copy. It writes the stamp to a temporary file in the monthly directory and renames it to `.complete` only after the copy and stamp write succeed. A failed rename blocks later steps and attempts to remove the temporary marker. Existing completed snapshots are preserved. A failed copy can leave unmarked files, and previously written markers still require verification. A marker records the selected stamp and successful local copy; it does not attest to recovery-set integrity.
+
+- Red run: **2 failures and 1 passing control** against the pre-fix function. The partial-copy case directly reproduced false completion; the marker-publication case required an atomic rename the old function did not perform.
+- Green run: **12/12 backup tests** including four monthly cases (successful copy and stamp, partial copy, failed marker rename, existing completed month). Rollback handler and hosted quarantine checks passed **5/5** and **4/4**. Bash syntax validation passed.
+- Only temporary synthetic files were created or copied. Pruning and upload were stubbed. Approved rehearsal still needs independent checks of artifacts for the marker stamp, encryption, retrieval and actual restore before any release decision.
+
+## RR-12 post-rollback core health checks
+
+11 November 2026. Source `a39ac847298e1fbe7667cf42671e7ec265500fcd` on `codex/release-rollback-health`, based on `24a62997`. After a successful previous-image restart, the handler now checks the configured public frontend, `/health` and `/ready`. Each curl request has connection/request timeouts and bounded retries for startup failures. A failed check reports that manual recovery is required and collects status/logs. The deployment's original nonzero status is retained even if diagnostics fail. The success message is limited to restarted core images and passing public core checks.
+
+The red test run executed six methods and reported four assertion failures because the old handler performed no probes. The green run passed six methods, including failed-probe subcases for all three endpoints, Compose failure, incomplete references, pre-replacement failure and successful deployment. Probe failures and Compose failure also use failing diagnostic stubs to verify original-status preservation. Twelve backup tests and shell syntax validation passed separately. Earlier five-test receipts remain historical and overlap this run.
+
+Only the extracted exit handler/trap ran with shell functions substituting Compose, Docker, curl and cleanup. The full deployment script was never sourced or executed, and no network/containers/models were used. Raw red/green/backup/syntax receipts are in `evidence/rollback-health/`. Actual startup timing, proxy routing, exact image identity, post-migration writes and database/files/keys/tombstone recovery still need the authorised isolated rehearsal. Passing public probes alone does not establish recovery correctness.
@@ -4,7 +4,7 @@ Discovery only, 2026-11-02. These are qualification gaps unless a concrete defec

 | ID | Historical item / current finding | Status / closure evidence needed |
 | --- | --- | --- |
-| RR-01 | Phase 9/10 and production NO-GO | Phase 10 assessment now independently published, implementation still awaits exact approval. Later roadmap acceptance absent. BLOCKED for final RC |
+| RR-01 | Phase 9/10 and production NO-GO | CLEAN-20 implementation and verification are published. Independent Phase 10 acceptance and later roadmap acceptance remain open. BLOCKED for final RC |
 | RR-02 | Old Linux 11/24 and later S01 failure narratives | Latest handoff records approved S01 36/36 per platform and owner visual acceptance for unchanged development output. Do not reopen these metrics without affected change; physical print/device remain NOT TESTED |
 | RR-03 | Python advisory/native parser/image qualification | Latest scoped conditional disposition retained; current advisories and production-equivalent native isolation NOT TESTED. No new scan or risk waiver |
 | RR-04 | Hosted MariaDB/isolation | Source explicitly fails before execution until isolated runner proof. BLOCKED; fresh local 5 guard tests do not close it |
@@ -14,8 +14,9 @@ Discovery only, 2026-11-02. These are qualification gaps unless a concrete defec
 | RR-08 | MySQL TLS / dependency / production identity | Latest handoff retains MYSQL-TLS-01 and OPS-01. No live configuration read; current advisory disposition and deployed equivalence unverified |
 | RR-09 | Backup/restore and rollback | Existing procedures and historical seeded rehearsal are not a current restore. Need isolated recovery-set completeness, decryption, tombstone behavior, post-write rollback compatibility and agreed RPO/RTO |
 | RR-10 | AI quality/activation | Deterministic Docling fixtures are distinct from real-model semantic accuracy. Optional unqualified adapter remains disabled; no AI invocation authorised here |
-| RR-11 | Backup host parsing rejects supported `Host=` alias; uppercase spelling also unrecognised | REPRODUCED with synthetic input and shell-native positive control. Deployment aborts before backup; proposed P2 compatibility defect. Fix scope awaits authorised release remediation |
-| RR-12 | Rollback restores core images without post-rollback health qualification; no schema/data/AI reversal | VERIFIED source limitation, runtime impact NOT TESTED. Requires isolated post-migration/post-write recovery rehearsal; no rollback guarantee |
-| RR-13 | Off-host encryption and recovery-set consistency not established by script success messages | NOT TESTED operational qualification. Verify remote encryption, retrieval/decryption, database/files/key/tombstone consistency in approved isolation |
+| RR-11 | Backup host parsing rejected supported `Host=` alias and uppercase spelling | FIXED AND OFFLINE-TESTED. See [bounded remediation](RR-11_IMPLEMENTATION.md). Real dump/restore qualification remains open |
+| RR-12 | Previous handler omitted post-rollback health checks; schema/data/AI recovery remains separate | Health probes FIXED in `a39ac847` and six offline handler methods passed, including failed endpoints and diagnostics. See [health follow-up](11_BACKUP_RESTORE_ROLLBACK.md#rr-12-post-rollback-core-health-checks). Actual image identity, startup timing and recovery after migration/writes still require isolated rehearsal |
+| RR-13 | Off-host encryption and recovery-set consistency need independent proof. Failed transfer and partial monthly copy previously let deployment continue | Transfer and monthly marker failure gates FIXED AND OFFLINE-TESTED. Actual encryption, retrieval/decryption and database/files/key/tombstone consistency require approved isolation |
+| RR-14 | July first-deploy runbook inferred rollback safety from its no-op migration baseline, while current migration Up methods add schema | Historical scope and current qualification links clarified. `AddJobRevision` adds Version and `AddUserOperationEvents` creates a table/indexes. Exact old/new image compatibility after migration/writes remains an isolated rehearsal gate |

-Next remediation is evidence work: complete the source/endpoint/environment census and resolve historical records against their exact latest evidence. RR-11 now has a bounded future fix proposal in 11_BACKUP_RESTORE_ROLLBACK.md; no runtime implementation is authorised by this discovery checkpoint. Every fix needs minimal scope, targeted verification and the applicable stage approval.
+Continue evidence work against the [source endpoint and environment census](03_ENDPOINT_ENVIRONMENT_CENSUS.md). Actual qualification requires an accepted candidate and isolated environment. Every fix needs minimal scope, targeted verification and the applicable stage approval.
@@ -1,5 +1,17 @@
 # Release discovery handoff

+11 November. Branch `codex/release-rollback-health`, baseline `24a62997`, source `a39ac847` adds bounded post-rollback frontend/liveness/readiness probes and failing-diagnostics coverage while preserving the original exit status. Six extracted-handler tests, twelve backup tests and shell syntax validation passed. Full deploy, Docker, network, models and real restore were not executed. [Evidence and remaining qualification](11_BACKUP_RESTORE_ROLLBACK.md#rr-12-post-rollback-core-health-checks).
+
+RR-14 clarifies that the first-production runbook is a July baseline. Current Up migrations add the Version column and operation-event schema. Its old no-op argument cannot qualify a current old/new image pair. Current candidate, migration/write and recovery rehearsal requirements are now linked from that runbook. This was source/document review only.
+
+RR-12 offline failure injection passed 5/5 against the unchanged deploy exit handler. The evidence covers synthetic prior-image recreation, failed rollback diagnostics, original exit status and conditions that skip rollback. [Recovery report](11_BACKUP_RESTORE_ROLLBACK.md#rr-12-offline-failure-injection) distinguishes this from a real post-migration/post-write rehearsal. RR-13 off-host retrieval/decryption and consistent restore remain open. No production qualification.
+
+RR-13 transfer guard [reproduced and fixed offline](11_BACKUP_RESTORE_ROLLBACK.md#rr-13-off-host-upload-failure-handling). Daily and monthly copy errors stop deployment, and the success log makes no encryption claim. Actual remote encryption and restore remain open.
+
+RR-13 [monthly completion guard](11_BACKUP_RESTORE_ROLLBACK.md#rr-13-monthly-completion-marker) now blocks the gate after a partial copy or marker publication failure. The successful path preserves artifact bytes and the stamp in synthetic fixtures. Existing markers and real restoration remain unverified.
+
+10 November: [endpoint and environment census](03_ENDPOINT_ENVIRONMENT_CENSUS.md) maps 41 source route families, anonymous exceptions and failure cases to qualification owners on the backup-parser branch. No runtime qualification. Carry the map into Stage 2 only after the release candidate and environment are accepted.
+
 Branch `planning/release-readiness-discovery`, baseline `5cb903b989742b7a96cb22ddf4c0e24993b18980`; documentation only. Stage 1 IN_PROGRESS, not complete. Full authoritative release prompt read. No final GO/NO-GO completion report or deployable release claimed.

 Completed: initial source inventory, historical blocker reconciliation, preliminary 18-gate matrix and five fresh offline MariaDB runner guard tests. Preserved scoped PDF acceptance and separate model/provider/human/hosted/production gates.
@@ -0,0 +1,19 @@
+# RR-11 bounded remediation
+
+Authorised by the owner's Continuous Roadmap Execution instruction explicitly prioritising this fix. Branch `codex/release-backup-parser`, based on published discovery `653f8d04d469dcd52c883d4b98bea76b8adacada`. Separate from CLEAN-20, full release qualification and infrastructure operations.
+
+Replaced five inconsistent sed expressions with one stdlib Python parser and a shell-safe field transfer. Case-insensitive aliases, quoted separators/doubled quotes, Unicode and empty passwords survive to the dump command. Invalid ports, incomplete/ambiguous/control-bearing values and unsupported multi-host/socket/path forms fail before dumping. No eval; connection input travels via stdin, not argv; errors never include input. No dependency added. Existing dump/TLS/retention/rollback behavior is otherwise unchanged.
+
+## Evidence
+
+- Regression first: four test methods ran against the original backup function; eight subcase failures reproduced alias/case rejection and quoted-password truncation. Canonical/negative controls executed. No real dump or deployment.
+- Final targeted suite: **5/5 methods PASSED**, including 7 host spellings, quoted semicolon/equal/doubled quote/shell characters, Unicode/single quotes, empty password, and six invalid/ambiguous input cases. Tests run the actual backup function and mock only external dump/Docker boundaries.
+- Full offline script unittest set: **31/31 PASSED**, including targeted cases, ingress Compose rendering with synthetic env, workflow/runner guards, retry guard, supply-chain fixtures and offline model-evaluation helpers. No model invocation. Printed exit17/exit1 are expected mocked runner negative cases. Do not add targeted five to the overlapping total.
+- Ordinary unittest discovery found zero tests because repository filenames contain hyphens; recorded as NOT A PASS. Explicit importlib loading of every `scripts/test-*.py` unittest module (excluding standalone Docker topology runner) executed the 31 tests. Docker topology qualification NOT RUN.
+- Git Bash `bash -n deploy/deploy.sh`: PASSED.
+- Graphify AST update completed locally: 18,504 nodes / 45,920 edges; missing optional PDF/SQL extractors reported. Newly generated untracked graph output retained locally, not a runtime verification or introduced repository artifact.
+- Product/backend/frontend/full database/browser suites NOT RUN: this package changes only deploy parsing and its offline tests. Real dump/restore, Linux-host operational behavior, hosted CI and production validation remain NOT TESTED.
+
+The first Unicode test attempt failed in the Windows test harness encoding before reaching product code; the harness was corrected to UTF-8, then passed. Parser stdin/stdout are explicitly UTF-8. No application user data or production credentials used.
+
+Rollback: revert this bounded implementation commit to restore the prior parser (and its known limitation). No database/schema change needs reversal. RR-11 is FIXED AND OFFLINE-TESTED on this development branch; RR-12/13 and production remain open. No main merge/deploy authorised.
@@ -0,0 +1,5 @@
+............
+----------------------------------------------------------------------
+Ran 12 tests in 5.884s
+
+OK
@@ -0,0 +1,73 @@
+FFFF....
+======================================================================
+FAIL: test_each_failed_health_probe_preserves_failure_and_requires_manual_recovery (__main__.DeployRollbackTests.test_each_failed_health_probe_preserves_failure_and_requires_manual_recovery) (path='/')
+----------------------------------------------------------------------
+Traceback (most recent call last):
+  File "C:\Users\Connor\.codex\worktrees\release-readiness-discovery\Job tracker\scripts\test-deploy-rollback.py", line 72, in test_each_failed_health_probe_preserves_failure_and_requires_manual_recovery
+    self.assertEqual(self.probes()[:index + 1], calls[1:-2])
+AssertionError: Lists differ: ['CALL curl -fsS --connect-timeout 5 --max[97 chars]id/'] != []
+
+First list contains 1 additional elements.
+First extra element 0:
+'CALL curl -fsS --connect-timeout 5 --max-time 5 --retry 5 --retry-delay 2 --retry-connrefused --retry-max-time 30 https://synthetic.invalid/'
+
+ []
+- ['CALL curl -fsS --connect-timeout 5 --max-time 5 --retry 5 --retry-delay 2 '
+-  '--retry-connrefused --retry-max-time 30 https://synthetic.invalid/']
+
+======================================================================
+FAIL: test_each_failed_health_probe_preserves_failure_and_requires_manual_recovery (__main__.DeployRollbackTests.test_each_failed_health_probe_preserves_failure_and_requires_manual_recovery) (path='/health')
+----------------------------------------------------------------------
+Traceback (most recent call last):
+  File "C:\Users\Connor\.codex\worktrees\release-readiness-discovery\Job tracker\scripts\test-deploy-rollback.py", line 72, in test_each_failed_health_probe_preserves_failure_and_requires_manual_recovery
+    self.assertEqual(self.probes()[:index + 1], calls[1:-2])
+AssertionError: Lists differ: ['CALL curl -fsS --connect-timeout 5 --max[247 chars]lth'] != []
+
+First list contains 2 additional elements.
+First extra element 0:
+'CALL curl -fsS --connect-timeout 5 --max-time 5 --retry 5 --retry-delay 2 --retry-connrefused --retry-max-time 30 https://synthetic.invalid/'
+
+ []
+- ['CALL curl -fsS --connect-timeout 5 --max-time 5 --retry 5 --retry-delay 2 '
+-  '--retry-connrefused --retry-max-time 30 https://synthetic.invalid/',
+-  'CALL curl -fsS --connect-timeout 5 --max-time 5 --retry 5 --retry-delay 2 '
+-  '--retry-connrefused --retry-max-time 30 https://synthetic.invalid/health']
+
+======================================================================
+FAIL: test_each_failed_health_probe_preserves_failure_and_requires_manual_recovery (__main__.DeployRollbackTests.test_each_failed_health_probe_preserves_failure_and_requires_manual_recovery) (path='/ready')
+----------------------------------------------------------------------
+Traceback (most recent call last):
+  File "C:\Users\Connor\.codex\worktrees\release-readiness-discovery\Job tracker\scripts\test-deploy-rollback.py", line 72, in test_each_failed_health_probe_preserves_failure_and_requires_manual_recovery
+    self.assertEqual(self.probes()[:index + 1], calls[1:-2])
+AssertionError: Lists differ: ['CALL curl -fsS --connect-timeout 5 --max[396 chars]ady'] != []
+
+First list contains 3 additional elements.
+First extra element 0:
+'CALL curl -fsS --connect-timeout 5 --max-time 5 --retry 5 --retry-delay 2 --retry-connrefused --retry-max-time 30 https://synthetic.invalid/'
+
+ []
+- ['CALL curl -fsS --connect-timeout 5 --max-time 5 --retry 5 --retry-delay 2 '
+-  '--retry-connrefused --retry-max-time 30 https://synthetic.invalid/',
+-  'CALL curl -fsS --connect-timeout 5 --max-time 5 --retry 5 --retry-delay 2 '
+-  '--retry-connrefused --retry-max-time 30 https://synthetic.invalid/health',
+-  'CALL curl -fsS --connect-timeout 5 --max-time 5 --retry 5 --retry-delay 2 '
+-  '--retry-connrefused --retry-max-time 30 https://synthetic.invalid/ready']
+
+======================================================================
+FAIL: test_failed_replacement_recreates_only_prior_core_images (__main__.DeployRollbackTests.test_failed_replacement_recreates_only_prior_core_images)
+----------------------------------------------------------------------
+Traceback (most recent call last):
+  File "C:\Users\Connor\.codex\worktrees\release-readiness-discovery\Job tracker\scripts\test-deploy-rollback.py", line 55, in test_failed_replacement_recreates_only_prior_core_images
+    self.assertEqual([
+AssertionError: Lists differ: ['CAL[121 chars]ous]', 'CALL curl -fsS --connect-timeout 5 --m[398 chars]ady'] != ['CAL[121 chars]ous]']
+
+First list contains 3 additional elements.
+First extra element 1:
+'CALL curl -fsS --connect-timeout 5 --max-time 5 --retry 5 --retry-delay 2 --retry-connrefused --retry-max-time 30 https://synthetic.invalid/'
+
+Diff is 808 characters long. Set self.maxDiff to None to see it.
+
+----------------------------------------------------------------------
+Ran 6 tests in 0.420s
+
+FAILED (failures=4)
@@ -0,0 +1 @@
+Git Bash bash -n deploy/deploy.sh exited 0. Full script was not executed.
@@ -0,0 +1,5 @@
+......
+----------------------------------------------------------------------
+Ran 6 tests in 0.374s
+
+OK
@@ -0,0 +1,68 @@
+"""Read a deployment connection string from stdin; emit five fields plus a sentinel.
+
+This is a single-host dump contract, not a replacement for MySqlConnector. Reject
+ambiguous aliases/control characters instead of choosing a different backup target.
+Never include input values in errors or pass the connection string as an argument.
+"""
+import re
+import sys
+
+
+ALIASES = {
+    **dict.fromkeys(['server', 'host', 'data source', 'datasource', 'address', 'addr', 'network address'], 'host'),
+    **dict.fromkeys(['user id', 'userid', 'username', 'uid', 'user name', 'user'], 'user'),
+    **dict.fromkeys(['password', 'pwd'], 'password'),
+    **dict.fromkeys(['database', 'initial catalog'], 'database'),
+    'port': 'port',
+}
+PAIR = re.compile(r'''\s*([^=;]+?)\s*=\s*(?:"((?:""|[^"])*)"|'((?:''|[^'])*)'|([^;]*))\s*(?:;|$)''')
+
+
+def parse(connection):
+    if any(ord(c) < 32 or ord(c) == 127 for c in connection):
+        raise ValueError('Control characters are unsupported in backup connection settings.')
+    fields = {}
+    position = 0
+    while position < len(connection):
+        if connection[position] == ';' or connection[position].isspace():
+            position += 1
+            continue
+        match = PAIR.match(connection, position)
+        if not match:
+            raise ValueError('Malformed backup connection settings.')
+        key, double, single, plain = match.groups()
+        if double is not None:
+            value = double.replace('""', '"')
+        elif single is not None:
+            value = single.replace("''", "'")
+        else:
+            value = plain.strip()
+            if value.startswith(('"', "'")):
+                raise ValueError('Malformed quoted backup connection value.')
+        canonical = ALIASES.get(key.strip().lower())
+        if canonical:
+            if canonical in fields:
+                raise ValueError('Duplicate backup connection setting or alias.')
+            fields[canonical] = value
+        position = match.end()
+    if any(not fields.get(key, '').strip() for key in ['host', 'database', 'user']):
+        raise ValueError('Backup connection requires host, database and user.')
+    port = fields.get('port', '3306')
+    if not re.fullmatch(r'[0-9]+', port) or not 1 <= int(port) <= 65535:
+        raise ValueError('Backup connection port must be between 1 and 65535.')
+    if any(c in fields['host'] for c in ',/\\'):
+        raise ValueError('Backup requires one TCP host, not a host list or socket path.')
+    if any(c in fields['database'] for c in '/\\'):
+        raise ValueError('Database name cannot contain backup path separators.')
+    return [fields['host'], str(int(port)), fields['database'], fields['user'], fields.get('password', '')]
+
+
+if __name__ == '__main__':
+    try:
+        values = parse(sys.stdin.buffer.read().decode('utf-8'))
+    except (ValueError, UnicodeError):
+        print('Invalid or unsupported backup connection settings.', file=sys.stderr)
+        sys.exit(1)
+    sys.stdout.reconfigure(encoding='utf-8', newline='\n')
+    # Sentinel keeps an empty final password intact through shell command substitution.
+    sys.stdout.write('\n'.join([*values, 'END']) + '\n')
@@ -0,0 +1,239 @@
+"""Exercise deployment backup functions with synthetic commands; never deploy."""
+import os
+from pathlib import Path
+import shlex
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+
+ROOT = Path(__file__).resolve().parents[1]
+BASH = 'C:/Program Files/Git/bin/bash.exe' if os.name == 'nt' else shutil.which('bash')
+
+
+class BackupConnectionTests(unittest.TestCase):
+    def backup(self, connection):
+        source = (ROOT / 'deploy/deploy.sh').read_text(encoding='utf-8')
+        start = source.index('backup_database() {')
+        function = source[start:source.index('\n}\n', start) + 3]
+        with tempfile.TemporaryDirectory() as temporary:
+            folder = Path(temporary)
+            script = '\n'.join([
+                'set -euo pipefail',
+                'BACKUP_DIR=' + shlex.quote(folder.as_posix()),
+                'BACKUP_STAMP=synthetic',
+                'DATABASE_PROVIDER=mariadb',
+                'JOBTRACKER_CONNECTION_STRING=' + shlex.quote(connection),
+                'python3() { ' + shlex.quote(Path(sys.executable).as_posix()) + ' "$@"; }',
+                'docker() { echo "Unexpected Docker call" >&2; return 99; }',
+                'mariadb-dump() { printf "%s\\0" "$@" > "$BACKUP_DIR/args"; '
+                'printf "%s" "$MYSQL_PWD" > "$BACKUP_DIR/password"; '
+                'printf "CREATE TABLE synthetic (id int);\\n-- Dump completed\\n"; }',
+                'verify_sql_backup() { gzip -t "$1"; }',
+                function,
+                'backup_database',
+            ])
+            result = subprocess.run([BASH, '--noprofile', '--norc'], input=script,
+                                    text=True, encoding='utf-8', capture_output=True, cwd=ROOT,
+                                    env={**os.environ, 'DEPLOY_SKIP_DB_BACKUP': 'false'})
+            arguments = (folder / 'args').read_bytes().split(b'\0')[:-1] if (folder / 'args').exists() else None
+            password = (folder / 'password').read_text(encoding='utf-8') if (folder / 'password').exists() else None
+            return result, arguments, password
+
+    def test_supported_host_alias_and_case_reach_dump_unchanged(self):
+        for host in ['Server', 'SERVER', 'Host', 'Data Source', 'Address', 'Addr', 'Network Address']:
+            with self.subTest(host=host):
+                result, args, password = self.backup(f'{host}=127.0.0.1;DATABASE=synthetic;UID=reader;PWD=dummy-secret')
+                self.assertEqual(0, result.returncode, result.stderr + result.stdout)
+                self.assertEqual([b'--host=127.0.0.1', b'--port=3306', b'--user=reader',
+                                  b'--single-transaction', b'--routines', b'--events', b'--quick', b'synthetic'], args)
+                self.assertEqual('dummy-secret', password)
+                self.assertNotIn('dummy-secret', result.stdout + result.stderr)
+
+    def test_quoted_separators_and_shell_characters_remain_data(self):
+        result, args, password = self.backup('Server=127.0.0.1;Port=3307;Database=synthetic;User ID=reader;Password="semi;eq=quote""$(false)"')
+        self.assertEqual(0, result.returncode, result.stderr + result.stdout)
+        self.assertIn(b'--port=3307', args)
+        self.assertEqual('semi;eq=quote"$(false)', password)
+
+    def test_empty_password_is_preserved(self):
+        result, args, password = self.backup('Server=127.0.0.1;Database=synthetic;User ID=reader;Password=')
+        self.assertEqual(0, result.returncode, result.stderr + result.stdout)
+        self.assertIsNotNone(args)
+        self.assertEqual('', password)
+
+    def test_single_quoted_unicode_password_and_database_alias_are_preserved(self):
+        result, args, password = self.backup("Host=127.0.0.1;Initial Catalog=synthetic;Username=reader;Password='søknad你好;it''s';SslMode=VerifyFull")
+        self.assertEqual(0, result.returncode, result.stderr + result.stdout)
+        self.assertEqual(b'synthetic', args[-1])
+        self.assertEqual("søknad你好;it's", password)
+
+    def test_invalid_or_ambiguous_input_never_invokes_dump_or_exposes_input(self):
+        for connection in [
+            'Host=127.0.0.1;Database=synthetic;Uid=reader;Port=0;Pwd=dummy-secret',
+            'Host=127.0.0.1;Database=synthetic;Uid=reader;Port=65536;Pwd=dummy-secret',
+            'Host=127.0.0.1;Server=other;Database=synthetic;Uid=reader;Pwd=dummy-secret',
+            'Host=127.0.0.1;Database=synthetic;Uid=reader;Pwd="dummy-secret',
+            'Host=127.0.0.1;Database=synthetic;Uid=reader;Pwd=dummy-secret\nline',
+            'Host=127.0.0.1;Uid=reader;Pwd=dummy-secret',
+        ]:
+            with self.subTest(connection=connection.replace('dummy-secret', 'REDACTED')):
+                result, args, _ = self.backup(connection)
+                self.assertNotEqual(0, result.returncode)
+                self.assertIsNone(args)
+                self.assertNotIn('dummy-secret', result.stdout + result.stderr)
+
+
+class BackupUploadTests(unittest.TestCase):
+    def upload(self, fail_operation=''):
+        source = (ROOT / 'deploy/deploy.sh').read_text(encoding='utf-8')
+        start = source.index('upload_current_backup_set() {')
+        function = source[start:source.index('\n}\n', start) + 3]
+        gate_start = source.index('if ! apply_backup_retention || ! upload_current_backup_set; then')
+        gate = source[gate_start:source.index('\nfi\n', gate_start) + 4]
+        with tempfile.TemporaryDirectory() as temporary:
+            folder = Path(temporary)
+            for filename in ['jobtracker-db-synthetic.sql.gz', 'jobtracker-files-synthetic.tar.gz',
+                             'jobtracker-db-older.sql.gz']:
+                (folder / filename).touch()
+            (folder / 'monthly' / '2099-01').mkdir(parents=True)
+            script = '\n'.join([
+                'set -euo pipefail',
+                'BACKUP_DIR=' + shlex.quote(folder.as_posix()),
+                'BACKUP_STAMP=synthetic',
+                'BACKUP_RCLONE_REMOTE=fixture:encrypted/',
+                'FAIL_OPERATION=' + shlex.quote(fail_operation),
+                'date() { echo 2099-01; }',
+                'apply_backup_retention() { return 0; }',
+                'rclone() {',
+                '  printf "UPLOAD %s %s %s\\n" "$1" "$3" "$4";',
+                '  if [ "$1" = "$FAIL_OPERATION" ]; then return 23; fi;',
+                '}',
+                function,
+                gate,
+                'echo "DEPLOY WOULD CONTINUE"',
+            ])
+            result = subprocess.run([BASH, '--noprofile', '--norc'], input=script,
+                                    text=True, encoding='utf-8', capture_output=True,
+                                    cwd=ROOT, timeout=10,
+                                    env={key: value for key, value in os.environ.items()
+                                         if key.upper() in {'PATH', 'SYSTEMROOT', 'WINDIR', 'TEMP', 'TMP'}})
+        calls = [line for line in result.stdout.splitlines() if line.startswith('UPLOAD ')]
+        return result, calls
+
+    def test_uploads_only_current_daily_files_then_monthly_snapshot(self):
+        result, calls = self.upload()
+        self.assertEqual(0, result.returncode, result.stdout + result.stderr)
+        self.assertEqual([
+            'UPLOAD copyto fixture:encrypted/daily/jobtracker-db-synthetic.sql.gz --immutable',
+            'UPLOAD copyto fixture:encrypted/daily/jobtracker-files-synthetic.tar.gz --immutable',
+            'UPLOAD copy fixture:encrypted/monthly/2099-01 --immutable',
+        ], calls)
+        self.assertIn('DEPLOY WOULD CONTINUE', result.stdout)
+        self.assertIn('Current recovery set copied to configured off-host remote.', result.stdout)
+
+    def test_daily_copy_failure_stops_before_remaining_files_and_monthly_copy(self):
+        result, calls = self.upload('copyto')
+        self.assertEqual(1, result.returncode, result.stdout + result.stderr)
+        self.assertEqual([
+            'UPLOAD copyto fixture:encrypted/daily/jobtracker-db-synthetic.sql.gz --immutable',
+        ], calls)
+        self.assertIn('Daily off-host backup upload failed', result.stdout)
+        self.assertNotIn('DEPLOY WOULD CONTINUE', result.stdout)
+        self.assertNotIn('Current recovery set copied', result.stdout)
+
+    def test_monthly_copy_failure_blocks_deploy(self):
+        result, calls = self.upload('copy')
+        self.assertEqual(1, result.returncode, result.stdout + result.stderr)
+        self.assertEqual(3, len(calls))
+        self.assertEqual('UPLOAD copy fixture:encrypted/monthly/2099-01 --immutable', calls[-1])
+        self.assertIn('Monthly off-host backup upload failed', result.stdout)
+        self.assertNotIn('DEPLOY WOULD CONTINUE', result.stdout)
+        self.assertNotIn('Current recovery set copied', result.stdout)
+
+
+class BackupRetentionTests(unittest.TestCase):
+    def retain(self, fail_operation='', *, previous_snapshot=False):
+        source = (ROOT / 'deploy/deploy.sh').read_text(encoding='utf-8')
+        start = source.index('apply_backup_retention() {')
+        function = source[start:source.index('\n}\n', start) + 3]
+        gate_start = source.index('if ! apply_backup_retention || ! upload_current_backup_set; then')
+        gate = source[gate_start:source.index('\nfi\n', gate_start) + 4]
+        fixtures = {'jobtracker-db-synthetic.sql.gz': b'database fixture',
+                    'jobtracker-files-synthetic.tar.gz': b'files fixture'}
+        with tempfile.TemporaryDirectory() as temporary:
+            folder = Path(temporary)
+            for name, contents in fixtures.items():
+                (folder / name).write_bytes(contents)
+            monthly = folder / 'monthly' / '2099-01'
+            if previous_snapshot:
+                monthly.mkdir(parents=True)
+                (monthly / 'jobtracker-db-earlier.sql.gz').write_bytes(b'earlier fixture')
+                (monthly / '.complete').write_bytes(b'earlier\n')
+            script = '\n'.join([
+                'set -euo pipefail',
+                'BACKUP_DIR=' + shlex.quote(folder.as_posix()),
+                'BACKUP_STAMP=synthetic',
+                'FAIL_OPERATION=' + shlex.quote(fail_operation),
+                'date() { echo 2099-01; }',
+                'upload_current_backup_set() { echo "UPLOAD WOULD START"; }',
+                'find() { if [[ "$*" == *-delete* ]]; then echo "PRUNING WOULD START"; fi; }',
+                'rm() { echo "CLEANUP REQUEST $1 $2"; }',
+                'cp() {',
+                '  if [ "$FAIL_OPERATION" = cp ]; then',
+                '    printf partial > "${@: -1}/partial"; return 23;',
+                '  fi;',
+                '  command cp "$@";',
+                '}',
+                'mv() { if [ "$FAIL_OPERATION" = mv ]; then return 24; fi; command mv "$@"; }',
+                function,
+                gate,
+                'echo "DEPLOY WOULD CONTINUE"',
+            ])
+            result = subprocess.run([BASH, '--noprofile', '--norc'], input=script,
+                                    text=True, encoding='utf-8', capture_output=True,
+                                    cwd=ROOT, timeout=10,
+                                    env={key: value for key, value in os.environ.items()
+                                         if key.upper() in {'PATH', 'SYSTEMROOT', 'WINDIR', 'TEMP', 'TMP'}})
+            artifacts = {file.name: file.read_bytes() for file in monthly.iterdir()} if monthly.exists() else {}
+        return result, artifacts
+
+    def test_completed_monthly_snapshot_contains_both_artifacts_and_stamp(self):
+        result, artifacts = self.retain()
+        self.assertEqual(0, result.returncode, result.stdout + result.stderr)
+        self.assertEqual({'jobtracker-db-synthetic.sql.gz': b'database fixture',
+                          'jobtracker-files-synthetic.tar.gz': b'files fixture',
+                          '.complete': b'synthetic\n'}, artifacts)
+        self.assertIn('PRUNING WOULD START', result.stdout)
+        self.assertIn('UPLOAD WOULD START', result.stdout)
+        self.assertIn('DEPLOY WOULD CONTINUE', result.stdout)
+
+    def test_partial_monthly_copy_cannot_mark_snapshot_complete_or_proceed(self):
+        result, artifacts = self.retain('cp')
+        self.assertEqual(1, result.returncode, result.stdout + result.stderr)
+        self.assertEqual({'partial': b'partial'}, artifacts)
+        self.assertIn('Could not retain monthly recovery files', result.stdout)
+        self.assertNotIn('PRUNING WOULD START', result.stdout)
+        self.assertNotIn('UPLOAD WOULD START', result.stdout)
+        self.assertNotIn('DEPLOY WOULD CONTINUE', result.stdout)
+
+    def test_marker_publish_failure_does_not_expose_complete_marker_or_proceed(self):
+        result, artifacts = self.retain('mv')
+        self.assertEqual(1, result.returncode, result.stdout + result.stderr)
+        self.assertNotIn('.complete', artifacts)
+        self.assertIn('CLEANUP REQUEST -f --', result.stdout)
+        self.assertIn('Could not publish monthly recovery marker', result.stdout)
+        self.assertNotIn('PRUNING WOULD START', result.stdout)
+        self.assertNotIn('UPLOAD WOULD START', result.stdout)
+        self.assertNotIn('DEPLOY WOULD CONTINUE', result.stdout)
+
+    def test_preserves_the_existing_completed_monthly_snapshot(self):
+        result, artifacts = self.retain('cp', previous_snapshot=True)
+        self.assertEqual(0, result.returncode, result.stdout + result.stderr)
+        self.assertEqual({'jobtracker-db-earlier.sql.gz': b'earlier fixture', '.complete': b'earlier\n'}, artifacts)
+        self.assertIn('DEPLOY WOULD CONTINUE', result.stdout)
+
+
+if __name__ == '__main__':
+    unittest.main()
@@ -0,0 +1,105 @@
+"""Exercise the deployment exit handler with synthetic image references and no Docker."""
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import unittest
+
+ROOT = Path(__file__).resolve().parents[1]
+BASH = 'C:/Program Files/Git/bin/bash.exe' if os.name == 'nt' else shutil.which('bash')
+
+
+class DeployRollbackTests(unittest.TestCase):
+    def exit_deploy(self, status, *, replaced=True, backend=True, frontend=True, compose_status=0, failed_probe='', diagnostics_status=0):
+        source = (ROOT / 'deploy/deploy.sh').read_text(encoding='utf-8')
+        start = source.index('finish_deploy() {')
+        handler = source[start:source.index('\ntrap finish_deploy EXIT\n', start) + len('\ntrap finish_deploy EXIT\n')]
+        script = '\n'.join([
+            'set -euo pipefail',
+            'exec 3>&1',
+            'APP_PUBLIC_BASE_URL=https://synthetic.invalid/',
+            f'replacement_started={str(replaced).lower()}',
+            'previous_backend_ref=' + ('synthetic/backend:previous' if backend else ''),
+            'previous_frontend_ref=' + ('synthetic/frontend:previous' if frontend else ''),
+            'auth_config_body_file=synthetic-body',
+            'auth_config_headers_file=synthetic-headers',
+            'rm() { printf "CALL rm %s\\n" "$*"; }',
+            'docker() { echo "CALL unexpected docker"; return 99; }',
+            'curl() {',
+            '  printf "CALL curl %s\\n" "$*" >&3;',
+            f'  if [ "${{@: -1}}" = "https://synthetic.invalid{failed_probe}" ]; then return 22; fi;',
+            '  return 0;',
+            '}',
+            'compose() {',
+            '  printf "CALL compose %s [%s] [%s]\\n" "$*" "${BACKEND_IMAGE:-}" "${FRONTEND_IMAGE:-}";',
+            f'  if [ "$1" = up ]; then return {compose_status}; fi;',
+            f'  return {diagnostics_status};',
+            '}',
+            handler,
+            f'exit {status}',
+        ])
+        # Exclude inherited deployment settings from the fixture.
+        environment = {key: value for key, value in os.environ.items()
+                       if key.upper() in {'PATH', 'SYSTEMROOT', 'WINDIR', 'TEMP', 'TMP'}}
+        result = subprocess.run([BASH, '--noprofile', '--norc'], input=script,
+                                text=True, encoding='utf-8', capture_output=True,
+                                cwd=ROOT, env=environment, timeout=10)
+        self.assertEqual(status, result.returncode, result.stdout + result.stderr)
+        self.assertEqual('', result.stderr)
+        calls = [line for line in result.stdout.splitlines() if line.startswith('CALL ')]
+        self.assertEqual(['CALL rm -f synthetic-body', 'CALL rm -f synthetic-headers'], calls[:2])
+        return result.stdout, calls[2:]
+
+    def test_failed_replacement_recreates_only_prior_core_images(self):
+        stdout, calls = self.exit_deploy(41)
+        self.assertEqual([
+            'CALL compose up -d --force-recreate --remove-orphans backend frontend '
+            '[synthetic/backend:previous] [synthetic/frontend:previous]',
+            *self.probes(),
+        ], calls)
+        self.assertIn('public core checks passed', stdout)
+        self.assertIn('Deployment remains failed', stdout)
+
+    @staticmethod
+    def probes():
+        flags = '-fsS --connect-timeout 5 --max-time 5 --retry 5 --retry-delay 2 --retry-connrefused --retry-max-time 30'
+        return [f'CALL curl {flags} https://synthetic.invalid{path}' for path in ['/', '/health', '/ready']]
+
+    def test_each_failed_health_probe_preserves_failure_and_requires_manual_recovery(self):
+        for index, path in enumerate(['/', '/health', '/ready']):
+            with self.subTest(path=path):
+                stdout, calls = self.exit_deploy(41, failed_probe=path, diagnostics_status=17)
+                self.assertEqual(self.probes()[:index + 1], calls[1:-2])
+                self.assertEqual(['CALL compose ps [] []', 'CALL compose logs --tail=200 backend frontend [] []'], calls[-2:])
+                self.assertIn('rollback health verification failed', stdout)
+                self.assertIn('Manual recovery is required', stdout)
+                self.assertNotIn('public core checks passed', stdout)
+
+    def test_rollback_failure_retains_original_status_and_collects_diagnostics(self):
+        stdout, calls = self.exit_deploy(41, compose_status=19, diagnostics_status=17)
+        self.assertEqual([
+            'CALL compose up -d --force-recreate --remove-orphans backend frontend '
+            '[synthetic/backend:previous] [synthetic/frontend:previous]',
+            'CALL compose ps [] []',
+            'CALL compose logs --tail=200 backend frontend [] []',
+        ], calls)
+        self.assertIn('Manual recovery is required', stdout)
+
+    def test_incomplete_previous_release_does_not_recreate_any_service(self):
+        for backend, frontend in [(False, True), (True, False), (False, False)]:
+            with self.subTest(backend=backend, frontend=frontend):
+                stdout, calls = self.exit_deploy(41, backend=backend, frontend=frontend)
+                self.assertEqual([], calls)
+                self.assertIn('no complete previous core release', stdout)
+
+    def test_failure_before_replacement_does_not_recreate_any_service(self):
+        _, calls = self.exit_deploy(41, replaced=False)
+        self.assertEqual([], calls)
+
+    def test_success_does_not_recreate_any_service(self):
+        _, calls = self.exit_deploy(0)
+        self.assertEqual([], calls)
+
+
+if __name__ == '__main__':
+    unittest.main()
Author	SHA1	Message	Date
cesnimda	3cca441874	docs(release): scope legacy rollback guidance to its migration baseline	2026-11-11 07:32:04 +01:00
cesnimda	55431b5ad0	docs(release): record offline rollback health verification	2026-11-11 07:20:46 +01:00
cesnimda	a39ac84729	fix(release): verify core health after automatic image rollback	2026-11-11 07:19:45 +01:00
cesnimda	24a629972e	fix(release): publish monthly backup marker after successful copy	2026-11-10 13:26:37 +01:00
cesnimda	cedf33a358	fix(release): stop deployment when off-host backup transfer fails	2026-11-10 13:24:30 +01:00
cesnimda	e73681008a	test(release): exercise rollback handler offline	2026-11-10 13:21:56 +01:00
cesnimda	47c6834cee	docs(release): map endpoint and environment qualification	2026-11-10 13:11:14 +01:00
cesnimda	f19e7ebf4a	fix(deploy): parse backup connection aliases and quoted values safely	2026-11-02 12:11:48 +01:00
				`@@ -0,0 +1 @@`
				`Git Bash bash -n deploy/deploy.sh exited 0. Full script was not executed.`