diff --git a/deploy/deploy.sh b/deploy/deploy.sh index dbe1921c..32ec12ba 100644 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -559,14 +559,29 @@ apply_backup_retention() { echo "Monthly recovery set retained: ${monthly_dir}" fi - find "$BACKUP_DIR" -maxdepth 1 -type f -name 'jobtracker-*' -mtime "+$daily_days" -delete + if ! find "$BACKUP_DIR" -maxdepth 1 -type f -name 'jobtracker-*' -mtime "+$daily_days" -delete; then + echo "Daily backup retention failed. Aborting deploy." + return 1 + fi - local monthly_dirs=() - mapfile -t monthly_dirs < <(find "$monthly_root" -mindepth 1 -maxdepth 1 -type d -name '20??-??' | sort) + local monthly_dirs=() monthly_listing + # Check the pipeline directly; mapfile cannot report process-substitution failures. + if ! monthly_listing="$(find "$monthly_root" -mindepth 1 -maxdepth 1 -type d -name '20??-??' | sort)"; then + echo "Could not enumerate monthly backups. Aborting deploy." + return 1 + fi + if [ -n "$monthly_listing" ]; then + mapfile -t monthly_dirs <<< "$monthly_listing" + fi while [ "${#monthly_dirs[@]}" -gt "$monthly_count" ]; do local oldest="${monthly_dirs[0]}" case "$oldest" in - "$monthly_root"/20??-??) rm -rf -- "$oldest" ;; + "$monthly_root"/20??-??) + if ! rm -rf -- "$oldest"; then + echo "Monthly backup retention failed. Aborting deploy." + return 1 + fi + ;; *) echo "Refusing to prune unexpected monthly backup path: $oldest"; return 1 ;; esac monthly_dirs=("${monthly_dirs[@]:1}") diff --git a/docs/release-readiness/11_BACKUP_RESTORE_ROLLBACK.md b/docs/release-readiness/11_BACKUP_RESTORE_ROLLBACK.md index 58ca1595..35a75c77 100644 --- a/docs/release-readiness/11_BACKUP_RESTORE_ROLLBACK.md +++ b/docs/release-readiness/11_BACKUP_RESTORE_ROLLBACK.md @@ -79,3 +79,21 @@ The fix explicitly checks preparation and copy. It writes the stamp to a tempora The red test run executed six methods and reported four assertion failures because the old handler performed no probes. The green run passed six methods, including failed-probe subcases for all three endpoints, Compose failure, incomplete references, pre-replacement failure and successful deployment. Probe failures and Compose failure also use failing diagnostic stubs to verify original-status preservation. Twelve backup tests and shell syntax validation passed separately. Earlier five-test receipts remain historical and overlap this run. Only the extracted exit handler/trap ran with shell functions substituting Compose, Docker, curl and cleanup. The full deployment script was never sourced or executed, and no network/containers/models were used. Raw red/green/backup/syntax receipts are in `evidence/rollback-health/`. Actual startup timing, proxy routing, exact image identity, post-migration writes and database/files/keys/tombstone recovery still need the authorised isolated rehearsal. Passing public probes alone does not establish recovery correctness. + +## RR-13 retention failure propagation + +11 November 2026. Based on `3cca441874abfab6ad581c5a07913a6d66622423` on `codex/release-rollback-health`. Bounded offline remediation under prompt 11, sections 13 and 16. Engineering priority P2: retention failures could falsely report success and permit upload/deployment, leaving retention incomplete; no actual data loss or production impact was observed. No security severity or release-risk acceptance is implied. + +The real conditional deployment caller suppresses Bash errexit within `apply_backup_retention`. Previously a failed daily `find -delete` or monthly `rm` was masked by the final success message. Monthly `find | sort` failures were also hidden by process substitution feeding `mapfile`. Seven retention test methods reproduced four failing assertions: daily prune, monthly enumeration, sorting and monthly prune all returned success and reached the mocked upload/deployment boundary. + +Daily pruning and each monthly removal now explicitly return failure. Monthly enumeration is captured in a checked command substitution under the script's existing `pipefail`, then split into the same ordered array; an empty listing produces an empty array. Failure prevents later monthly removals, upload and deployment, and suppresses the final retention-success message. Existing daily age/count defaults, monthly path guard, copy/atomic-marker behavior, archive/dump/upload functions and backend error handling remain unchanged. Pruning can already have removed some eligible old files before a command fails; this change does not undo prior removals or prove recoverability. + +Fresh evidence, Windows / Python 3.12.10 / Git Bash: + +- `python scripts/test-deploy-backup.py BackupRetentionTests -v`: red run **7 methods, 4 assertion failures** (monthly enumeration has two failing subcases). Four existing methods passed. An additional empty-list control was added afterward. +- `python scripts/test-deploy-backup.py -v`: final **16/16 methods PASSED**, including 8 retention, 5 connection-parser and 3 upload methods. Healthy retention requests removal of the two oldest synthetic months in order, preserves current copied bytes/stamp and an existing completed snapshot, and reaches upload/deployment. Daily failure prevents enumeration; either pipeline failure prevents monthly pruning; first monthly removal failure prevents further removals. Empty-list control makes no removal request. The red and earlier green totals overlap; do not add them. +- `python scripts/test-deploy-rollback.py -v`: **6/6 PASSED** independently; original failure statuses, rollback diagnostics and health-check gates retained. +- Git Bash `bash -n deploy/deploy.sh`: **PASSED**, syntax only. Raw receipts and SHA-256 source identities: [retention-failures evidence](evidence/retention-failures/source-identity.json), alongside `red.txt`, `green-backup.txt`, `rollback.txt`, and `syntax.txt`. +- `graphify update .`: AST-only update completed, 18,539 nodes / 45,993 edges. Optional SQL extractor missing and unsupported/no-symbol files reported; no dependency installed or semantic/model extraction run. Generated graph remains untracked local navigation material. + +Tests execute only extracted functions/caller with synthetic temporary files. Every pruning `find`, `rm` and upload operation is a shell stub; sorting uses only mocked listing text. The full deployment script was never executed or sourced. No real backup, deletion, network transfer, containers, database, migration, providers, model, infrastructure repair or production operation was performed. Product/browser/full MariaDB/hosted/security suites were not rerun for this shell-only package. Actual Linux-host pruning, encrypted retrieval/decryption, consistent restore, post-write rollback and agreed RTO/RPO remain unqualified. RR-09/RR-12/RR-13 recovery gates and production NO-GO remain open. Reverting this bounded commit restores the prior failure-masking behavior; no schema or user-data reversal is needed. diff --git a/docs/release-readiness/13_REMEDIATION_REGISTER.md b/docs/release-readiness/13_REMEDIATION_REGISTER.md index 7cffacdb..d9641f3e 100644 --- a/docs/release-readiness/13_REMEDIATION_REGISTER.md +++ b/docs/release-readiness/13_REMEDIATION_REGISTER.md @@ -16,7 +16,7 @@ Discovery only, 2026-11-02. These are qualification gaps unless a concrete defec | RR-10 | AI quality/activation | Deterministic Docling fixtures are distinct from real-model semantic accuracy. Optional unqualified adapter remains disabled; no AI invocation authorised here | | RR-11 | Backup host parsing rejected supported `Host=` alias and uppercase spelling | FIXED AND OFFLINE-TESTED. See [bounded remediation](RR-11_IMPLEMENTATION.md). Real dump/restore qualification remains open | | RR-12 | Previous handler omitted post-rollback health checks; schema/data/AI recovery remains separate | Health probes FIXED in `a39ac847` and six offline handler methods passed, including failed endpoints and diagnostics. See [health follow-up](11_BACKUP_RESTORE_ROLLBACK.md#rr-12-post-rollback-core-health-checks). Actual image identity, startup timing and recovery after migration/writes still require isolated rehearsal | -| RR-13 | Off-host encryption and recovery-set consistency need independent proof. Failed transfer and partial monthly copy previously let deployment continue | Transfer and monthly marker failure gates FIXED AND OFFLINE-TESTED. Actual encryption, retrieval/decryption and database/files/key/tombstone consistency require approved isolation | +| RR-13 | Off-host encryption and recovery-set consistency need independent proof. Failed transfer, partial monthly copy and retention errors previously let deployment continue | Transfer/marker gates and explicit daily prune, monthly enumeration/sort and removal failure propagation FIXED AND OFFLINE-TESTED. Final backup suite 16/16 methods; four retention failure assertions reproduced first. See [retention follow-up](11_BACKUP_RESTORE_ROLLBACK.md#rr-13-retention-failure-propagation). Actual encryption, retrieval/decryption and database/files/key/tombstone consistency require approved isolation | | RR-14 | July first-deploy runbook inferred rollback safety from its no-op migration baseline, while current migration Up methods add schema | Historical scope and current qualification links clarified. `AddJobRevision` adds Version and `AddUserOperationEvents` creates a table/indexes. Exact old/new image compatibility after migration/writes remains an isolated rehearsal gate | Continue evidence work against the [source endpoint and environment census](03_ENDPOINT_ENVIRONMENT_CENSUS.md). Actual qualification requires an accepted candidate and isolated environment. Every fix needs minimal scope, targeted verification and the applicable stage approval. diff --git a/docs/release-readiness/evidence/retention-failures/green-backup.txt b/docs/release-readiness/evidence/retention-failures/green-backup.txt new file mode 100644 index 00000000..5df7e632 --- /dev/null +++ b/docs/release-readiness/evidence/retention-failures/green-backup.txt @@ -0,0 +1,23 @@ +test_empty_password_is_preserved (__main__.BackupConnectionTests.test_empty_password_is_preserved) ... ok +test_invalid_or_ambiguous_input_never_invokes_dump_or_exposes_input (__main__.BackupConnectionTests.test_invalid_or_ambiguous_input_never_invokes_dump_or_exposes_input) ... ok +test_quoted_separators_and_shell_characters_remain_data (__main__.BackupConnectionTests.test_quoted_separators_and_shell_characters_remain_data) ... ok +test_single_quoted_unicode_password_and_database_alias_are_preserved (__main__.BackupConnectionTests.test_single_quoted_unicode_password_and_database_alias_are_preserved) ... ok +test_supported_host_alias_and_case_reach_dump_unchanged (__main__.BackupConnectionTests.test_supported_host_alias_and_case_reach_dump_unchanged) ... ok +test_completed_monthly_snapshot_contains_both_artifacts_and_stamp (__main__.BackupRetentionTests.test_completed_monthly_snapshot_contains_both_artifacts_and_stamp) ... ok +test_daily_prune_failure_blocks_monthly_pruning_upload_and_deploy (__main__.BackupRetentionTests.test_daily_prune_failure_blocks_monthly_pruning_upload_and_deploy) ... ok +test_empty_monthly_enumeration_does_not_request_removal (__main__.BackupRetentionTests.test_empty_monthly_enumeration_does_not_request_removal) ... ok +test_marker_publish_failure_does_not_expose_complete_marker_or_proceed (__main__.BackupRetentionTests.test_marker_publish_failure_does_not_expose_complete_marker_or_proceed) ... ok +test_monthly_enumeration_failure_blocks_pruning_upload_and_deploy (__main__.BackupRetentionTests.test_monthly_enumeration_failure_blocks_pruning_upload_and_deploy) ... ok +test_monthly_prune_failure_stops_before_remaining_months_upload_and_deploy (__main__.BackupRetentionTests.test_monthly_prune_failure_stops_before_remaining_months_upload_and_deploy) ... ok +test_partial_monthly_copy_cannot_mark_snapshot_complete_or_proceed (__main__.BackupRetentionTests.test_partial_monthly_copy_cannot_mark_snapshot_complete_or_proceed) ... ok +test_preserves_the_existing_completed_monthly_snapshot (__main__.BackupRetentionTests.test_preserves_the_existing_completed_monthly_snapshot) ... ok +test_daily_copy_failure_stops_before_remaining_files_and_monthly_copy (__main__.BackupUploadTests.test_daily_copy_failure_stops_before_remaining_files_and_monthly_copy) ... ok +test_monthly_copy_failure_blocks_deploy (__main__.BackupUploadTests.test_monthly_copy_failure_blocks_deploy) ... ok +test_uploads_only_current_daily_files_then_monthly_snapshot (__main__.BackupUploadTests.test_uploads_only_current_daily_files_then_monthly_snapshot) ... ok + +---------------------------------------------------------------------- +Ran 16 tests in 6.842s + +OK + +PROCESS_EXIT=0 diff --git a/docs/release-readiness/evidence/retention-failures/red.txt b/docs/release-readiness/evidence/retention-failures/red.txt new file mode 100644 index 00000000..fceac514 --- /dev/null +++ b/docs/release-readiness/evidence/retention-failures/red.txt @@ -0,0 +1,84 @@ +test_completed_monthly_snapshot_contains_both_artifacts_and_stamp (__main__.BackupRetentionTests.test_completed_monthly_snapshot_contains_both_artifacts_and_stamp) ... ok +test_daily_prune_failure_blocks_monthly_pruning_upload_and_deploy (__main__.BackupRetentionTests.test_daily_prune_failure_blocks_monthly_pruning_upload_and_deploy) ... FAIL +test_marker_publish_failure_does_not_expose_complete_marker_or_proceed (__main__.BackupRetentionTests.test_marker_publish_failure_does_not_expose_complete_marker_or_proceed) ... ok +test_monthly_enumeration_failure_blocks_pruning_upload_and_deploy (__main__.BackupRetentionTests.test_monthly_enumeration_failure_blocks_pruning_upload_and_deploy) ... + test_monthly_enumeration_failure_blocks_pruning_upload_and_deploy (__main__.BackupRetentionTests.test_monthly_enumeration_failure_blocks_pruning_upload_and_deploy) (operation='monthly-find') ... FAIL + test_monthly_enumeration_failure_blocks_pruning_upload_and_deploy (__main__.BackupRetentionTests.test_monthly_enumeration_failure_blocks_pruning_upload_and_deploy) (operation='sort') ... FAIL +test_monthly_prune_failure_stops_before_remaining_months_upload_and_deploy (__main__.BackupRetentionTests.test_monthly_prune_failure_stops_before_remaining_months_upload_and_deploy) ... FAIL +test_partial_monthly_copy_cannot_mark_snapshot_complete_or_proceed (__main__.BackupRetentionTests.test_partial_monthly_copy_cannot_mark_snapshot_complete_or_proceed) ... ok +test_preserves_the_existing_completed_monthly_snapshot (__main__.BackupRetentionTests.test_preserves_the_existing_completed_monthly_snapshot) ... ok + +====================================================================== +FAIL: test_daily_prune_failure_blocks_monthly_pruning_upload_and_deploy (__main__.BackupRetentionTests.test_daily_prune_failure_blocks_monthly_pruning_upload_and_deploy) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "C:\Users\Connor\.codex\worktrees\release-readiness-discovery\Job tracker\scripts\test-deploy-backup.py", line 258, in test_daily_prune_failure_blocks_monthly_pruning_upload_and_deploy + self.assertEqual(1, result.returncode, result.stdout + result.stderr) +AssertionError: 1 != 0 : Monthly recovery set retained: C:/Users/Connor/AppData/Local/Temp/tmp_6t6unbl/monthly/2099-01 +PRUNING WOULD START +CLEANUP REQUEST -rf -- +MONTHLY PRUNE 2098-11 +CLEANUP REQUEST -rf -- +MONTHLY PRUNE 2098-12 +Backup retention applied: 30 days daily, 1 monthly sets. +UPLOAD WOULD START +DEPLOY WOULD CONTINUE +MONTHLY ENUMERATION + + +====================================================================== +FAIL: test_monthly_enumeration_failure_blocks_pruning_upload_and_deploy (__main__.BackupRetentionTests.test_monthly_enumeration_failure_blocks_pruning_upload_and_deploy) (operation='monthly-find') +---------------------------------------------------------------------- +Traceback (most recent call last): + File "C:\Users\Connor\.codex\worktrees\release-readiness-discovery\Job tracker\scripts\test-deploy-backup.py", line 270, in test_monthly_enumeration_failure_blocks_pruning_upload_and_deploy + self.assertEqual(1, result.returncode, result.stdout + result.stderr) +AssertionError: 1 != 0 : Monthly recovery set retained: C:/Users/Connor/AppData/Local/Temp/tmpk3z7klep/monthly/2099-01 +PRUNING WOULD START +CLEANUP REQUEST -rf -- +MONTHLY PRUNE 2098-11 +CLEANUP REQUEST -rf -- +MONTHLY PRUNE 2098-12 +Backup retention applied: 30 days daily, 1 monthly sets. +UPLOAD WOULD START +DEPLOY WOULD CONTINUE +MONTHLY ENUMERATION + + +====================================================================== +FAIL: test_monthly_enumeration_failure_blocks_pruning_upload_and_deploy (__main__.BackupRetentionTests.test_monthly_enumeration_failure_blocks_pruning_upload_and_deploy) (operation='sort') +---------------------------------------------------------------------- +Traceback (most recent call last): + File "C:\Users\Connor\.codex\worktrees\release-readiness-discovery\Job tracker\scripts\test-deploy-backup.py", line 270, in test_monthly_enumeration_failure_blocks_pruning_upload_and_deploy + self.assertEqual(1, result.returncode, result.stdout + result.stderr) +AssertionError: 1 != 0 : Monthly recovery set retained: C:/Users/Connor/AppData/Local/Temp/tmp6bu45jcv/monthly/2099-01 +PRUNING WOULD START +Backup retention applied: 30 days daily, 1 monthly sets. +UPLOAD WOULD START +DEPLOY WOULD CONTINUE +MONTHLY ENUMERATION + + +====================================================================== +FAIL: test_monthly_prune_failure_stops_before_remaining_months_upload_and_deploy (__main__.BackupRetentionTests.test_monthly_prune_failure_stops_before_remaining_months_upload_and_deploy) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "C:\Users\Connor\.codex\worktrees\release-readiness-discovery\Job tracker\scripts\test-deploy-backup.py", line 279, in test_monthly_prune_failure_stops_before_remaining_months_upload_and_deploy + self.assertEqual(1, result.returncode, result.stdout + result.stderr) +AssertionError: 1 != 0 : Monthly recovery set retained: C:/Users/Connor/AppData/Local/Temp/tmpesmkwjd9/monthly/2099-01 +PRUNING WOULD START +CLEANUP REQUEST -rf -- +MONTHLY PRUNE 2098-11 +CLEANUP REQUEST -rf -- +MONTHLY PRUNE 2098-12 +Backup retention applied: 30 days daily, 1 monthly sets. +UPLOAD WOULD START +DEPLOY WOULD CONTINUE +MONTHLY ENUMERATION + + +---------------------------------------------------------------------- +Ran 7 tests in 1.787s + +FAILED (failures=4) + +PROCESS_EXIT=1 diff --git a/docs/release-readiness/evidence/retention-failures/rollback.txt b/docs/release-readiness/evidence/retention-failures/rollback.txt new file mode 100644 index 00000000..9a6f4959 --- /dev/null +++ b/docs/release-readiness/evidence/retention-failures/rollback.txt @@ -0,0 +1,13 @@ +test_each_failed_health_probe_preserves_failure_and_requires_manual_recovery (__main__.DeployRollbackTests.test_each_failed_health_probe_preserves_failure_and_requires_manual_recovery) ... ok +test_failed_replacement_recreates_only_prior_core_images (__main__.DeployRollbackTests.test_failed_replacement_recreates_only_prior_core_images) ... ok +test_failure_before_replacement_does_not_recreate_any_service (__main__.DeployRollbackTests.test_failure_before_replacement_does_not_recreate_any_service) ... ok +test_incomplete_previous_release_does_not_recreate_any_service (__main__.DeployRollbackTests.test_incomplete_previous_release_does_not_recreate_any_service) ... ok +test_rollback_failure_retains_original_status_and_collects_diagnostics (__main__.DeployRollbackTests.test_rollback_failure_retains_original_status_and_collects_diagnostics) ... ok +test_success_does_not_recreate_any_service (__main__.DeployRollbackTests.test_success_does_not_recreate_any_service) ... ok + +---------------------------------------------------------------------- +Ran 6 tests in 0.369s + +OK + +PROCESS_EXIT=0 diff --git a/docs/release-readiness/evidence/retention-failures/source-identity.json b/docs/release-readiness/evidence/retention-failures/source-identity.json new file mode 100644 index 00000000..69c5837f --- /dev/null +++ b/docs/release-readiness/evidence/retention-failures/source-identity.json @@ -0,0 +1,40 @@ +{ + "baseline": "3cca441874abfab6ad581c5a07913a6d66622423", + "python": "3.12.10 (tags/v3.12.10:0cc8128, Apr 8 2025, 12:21:36) [MSC v.1943 64 bit (AMD64)]", + "files": { + "deploy/deploy.sh": "f25723f68aa93f2e1ad6afd467facba5d0fd97004122f02db3f77f967237f334", + "scripts/test-deploy-backup.py": "a9d3a69214888c76407b9a47c9e5394144f4a5f01dba134717d0ea9ff8f40ef1", + "scripts/test-deploy-rollback.py": "4d3ec3074ca7582fda52960285909e5f4fbad1e497343b9dbe54a8d8acc90fc4" + }, + "checks": [ + { + "name": "green-backup", + "command": [ + "python", + "scripts/test-deploy-backup.py", + "-v" + ], + "exit": 0 + }, + { + "name": "rollback", + "command": [ + "python", + "scripts/test-deploy-rollback.py", + "-v" + ], + "exit": 0 + }, + { + "name": "syntax", + "command": [ + "C:/Program Files/Git/bin/bash.exe", + "-n", + "deploy/deploy.sh" + ], + "exit": 0 + } + ], + "red_deploy_sha256": "e5290bf766627517701c269c866398d6466843c05d1dcb832a866815c228291d", + "red_note": "Seven retention methods against baseline deploy source; empty-list control added afterward. Raw worktree SHA-256 identities preserve exact tested bytes, including line endings." +} diff --git a/docs/release-readiness/evidence/retention-failures/syntax.txt b/docs/release-readiness/evidence/retention-failures/syntax.txt new file mode 100644 index 00000000..79ed8c9f --- /dev/null +++ b/docs/release-readiness/evidence/retention-failures/syntax.txt @@ -0,0 +1,2 @@ + +PROCESS_EXIT=0 diff --git a/scripts/test-deploy-backup.py b/scripts/test-deploy-backup.py index ee548940..13835b66 100644 --- a/scripts/test-deploy-backup.py +++ b/scripts/test-deploy-backup.py @@ -175,11 +175,29 @@ class BackupRetentionTests(unittest.TestCase): 'set -euo pipefail', 'BACKUP_DIR=' + shlex.quote(folder.as_posix()), 'BACKUP_STAMP=synthetic', + 'BACKUP_DAILY_RETENTION_DAYS=30', + 'BACKUP_MONTHLY_RETENTION_COUNT=1', 'FAIL_OPERATION=' + shlex.quote(fail_operation), 'date() { echo 2099-01; }', 'upload_current_backup_set() { echo "UPLOAD WOULD START"; }', - 'find() { if [[ "$*" == *-delete* ]]; then echo "PRUNING WOULD START"; fi; }', - 'rm() { echo "CLEANUP REQUEST $1 $2"; }', + 'find() {', + ' if [[ "$*" == *-delete* ]]; then', + ' echo "PRUNING WOULD START";', + ' [ "$FAIL_OPERATION" != daily-find ]; return;', + ' fi;', + ' echo "MONTHLY ENUMERATION" >&2;', + ' if [ "$FAIL_OPERATION" = empty-enumeration ]; then return 0; fi;', + ' printf "%s/monthly/%s\\n" "$BACKUP_DIR" 2098-12 "$BACKUP_DIR" 2099-01 "$BACKUP_DIR" 2098-11;', + ' [ "$FAIL_OPERATION" != monthly-find ];', + '}', + 'sort() { if [ "$FAIL_OPERATION" = sort ]; then return 25; fi; command sort "$@"; }', + 'rm() {', + ' echo "CLEANUP REQUEST $1 $2";', + ' if [ "$1" = -rf ]; then', + ' echo "MONTHLY PRUNE ${3##*/}";', + ' [ "$FAIL_OPERATION" != monthly-rm ];', + ' fi;', + '}', 'cp() {', ' if [ "$FAIL_OPERATION" = cp ]; then', ' printf partial > "${@: -1}/partial"; return 23;', @@ -206,6 +224,8 @@ class BackupRetentionTests(unittest.TestCase): 'jobtracker-files-synthetic.tar.gz': b'files fixture', '.complete': b'synthetic\n'}, artifacts) self.assertIn('PRUNING WOULD START', result.stdout) + self.assertEqual(['MONTHLY PRUNE 2098-11', 'MONTHLY PRUNE 2098-12'], + [line for line in result.stdout.splitlines() if line.startswith('MONTHLY PRUNE ')]) self.assertIn('UPLOAD WOULD START', result.stdout) self.assertIn('DEPLOY WOULD CONTINUE', result.stdout) @@ -234,6 +254,45 @@ class BackupRetentionTests(unittest.TestCase): self.assertEqual({'jobtracker-db-earlier.sql.gz': b'earlier fixture', '.complete': b'earlier\n'}, artifacts) self.assertIn('DEPLOY WOULD CONTINUE', result.stdout) + def test_daily_prune_failure_blocks_monthly_pruning_upload_and_deploy(self): + result, _ = self.retain('daily-find') + self.assertEqual(1, result.returncode, result.stdout + result.stderr) + self.assertIn('Daily backup retention failed', result.stdout) + self.assertNotIn('MONTHLY ENUMERATION', result.stderr) + self.assertNotIn('MONTHLY PRUNE', result.stdout) + self.assertNotIn('UPLOAD WOULD START', result.stdout) + self.assertNotIn('DEPLOY WOULD CONTINUE', result.stdout) + self.assertNotIn('Backup retention applied', result.stdout) + + def test_monthly_enumeration_failure_blocks_pruning_upload_and_deploy(self): + for operation in ['monthly-find', 'sort']: + with self.subTest(operation=operation): + result, _ = self.retain(operation) + self.assertEqual(1, result.returncode, result.stdout + result.stderr) + self.assertIn('Could not enumerate monthly backups', result.stdout) + self.assertNotIn('MONTHLY PRUNE', result.stdout) + self.assertNotIn('UPLOAD WOULD START', result.stdout) + self.assertNotIn('DEPLOY WOULD CONTINUE', result.stdout) + self.assertNotIn('Backup retention applied', result.stdout) + + def test_monthly_prune_failure_stops_before_remaining_months_upload_and_deploy(self): + result, _ = self.retain('monthly-rm') + self.assertEqual(1, result.returncode, result.stdout + result.stderr) + self.assertIn('Monthly backup retention failed', result.stdout) + self.assertIn('MONTHLY PRUNE 2098-11', result.stdout) + self.assertNotIn('MONTHLY PRUNE 2098-12', result.stdout) + self.assertNotIn('UPLOAD WOULD START', result.stdout) + self.assertNotIn('DEPLOY WOULD CONTINUE', result.stdout) + self.assertNotIn('Backup retention applied', result.stdout) + + def test_empty_monthly_enumeration_does_not_request_removal(self): + result, _ = self.retain('empty-enumeration') + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertNotIn('MONTHLY PRUNE', result.stdout) + self.assertIn('Backup retention applied', result.stdout) + self.assertIn('UPLOAD WOULD START', result.stdout) + self.assertIn('DEPLOY WOULD CONTINUE', result.stdout) + if __name__ == '__main__': unittest.main()