fix(release): publish monthly backup marker after successful copy
This commit is contained in:
cesnimda
+20
-8
@@ -525,12 +525,14 @@ apply_backup_retention() {
|
||||
return 1
|
||||
fi
|
||||
|
||||
local month monthly_root monthly_dir
|
||||
local month monthly_root monthly_dir complete_tmp
|
||||
month="$(TZ="$timezone" date +%Y-%m)"
|
||||
monthly_root="$BACKUP_DIR/monthly"
|
||||
monthly_dir="$monthly_root/$month"
|
||||
mkdir -p "$monthly_root"
|
||||
chmod 700 "$monthly_root"
|
||||
if ! mkdir -p "$monthly_root" || ! chmod 700 "$monthly_root"; then
|
||||
echo "Could not prepare monthly backup directory. Aborting deploy."
|
||||
return 1
|
||||
fi
|
||||
|
||||
local current_files=("$BACKUP_DIR"/jobtracker-*"-$BACKUP_STAMP".*)
|
||||
if [ ! -e "${current_files[0]}" ]; then
|
||||
@@ -539,11 +541,21 @@ apply_backup_retention() {
|
||||
fi
|
||||
|
||||
if [ ! -f "$monthly_dir/.complete" ]; then
|
||||
mkdir -p "$monthly_dir"
|
||||
chmod 700 "$monthly_dir"
|
||||
cp --preserve=mode,timestamps -- "${current_files[@]}" "$monthly_dir/"
|
||||
printf '%s\n' "$BACKUP_STAMP" > "$monthly_dir/.complete"
|
||||
chmod 600 "$monthly_dir/.complete"
|
||||
if ! mkdir -p "$monthly_dir" || ! chmod 700 "$monthly_dir" ||
|
||||
! cp --preserve=mode,timestamps -- "${current_files[@]}" "$monthly_dir/"; then
|
||||
echo "Could not retain monthly recovery files. Aborting deploy."
|
||||
return 1
|
||||
fi
|
||||
if ! complete_tmp="$(mktemp "$monthly_dir/.complete.XXXXXX")"; then
|
||||
echo "Could not create monthly recovery marker. Aborting deploy."
|
||||
return 1
|
||||
fi
|
||||
if ! printf '%s\n' "$BACKUP_STAMP" > "$complete_tmp" ||
|
||||
! mv -T -- "$complete_tmp" "$monthly_dir/.complete"; then
|
||||
rm -f -- "$complete_tmp"
|
||||
echo "Could not publish monthly recovery marker. Aborting deploy."
|
||||
return 1
|
||||
fi
|
||||
echo "Monthly recovery set retained: ${monthly_dir}"
|
||||
fi
|
||||
|
||||
|
||||
@@ -61,3 +61,13 @@ The bounded fix explicitly returns failure on either copy error. A daily error s
|
||||
|
||||
- Red run: **2 failures and 1 passing control**. Green run: **8/8 backup tests** including the three upload cases. The independent rollback handler and hosted quarantine guard checks passed **5/5** and **4/4**. Bash syntax validation passed.
|
||||
- Encrypted off-host retrieval, decryption, completeness and consistent restore of database/files/keys/tombstones remain required. No actual transfer, restore, production configuration, keys or backup files were accessed.
|
||||
|
||||
## RR-13 monthly completion marker
|
||||
|
||||
The same conditional Bash behavior applies to `apply_backup_retention`. With a stubbed partial file copy returning 23, the old function wrote `.complete`, reported the month retained and continued through the mocked pruning, upload and deployment boundary. The successful control retained byte-identical synthetic files and the expected stamp.
|
||||
|
||||
The fix explicitly checks preparation and copy. It writes the stamp to a temporary file in the monthly directory and renames it to `.complete` only after the copy and stamp write succeed. A failed rename blocks later steps and attempts to remove the temporary marker. Existing completed snapshots are preserved. A failed copy can leave unmarked files, and previously written markers still require verification. A marker records the selected stamp and successful local copy; it does not attest to recovery-set integrity.
|
||||
|
||||
- Red run: **2 failures and 1 passing control** against the pre-fix function. The partial-copy case directly reproduced false completion; the marker-publication case required an atomic rename the old function did not perform.
|
||||
- Green run: **12/12 backup tests** including four monthly cases (successful copy and stamp, partial copy, failed marker rename, existing completed month). Rollback handler and hosted quarantine checks passed **5/5** and **4/4**. Bash syntax validation passed.
|
||||
- Only temporary synthetic files were created or copied. Pruning and upload were stubbed. Approved rehearsal still needs independent checks of artifacts for the marker stamp, encryption, retrieval and actual restore before any release decision.
|
||||
|
||||
@@ -16,6 +16,6 @@ Discovery only, 2026-11-02. These are qualification gaps unless a concrete defec
|
||||
| RR-10 | AI quality/activation | Deterministic Docling fixtures are distinct from real-model semantic accuracy. Optional unqualified adapter remains disabled; no AI invocation authorised here |
|
||||
| RR-11 | Backup host parsing rejected supported `Host=` alias and uppercase spelling | FIXED AND OFFLINE-TESTED. See [bounded remediation](RR-11_IMPLEMENTATION.md). Real dump/restore qualification remains open |
|
||||
| RR-12 | Rollback recreates core images without post-rollback health qualification; no schema/data/AI reversal | OFFLINE HANDLER CHECKS PASSED 5/5. See [failure-injection evidence](11_BACKUP_RESTORE_ROLLBACK.md#rr-12-offline-failure-injection). Actual image recovery and health/readiness after migration/writes still require isolated rehearsal |
|
||||
| RR-13 | Off-host encryption and recovery-set consistency need independent proof. A failed transfer previously let the deployment continue | Upload failure gate FIXED AND OFFLINE-TESTED. Actual encryption, retrieval/decryption and database/files/key/tombstone consistency require approved isolation |
|
||||
| RR-13 | Off-host encryption and recovery-set consistency need independent proof. Failed transfer and partial monthly copy previously let deployment continue | Transfer and monthly marker failure gates FIXED AND OFFLINE-TESTED. Actual encryption, retrieval/decryption and database/files/key/tombstone consistency require approved isolation |
|
||||
|
||||
Continue evidence work against the [source endpoint and environment census](03_ENDPOINT_ENVIRONMENT_CENSUS.md). Actual qualification requires an accepted candidate and isolated environment. Every fix needs minimal scope, targeted verification and the applicable stage approval.
|
||||
|
||||
@@ -4,6 +4,8 @@ RR-12 offline failure injection passed 5/5 against the unchanged deploy exit han
|
||||
|
||||
RR-13 transfer guard [reproduced and fixed offline](11_BACKUP_RESTORE_ROLLBACK.md#rr-13-off-host-upload-failure-handling). Daily and monthly copy errors stop deployment, and the success log makes no encryption claim. Actual remote encryption and restore remain open.
|
||||
|
||||
RR-13 [monthly completion guard](11_BACKUP_RESTORE_ROLLBACK.md#rr-13-monthly-completion-marker) now blocks the gate after a partial copy or marker publication failure. The successful path preserves artifact bytes and the stamp in synthetic fixtures. Existing markers and real restoration remain unverified.
|
||||
|
||||
10 November: [endpoint and environment census](03_ENDPOINT_ENVIRONMENT_CENSUS.md) maps 41 source route families, anonymous exceptions and failure cases to qualification owners on the backup-parser branch. No runtime qualification. Carry the map into Stage 2 only after the release candidate and environment are accepted.
|
||||
|
||||
Branch `planning/release-readiness-discovery`, baseline `5cb903b989742b7a96cb22ddf4c0e24993b18980`; documentation only. Stage 1 IN_PROGRESS, not complete. Full authoritative release prompt read. No final GO/NO-GO completion report or deployable release claimed.
|
||||
|
||||
@@ -153,5 +153,87 @@ class BackupUploadTests(unittest.TestCase):
|
||||
self.assertNotIn('Current recovery set copied', result.stdout)
|
||||
|
||||
|
||||
class BackupRetentionTests(unittest.TestCase):
|
||||
def retain(self, fail_operation='', *, previous_snapshot=False):
|
||||
source = (ROOT / 'deploy/deploy.sh').read_text(encoding='utf-8')
|
||||
start = source.index('apply_backup_retention() {')
|
||||
function = source[start:source.index('\n}\n', start) + 3]
|
||||
gate_start = source.index('if ! apply_backup_retention || ! upload_current_backup_set; then')
|
||||
gate = source[gate_start:source.index('\nfi\n', gate_start) + 4]
|
||||
fixtures = {'jobtracker-db-synthetic.sql.gz': b'database fixture',
|
||||
'jobtracker-files-synthetic.tar.gz': b'files fixture'}
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
folder = Path(temporary)
|
||||
for name, contents in fixtures.items():
|
||||
(folder / name).write_bytes(contents)
|
||||
monthly = folder / 'monthly' / '2099-01'
|
||||
if previous_snapshot:
|
||||
monthly.mkdir(parents=True)
|
||||
(monthly / 'jobtracker-db-earlier.sql.gz').write_bytes(b'earlier fixture')
|
||||
(monthly / '.complete').write_bytes(b'earlier\n')
|
||||
script = '\n'.join([
|
||||
'set -euo pipefail',
|
||||
'BACKUP_DIR=' + shlex.quote(folder.as_posix()),
|
||||
'BACKUP_STAMP=synthetic',
|
||||
'FAIL_OPERATION=' + shlex.quote(fail_operation),
|
||||
'date() { echo 2099-01; }',
|
||||
'upload_current_backup_set() { echo "UPLOAD WOULD START"; }',
|
||||
'find() { if [[ "$*" == *-delete* ]]; then echo "PRUNING WOULD START"; fi; }',
|
||||
'rm() { echo "CLEANUP REQUEST $1 $2"; }',
|
||||
'cp() {',
|
||||
' if [ "$FAIL_OPERATION" = cp ]; then',
|
||||
' printf partial > "${@: -1}/partial"; return 23;',
|
||||
' fi;',
|
||||
' command cp "$@";',
|
||||
'}',
|
||||
'mv() { if [ "$FAIL_OPERATION" = mv ]; then return 24; fi; command mv "$@"; }',
|
||||
function,
|
||||
gate,
|
||||
'echo "DEPLOY WOULD CONTINUE"',
|
||||
])
|
||||
result = subprocess.run([BASH, '--noprofile', '--norc'], input=script,
|
||||
text=True, encoding='utf-8', capture_output=True,
|
||||
cwd=ROOT, timeout=10,
|
||||
env={key: value for key, value in os.environ.items()
|
||||
if key.upper() in {'PATH', 'SYSTEMROOT', 'WINDIR', 'TEMP', 'TMP'}})
|
||||
artifacts = {file.name: file.read_bytes() for file in monthly.iterdir()} if monthly.exists() else {}
|
||||
return result, artifacts
|
||||
|
||||
def test_completed_monthly_snapshot_contains_both_artifacts_and_stamp(self):
|
||||
result, artifacts = self.retain()
|
||||
self.assertEqual(0, result.returncode, result.stdout + result.stderr)
|
||||
self.assertEqual({'jobtracker-db-synthetic.sql.gz': b'database fixture',
|
||||
'jobtracker-files-synthetic.tar.gz': b'files fixture',
|
||||
'.complete': b'synthetic\n'}, artifacts)
|
||||
self.assertIn('PRUNING WOULD START', result.stdout)
|
||||
self.assertIn('UPLOAD WOULD START', result.stdout)
|
||||
self.assertIn('DEPLOY WOULD CONTINUE', result.stdout)
|
||||
|
||||
def test_partial_monthly_copy_cannot_mark_snapshot_complete_or_proceed(self):
|
||||
result, artifacts = self.retain('cp')
|
||||
self.assertEqual(1, result.returncode, result.stdout + result.stderr)
|
||||
self.assertEqual({'partial': b'partial'}, artifacts)
|
||||
self.assertIn('Could not retain monthly recovery files', result.stdout)
|
||||
self.assertNotIn('PRUNING WOULD START', result.stdout)
|
||||
self.assertNotIn('UPLOAD WOULD START', result.stdout)
|
||||
self.assertNotIn('DEPLOY WOULD CONTINUE', result.stdout)
|
||||
|
||||
def test_marker_publish_failure_does_not_expose_complete_marker_or_proceed(self):
|
||||
result, artifacts = self.retain('mv')
|
||||
self.assertEqual(1, result.returncode, result.stdout + result.stderr)
|
||||
self.assertNotIn('.complete', artifacts)
|
||||
self.assertIn('CLEANUP REQUEST -f --', result.stdout)
|
||||
self.assertIn('Could not publish monthly recovery marker', result.stdout)
|
||||
self.assertNotIn('PRUNING WOULD START', result.stdout)
|
||||
self.assertNotIn('UPLOAD WOULD START', result.stdout)
|
||||
self.assertNotIn('DEPLOY WOULD CONTINUE', result.stdout)
|
||||
|
||||
def test_preserves_the_existing_completed_monthly_snapshot(self):
|
||||
result, artifacts = self.retain('cp', previous_snapshot=True)
|
||||
self.assertEqual(0, result.returncode, result.stdout + result.stderr)
|
||||
self.assertEqual({'jobtracker-db-earlier.sql.gz': b'earlier fixture', '.complete': b'earlier\n'}, artifacts)
|
||||
self.assertIn('DEPLOY WOULD CONTINUE', result.stdout)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user