Files
jobtrackingapp/scripts/test-deploy-backup.py
T

677 lines
39 KiB
Python

"""Exercise deployment backup functions with synthetic commands; never deploy."""
import os
import io
import gzip
from contextlib import closing
from pathlib import Path
import shlex
import shutil
import sqlite3
import subprocess
import sys
import tarfile
import tempfile
import unittest
ROOT = Path(__file__).resolve().parents[1]
BASH = 'C:/Program Files/Git/bin/bash.exe' if os.name == 'nt' else shutil.which('bash')
def retention_and_upload_gate(source):
start = source.index('if ! backup_recovery_volumes; then')
after_recovery = source.index('\nfi\n', start) + 4
return source[after_recovery:source.index('\nif [ "${DEPLOY_BACKUP_ONLY:-false}"', after_recovery)]
class BackupConnectionTests(unittest.TestCase):
def backup(self, connection):
source = (ROOT / 'deploy/deploy.sh').read_text(encoding='utf-8')
start = source.index('backup_database() {')
function = source[start:source.index('\n}\n', start) + 3]
with tempfile.TemporaryDirectory() as temporary:
folder = Path(temporary)
script = '\n'.join([
'set -euo pipefail',
'BACKUP_DIR=' + shlex.quote(folder.as_posix()),
'BACKUP_STAMP=synthetic',
'DATABASE_PROVIDER=mariadb',
'JOBTRACKER_CONNECTION_STRING=' + shlex.quote(connection),
'python3() { ' + shlex.quote(Path(sys.executable).as_posix()) + ' "$@"; }',
'docker() { echo "Unexpected Docker call" >&2; return 99; }',
'mariadb-dump() { printf "%s\\0" "$@" > "$BACKUP_DIR/args"; '
'printf "%s" "$MYSQL_PWD" > "$BACKUP_DIR/password"; '
'printf "CREATE TABLE synthetic (id int);\\n-- Dump completed\\n"; }',
'verify_sql_backup() { gzip -t "$1"; }',
function,
'backup_database',
])
result = subprocess.run([BASH, '--noprofile', '--norc'], input=script,
text=True, encoding='utf-8', capture_output=True, cwd=ROOT,
env={**os.environ, 'DEPLOY_SKIP_DB_BACKUP': 'false'})
arguments = (folder / 'args').read_bytes().split(b'\0')[:-1] if (folder / 'args').exists() else None
password = (folder / 'password').read_text(encoding='utf-8') if (folder / 'password').exists() else None
return result, arguments, password
def test_supported_host_alias_and_case_reach_dump_unchanged(self):
for host in ['Server', 'SERVER', 'Host', 'Data Source', 'Address', 'Addr', 'Network Address']:
with self.subTest(host=host):
result, args, password = self.backup(f'{host}=127.0.0.1;DATABASE=synthetic;UID=reader;PWD=dummy-secret')
self.assertEqual(0, result.returncode, result.stderr + result.stdout)
self.assertEqual([b'--host=127.0.0.1', b'--port=3306', b'--user=reader',
b'--single-transaction', b'--routines', b'--events', b'--quick', b'synthetic'], args)
self.assertEqual('dummy-secret', password)
self.assertNotIn('dummy-secret', result.stdout + result.stderr)
def test_quoted_separators_and_shell_characters_remain_data(self):
result, args, password = self.backup('Server=127.0.0.1;Port=3307;Database=synthetic;User ID=reader;Password="semi;eq=quote""$(false)"')
self.assertEqual(0, result.returncode, result.stderr + result.stdout)
self.assertIn(b'--port=3307', args)
self.assertEqual('semi;eq=quote"$(false)', password)
def test_empty_password_is_preserved(self):
result, args, password = self.backup('Server=127.0.0.1;Database=synthetic;User ID=reader;Password=')
self.assertEqual(0, result.returncode, result.stderr + result.stdout)
self.assertIsNotNone(args)
self.assertEqual('', password)
def test_single_quoted_unicode_password_and_database_alias_are_preserved(self):
result, args, password = self.backup("Host=127.0.0.1;Initial Catalog=synthetic;Username=reader;Password='søknad你好;it''s';SslMode=VerifyFull")
self.assertEqual(0, result.returncode, result.stderr + result.stdout)
self.assertEqual(b'synthetic', args[-1])
self.assertEqual("søknad你好;it's", password)
def test_invalid_or_ambiguous_input_never_invokes_dump_or_exposes_input(self):
for connection in [
'Host=127.0.0.1;Database=synthetic;Uid=reader;Port=0;Pwd=dummy-secret',
'Host=127.0.0.1;Database=synthetic;Uid=reader;Port=65536;Pwd=dummy-secret',
'Host=127.0.0.1;Server=other;Database=synthetic;Uid=reader;Pwd=dummy-secret',
'Host=127.0.0.1;Database=synthetic;Uid=reader;Pwd="dummy-secret',
'Host=127.0.0.1;Database=synthetic;Uid=reader;Pwd=dummy-secret\nline',
'Host=127.0.0.1;Uid=reader;Pwd=dummy-secret',
]:
with self.subTest(connection=connection.replace('dummy-secret', 'REDACTED')):
result, args, _ = self.backup(connection)
self.assertNotEqual(0, result.returncode)
self.assertIsNone(args)
self.assertNotIn('dummy-secret', result.stdout + result.stderr)
class BackupUploadTests(unittest.TestCase):
def upload(self, fail_operation=''):
source = (ROOT / 'deploy/deploy.sh').read_text(encoding='utf-8')
functions = []
for name in ['apply_backup_retention', 'upload_current_backup_set']:
start = source.index(name + '() {')
functions.append(source[start:source.index('\n}\n', start) + 3])
with tempfile.TemporaryDirectory() as temporary:
folder = Path(temporary)
for filename in ['jobtracker-db-synthetic.sql.gz', 'jobtracker-files-synthetic.tar.gz',
'jobtracker-db-older.sql.gz']:
(folder / filename).touch()
script = '\n'.join([
'set -euo pipefail',
'BACKUP_DIR=' + shlex.quote(folder.as_posix()),
'BACKUP_STAMP=synthetic',
'BACKUP_RCLONE_REMOTE=fixture:encrypted/',
'FAIL_OPERATION=' + shlex.quote(fail_operation),
'date() {',
' if [ "$FAIL_OPERATION" = clock ]; then return 2; fi;',
' if [ "$FAIL_OPERATION" = rollover ] && [ -f "$BACKUP_DIR/tick" ]; then echo 2099-02;',
' else touch "$BACKUP_DIR/tick"; echo 2099-01; fi;',
'}',
'find() { return 0; }',
'rclone() {',
' printf "UPLOAD %s %s %s\\n" "$1" "$3" "$4";',
' if [ "$1" = "$FAIL_OPERATION" ]; then return 23; fi;',
' if [ "$FAIL_OPERATION" = missing-monthly ] && [ -d "$BACKUP_DIR/monthly/2099-01" ]; then',
' mv "$BACKUP_DIR/monthly/2099-01" "$BACKUP_DIR/unavailable-snapshot"; fi;',
'}',
*functions,
retention_and_upload_gate(source),
'echo "DEPLOY WOULD CONTINUE"',
])
result = subprocess.run([BASH, '--noprofile', '--norc'], input=script,
text=True, encoding='utf-8', capture_output=True,
cwd=ROOT, timeout=10,
env={key: value for key, value in os.environ.items()
if key.upper() in {'PATH', 'SYSTEMROOT', 'WINDIR', 'TEMP', 'TMP'}})
calls = [line for line in result.stdout.splitlines() if line.startswith('UPLOAD ')]
return result, calls
def test_uploads_only_current_daily_files_then_monthly_snapshot(self):
result, calls = self.upload()
self.assertEqual(0, result.returncode, result.stdout + result.stderr)
self.assertEqual([
'UPLOAD copyto fixture:encrypted/daily/jobtracker-db-synthetic.sql.gz --immutable',
'UPLOAD copyto fixture:encrypted/daily/jobtracker-files-synthetic.tar.gz --immutable',
'UPLOAD copy fixture:encrypted/monthly/2099-01 --immutable',
], calls)
self.assertIn('DEPLOY WOULD CONTINUE', result.stdout)
self.assertIn('Current recovery set copied to configured off-host remote.', result.stdout)
def test_month_rollover_uploads_the_retained_snapshot(self):
result, calls = self.upload('rollover')
self.assertEqual(0, result.returncode, result.stdout + result.stderr)
self.assertIn('UPLOAD copy fixture:encrypted/monthly/2099-01 --immutable', calls)
def test_clock_failure_blocks_retention_and_upload(self):
result, calls = self.upload('clock')
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
self.assertEqual([], calls)
self.assertNotIn('Monthly recovery set retained', result.stdout)
self.assertNotIn('DEPLOY WOULD CONTINUE', result.stdout)
def test_missing_monthly_snapshot_stops_the_gate(self):
result, calls = self.upload('missing-monthly')
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
self.assertEqual(2, len(calls))
self.assertNotIn('Current recovery set copied', result.stdout)
self.assertNotIn('DEPLOY WOULD CONTINUE', result.stdout)
def test_daily_copy_failure_stops_before_remaining_files_and_monthly_copy(self):
result, calls = self.upload('copyto')
self.assertEqual(1, result.returncode, result.stdout + result.stderr)
self.assertEqual([
'UPLOAD copyto fixture:encrypted/daily/jobtracker-db-synthetic.sql.gz --immutable',
], calls)
self.assertIn('Daily off-host backup upload failed', result.stdout)
self.assertNotIn('DEPLOY WOULD CONTINUE', result.stdout)
self.assertNotIn('Current recovery set copied', result.stdout)
def test_monthly_copy_failure_blocks_deploy(self):
result, calls = self.upload('copy')
self.assertEqual(1, result.returncode, result.stdout + result.stderr)
self.assertEqual(3, len(calls))
self.assertEqual('UPLOAD copy fixture:encrypted/monthly/2099-01 --immutable', calls[-1])
self.assertIn('Monthly off-host backup upload failed', result.stdout)
self.assertNotIn('DEPLOY WOULD CONTINUE', result.stdout)
self.assertNotIn('Current recovery set copied', result.stdout)
class BackupRetentionTests(unittest.TestCase):
def retain(self, fail_operation='', *, previous_snapshot=False):
source = (ROOT / 'deploy/deploy.sh').read_text(encoding='utf-8')
start = source.index('apply_backup_retention() {')
function = source[start:source.index('\n}\n', start) + 3]
gate = retention_and_upload_gate(source)
fixtures = {'jobtracker-db-synthetic.sql.gz': b'database fixture',
'jobtracker-files-synthetic.tar.gz': b'files fixture'}
with tempfile.TemporaryDirectory() as temporary:
folder = Path(temporary)
for name, contents in fixtures.items():
(folder / name).write_bytes(contents)
monthly = folder / 'monthly' / '2099-01'
if previous_snapshot:
monthly.mkdir(parents=True)
(monthly / 'jobtracker-db-earlier.sql.gz').write_bytes(b'earlier fixture')
(monthly / '.complete').write_bytes(b'earlier\n')
script = '\n'.join([
'set -euo pipefail',
'BACKUP_DIR=' + shlex.quote(folder.as_posix()),
'BACKUP_STAMP=synthetic',
'BACKUP_DAILY_RETENTION_DAYS=30',
'BACKUP_MONTHLY_RETENTION_COUNT=1',
'FAIL_OPERATION=' + shlex.quote(fail_operation),
'date() { echo 2099-01; }',
'upload_current_backup_set() { echo "UPLOAD WOULD START"; }',
'find() {',
' if [[ "$*" == *-delete* ]]; then',
' echo "PRUNING WOULD START";',
' [ "$FAIL_OPERATION" != daily-find ]; return;',
' fi;',
' echo "MONTHLY ENUMERATION" >&2;',
' if [ "$FAIL_OPERATION" = empty-enumeration ]; then return 0; fi;',
' printf "%s/monthly/%s\\n" "$BACKUP_DIR" 2098-12 "$BACKUP_DIR" 2099-01 "$BACKUP_DIR" 2098-11;',
' [ "$FAIL_OPERATION" != monthly-find ];',
'}',
'sort() { if [ "$FAIL_OPERATION" = sort ]; then return 25; fi; command sort "$@"; }',
'rm() {',
' echo "CLEANUP REQUEST $1 $2";',
' if [ "$1" = -rf ]; then',
' echo "MONTHLY PRUNE ${3##*/}";',
' [ "$FAIL_OPERATION" != monthly-rm ];',
' fi;',
'}',
'cp() {',
' if [ "$FAIL_OPERATION" = cp ]; then',
' printf partial > "${@: -1}/partial"; return 23;',
' fi;',
' command cp "$@";',
'}',
'mv() { if [ "$FAIL_OPERATION" = mv ]; then return 24; fi; command mv "$@"; }',
function,
gate,
'echo "DEPLOY WOULD CONTINUE"',
])
result = subprocess.run([BASH, '--noprofile', '--norc'], input=script,
text=True, encoding='utf-8', capture_output=True,
cwd=ROOT, timeout=10,
env={key: value for key, value in os.environ.items()
if key.upper() in {'PATH', 'SYSTEMROOT', 'WINDIR', 'TEMP', 'TMP'}})
artifacts = {file.name: file.read_bytes() for file in monthly.iterdir()} if monthly.exists() else {}
return result, artifacts
def test_completed_monthly_snapshot_contains_both_artifacts_and_stamp(self):
result, artifacts = self.retain()
self.assertEqual(0, result.returncode, result.stdout + result.stderr)
self.assertEqual({'jobtracker-db-synthetic.sql.gz': b'database fixture',
'jobtracker-files-synthetic.tar.gz': b'files fixture',
'.complete': b'synthetic\n'}, artifacts)
self.assertIn('PRUNING WOULD START', result.stdout)
self.assertEqual(['MONTHLY PRUNE 2098-11', 'MONTHLY PRUNE 2098-12'],
[line for line in result.stdout.splitlines() if line.startswith('MONTHLY PRUNE ')])
self.assertIn('UPLOAD WOULD START', result.stdout)
self.assertIn('DEPLOY WOULD CONTINUE', result.stdout)
def test_partial_monthly_copy_cannot_mark_snapshot_complete_or_proceed(self):
result, artifacts = self.retain('cp')
self.assertEqual(1, result.returncode, result.stdout + result.stderr)
self.assertEqual({'partial': b'partial'}, artifacts)
self.assertIn('Could not retain monthly recovery files', result.stdout)
self.assertNotIn('PRUNING WOULD START', result.stdout)
self.assertNotIn('UPLOAD WOULD START', result.stdout)
self.assertNotIn('DEPLOY WOULD CONTINUE', result.stdout)
def test_marker_publish_failure_does_not_expose_complete_marker_or_proceed(self):
result, artifacts = self.retain('mv')
self.assertEqual(1, result.returncode, result.stdout + result.stderr)
self.assertNotIn('.complete', artifacts)
self.assertIn('CLEANUP REQUEST -f --', result.stdout)
self.assertIn('Could not publish monthly recovery marker', result.stdout)
self.assertNotIn('PRUNING WOULD START', result.stdout)
self.assertNotIn('UPLOAD WOULD START', result.stdout)
self.assertNotIn('DEPLOY WOULD CONTINUE', result.stdout)
def test_preserves_the_existing_completed_monthly_snapshot(self):
result, artifacts = self.retain('cp', previous_snapshot=True)
self.assertEqual(0, result.returncode, result.stdout + result.stderr)
self.assertEqual({'jobtracker-db-earlier.sql.gz': b'earlier fixture', '.complete': b'earlier\n'}, artifacts)
self.assertIn('DEPLOY WOULD CONTINUE', result.stdout)
def test_daily_prune_failure_blocks_monthly_pruning_upload_and_deploy(self):
result, _ = self.retain('daily-find')
self.assertEqual(1, result.returncode, result.stdout + result.stderr)
self.assertIn('Daily backup retention failed', result.stdout)
self.assertNotIn('MONTHLY ENUMERATION', result.stderr)
self.assertNotIn('MONTHLY PRUNE', result.stdout)
self.assertNotIn('UPLOAD WOULD START', result.stdout)
self.assertNotIn('DEPLOY WOULD CONTINUE', result.stdout)
self.assertNotIn('Backup retention applied', result.stdout)
def test_monthly_enumeration_failure_blocks_pruning_upload_and_deploy(self):
for operation in ['monthly-find', 'sort']:
with self.subTest(operation=operation):
result, _ = self.retain(operation)
self.assertEqual(1, result.returncode, result.stdout + result.stderr)
self.assertIn('Could not enumerate monthly backups', result.stdout)
self.assertNotIn('MONTHLY PRUNE', result.stdout)
self.assertNotIn('UPLOAD WOULD START', result.stdout)
self.assertNotIn('DEPLOY WOULD CONTINUE', result.stdout)
self.assertNotIn('Backup retention applied', result.stdout)
def test_monthly_prune_failure_stops_before_remaining_months_upload_and_deploy(self):
result, _ = self.retain('monthly-rm')
self.assertEqual(1, result.returncode, result.stdout + result.stderr)
self.assertIn('Monthly backup retention failed', result.stdout)
self.assertIn('MONTHLY PRUNE 2098-11', result.stdout)
self.assertNotIn('MONTHLY PRUNE 2098-12', result.stdout)
self.assertNotIn('UPLOAD WOULD START', result.stdout)
self.assertNotIn('DEPLOY WOULD CONTINUE', result.stdout)
self.assertNotIn('Backup retention applied', result.stdout)
def test_empty_monthly_enumeration_does_not_request_removal(self):
result, _ = self.retain('empty-enumeration')
self.assertEqual(0, result.returncode, result.stdout + result.stderr)
self.assertNotIn('MONTHLY PRUNE', result.stdout)
self.assertIn('Backup retention applied', result.stdout)
self.assertIn('UPLOAD WOULD START', result.stdout)
self.assertIn('DEPLOY WOULD CONTINUE', result.stdout)
class BackupPermissionTests(unittest.TestCase):
def permission_gate(self, kind, host_status, fallback_status):
source = (ROOT / 'deploy/deploy.sh').read_text(encoding='utf-8')
functions = []
for name in ['report_backup', 'verify_sql_backup', 'verify_volume_backup']:
start = source.index(name + '() {')
functions.append(source[start:source.index('\n}\n', start) + 3])
caller = 'backup_database' if kind == 'database' else 'backup_recovery_volumes'
start = source.index('if ! ' + caller + '; then')
gate = source[start:source.index('\nfi\n', start) + 4]
with tempfile.TemporaryDirectory(prefix='backup-permission-') as directory:
target = Path(directory) / 'synthetic-backup.gz'
payload = b'synthetic permission fixture'
if kind == 'recovery':
output = io.BytesIO()
with tarfile.open(fileobj=output, mode='w:gz') as bundle:
member = tarfile.TarInfo('jobtracker.db')
member.size = 16
bundle.addfile(member, io.BytesIO(b'SQLite format 3\x00'))
payload = output.getvalue()
target.write_bytes(payload)
script = '\n'.join([
'set -euo pipefail',
'BACKUP_DIR=' + shlex.quote(Path(directory).as_posix()),
'TARGET=' + shlex.quote(target.as_posix()),
'python3() { ' + shlex.quote(Path(sys.executable).as_posix()) + ' "$@"; }',
'chmod() { echo "HOST PERMISSIONS"; return ' + str(host_status) + '; }',
'docker() { if [[ "${@: -1}" == *chmod* ]]; then echo "FALLBACK PERMISSIONS"; return ' + str(fallback_status) + '; fi; return 0; }',
'du() { printf "1K\\tsynthetic\\n"; }',
'rm() { echo "UNEXPECTED REMOVAL"; return 99; }',
'gzip() { printf "CREATE TABLE synthetic (id int);\\n-- Dump completed\\n"; }',
'verify_backup_file_shape() { [ -s "$1" ]; }',
*functions,
'backup_database() { verify_sql_backup "$TARGET"; }',
'backup_recovery_volumes() { verify_volume_backup "$TARGET"; }',
gate,
'echo "NEXT DEPLOY GATE WOULD START"',
])
result = subprocess.run([BASH, '--noprofile', '--norc'], input=script, text=True,
encoding='utf-8', capture_output=True, cwd=ROOT)
self.assertEqual(payload, target.read_bytes(), 'Permission failure must not remove the backup artifact')
self.assertNotIn('UNEXPECTED REMOVAL', result.stdout)
return result
def test_host_permission_success_does_not_need_fallback(self):
for kind in ['database', 'recovery']:
with self.subTest(kind=kind):
result = self.permission_gate(kind, 0, 23)
self.assertEqual(0, result.returncode, result.stdout + result.stderr)
self.assertNotIn('FALLBACK PERMISSIONS', result.stdout)
self.assertIn('Backup verified', result.stdout)
self.assertIn('NEXT DEPLOY GATE WOULD START', result.stdout)
def test_fallback_permission_success_preserves_verification(self):
for kind in ['database', 'recovery']:
with self.subTest(kind=kind):
result = self.permission_gate(kind, 1, 0)
self.assertEqual(0, result.returncode, result.stdout + result.stderr)
self.assertIn('FALLBACK PERMISSIONS', result.stdout)
self.assertIn('Backup verified', result.stdout)
self.assertIn('NEXT DEPLOY GATE WOULD START', result.stdout)
def test_both_permission_failures_block_verification_and_next_gate(self):
for kind in ['database', 'recovery']:
with self.subTest(kind=kind):
result = self.permission_gate(kind, 1, 23)
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
self.assertNotIn('Backup verified', result.stdout)
self.assertNotIn('NEXT DEPLOY GATE WOULD START', result.stdout)
class BackupSqlTrailerTests(unittest.TestCase):
def verify_dump(self, text):
source = (ROOT / 'deploy/deploy.sh').read_text(encoding='utf-8')
functions = []
for name in ['verify_backup_file_shape', 'verify_sql_backup']:
start = source.index(name + '() {')
functions.append(source[start:source.index('\n}\n', start) + 3])
with tempfile.TemporaryDirectory() as temporary:
folder = Path(temporary)
target = folder / 'synthetic.sql.gz'
with gzip.open(target, 'wb') as bundle:
bundle.write(text.encode('utf-8'))
script = '\n'.join([
'set -euo pipefail',
'report_backup() { echo "FORMAT VERIFIED"; }',
'rm() { echo "INVALID ARTIFACT REJECTION REQUESTED"; }',
*functions,
'if ! verify_sql_backup ' + shlex.quote(target.as_posix()) + '; then exit 1; fi',
'echo "NEXT DEPLOY GATE WOULD START"',
])
return subprocess.run([BASH, '--noprofile', '--norc'], input=script,
text=True, encoding='utf-8', capture_output=True, cwd=ROOT,
timeout=10, env={key: value for key, value in os.environ.items()
if key.upper() in {'PATH', 'SYSTEMROOT', 'WINDIR', 'TEMP', 'TMP'}})
def test_completion_phrase_in_sql_value_is_not_a_dump_trailer(self):
result = self.verify_dump("CREATE TABLE synthetic (value text);\nINSERT INTO synthetic VALUES ('Dump completed');\n")
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
self.assertNotIn('FORMAT VERIFIED', result.stdout)
self.assertNotIn('NEXT DEPLOY GATE WOULD START', result.stdout)
def test_completion_comment_before_more_sql_is_not_a_final_trailer(self):
result = self.verify_dump("CREATE TABLE synthetic (value text);\n-- Dump completed on synthetic-date\nINSERT INTO synthetic VALUES ('more work');\n")
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
self.assertNotIn('FORMAT VERIFIED', result.stdout)
self.assertNotIn('NEXT DEPLOY GATE WOULD START', result.stdout)
def test_final_completion_comment_is_accepted_with_supported_line_endings(self):
for trailer, ending in [('-- Dump completed on synthetic-date', '\n'),
('-- Dump completed', '\n'),
('-- Dump completed on synthetic-date', '\r\n')]:
with self.subTest(trailer=trailer, ending=repr(ending)):
result = self.verify_dump(ending.join(['CREATE TABLE synthetic (id int);', trailer, '', ' ', '']))
self.assertEqual(0, result.returncode, result.stdout + result.stderr)
self.assertIn('FORMAT VERIFIED', result.stdout)
self.assertIn('NEXT DEPLOY GATE WOULD START', result.stdout)
def test_missing_completion_comment_remains_rejected(self):
result = self.verify_dump('CREATE TABLE synthetic (id int);\n')
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
self.assertNotIn('NEXT DEPLOY GATE WOULD START', result.stdout)
class BackupSqliteSnapshotTests(unittest.TestCase):
def backup(self, root):
source = (ROOT / 'deploy/deploy.sh').read_text(encoding='utf-8')
functions = []
for name in ['backup_database', 'verify_backup_file_shape', 'verify_volume_backup']:
start = source.index(name + '() {')
functions.append(source[start:source.index('\n}\n', start) + 3])
launch = next(line for line in functions[0].splitlines() if 'python:3.12.10-slim-bookworm sh -c' in line)
command = launch.split("sh -c '", 1)[1].split("'", 1)[0]
local_command = command.replace('exec python3', 'exec "$PYTHON"').replace(' /data ', ' "$VOLUME" ')
local_command = local_command.replace('"/backup/', '"$BACKUP_DIR/')
# Map only container paths and the Python executable to this host. The
# shell body, helper from stdin, SQLite, tar and validator all run.
script = '\n'.join([
'set -euo pipefail',
'BACKUP_DIR=' + shlex.quote((root / 'backups').as_posix()),
'VOLUME=' + shlex.quote((root / 'data').as_posix()),
'PYTHON=' + shlex.quote(Path(sys.executable).as_posix()),
'DATABASE_PROVIDER=sqlite; BACKUP_STAMP=fixture; DEPLOY_SKIP_DB_BACKUP=false; COMPOSE_PROJECT_NAME=fixture',
'python3() { "$PYTHON" "$@"; }',
'report_backup() { echo "FORMAT VERIFIED"; }',
'docker() {',
' if [[ "$1 $2" == "volume inspect" ]]; then return 0; fi',
' [[ "$*" == *"fixture_jobtracker_data:/data:ro "* && "$*" == *" --network none "* ]] || return 99',
' local arg',
' [[ "${!#}" == ' + shlex.quote(command) + ' ]] || return 99',
' for arg in "$@"; do case "$arg" in ARCHIVE_NAME=*) export "$arg";; esac; done',
' export PYTHON VOLUME BACKUP_DIR',
' bash -c ' + shlex.quote(local_command),
'}',
*functions,
'if ! backup_database; then exit 1; fi',
'echo "NEXT DEPLOY GATE WOULD START"',
])
env = os.environ.copy()
if os.name == 'nt':
env['PATH'] = str(Path(BASH).parents[1] / 'usr/bin') + os.pathsep + env['PATH']
return subprocess.run([BASH, '--noprofile', '--norc'], input=script, cwd=ROOT, env=env,
capture_output=True, text=True, encoding='utf-8', timeout=30)
def test_standalone_snapshot_contains_committed_wal_rows_with_writer_open(self):
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
volume = root / 'data'
nested = volume / 'nested [copy]'
nested.mkdir(parents=True)
(volume / 'jobtracker.db-notes').write_bytes(b'attachment data')
(nested / 'unrelated.db-wal').write_bytes(b'unrelated file')
# An identically named database in a nested directory retains its path.
with closing(sqlite3.connect(nested / 'jobtracker.db')) as other:
other.execute('CREATE TABLE nested_fixture (id INTEGER)')
with closing(sqlite3.connect(volume / 'jobtracker.db')) as writer:
self.assertEqual(('wal',), writer.execute('PRAGMA journal_mode=WAL').fetchone())
writer.execute('PRAGMA wal_autocheckpoint=0')
writer.execute('CREATE TABLE receipt (id INTEGER PRIMARY KEY)')
writer.execute('INSERT INTO receipt VALUES (1)')
writer.commit()
self.assertEqual(0, writer.execute('PRAGMA wal_checkpoint(TRUNCATE)').fetchone()[0])
writer.execute('INSERT INTO receipt VALUES (2)')
writer.commit()
writer.execute('INSERT INTO receipt VALUES (3)')
self.assertGreater((volume / 'jobtracker.db-wal').stat().st_size, 0)
result = self.backup(root)
self.assertEqual(0, result.returncode, result.stdout + result.stderr)
self.assertIn('NEXT DEPLOY GATE WOULD START', result.stdout)
writer.rollback()
self.assertEqual(0, writer.execute('PRAGMA wal_checkpoint(TRUNCATE)').fetchone()[0])
with tarfile.open(root / 'backups/jobtracker-sqlite-fixture.tar.gz') as bundle:
for sidecar in ['-wal', '-shm', '-journal']:
self.assertNotIn('./jobtracker.db' + sidecar, bundle.getnames())
for path, expected in [('./jobtracker.db-notes', b'attachment data'),
('./nested [copy]/unrelated.db-wal', b'unrelated file')]:
with bundle.extractfile(path) as stream:
self.assertEqual(expected, stream.read())
for name, query, expected in [('./jobtracker.db', 'SELECT id FROM receipt ORDER BY id', [(1,), (2,)]),
('./nested [copy]/jobtracker.db', 'SELECT * FROM nested_fixture', [])]:
with self.subTest(name=name), bundle.extractfile(name) as stream:
restored = root / 'restored.db'
restored.write_bytes(stream.read())
with closing(sqlite3.connect(restored.as_uri() + '?mode=ro', uri=True)) as copy:
self.assertEqual(expected, copy.execute(query).fetchall())
self.assertEqual(('ok',), copy.execute('PRAGMA integrity_check').fetchone())
self.assertEqual(('delete',), copy.execute('PRAGMA journal_mode').fetchone())
def test_invalid_database_prevents_archive_promotion(self):
for contents in [b'', b'SQLite format 3\x00' + bytes(200)]:
with self.subTest(size=len(contents)), tempfile.TemporaryDirectory() as temp:
root = Path(temp)
(root / 'data').mkdir()
(root / 'data/jobtracker.db').write_bytes(contents)
result = self.backup(root)
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
self.assertNotIn('FORMAT VERIFIED', result.stdout)
self.assertNotIn('NEXT DEPLOY GATE WOULD START', result.stdout)
self.assertEqual([], list((root / 'backups').iterdir()))
class BackupDatabaseMemberTests(unittest.TestCase):
def test_host_tar_producer_with_database_and_sidecar(self):
producer = shutil.which('tar')
self.assertIsNotNone(producer, 'The host tar command is required for producer qualification.')
with tempfile.TemporaryDirectory() as temporary:
folder = Path(temporary)
volume = folder / 'data'
volume.mkdir()
database = volume / 'jobtracker.db'
connection = sqlite3.connect(database)
try:
with connection:
connection.execute('CREATE TABLE synthetic (id INTEGER PRIMARY KEY)')
connection.execute('INSERT INTO synthetic VALUES (1)')
finally:
connection.close()
(volume / 'jobtracker.db-wal').write_bytes(b'fixture sidecar name only')
for case in ('database-and-sidecar', 'sidecar-only'):
with self.subTest(case=case):
if case == 'sidecar-only':
database.unlink()
archive = case + '.tar.gz'
subprocess.run([producer, 'czf', archive, '-C', 'data', '.'],
cwd=folder, check=True, capture_output=True, timeout=15)
result = subprocess.run([sys.executable, ROOT / 'scripts/validate-sqlite-backup.py', archive],
cwd=folder, capture_output=True, timeout=15)
self.assertEqual(0 if database.exists() else 1, result.returncode, result.stderr)
if database.exists():
with tarfile.open(folder / archive, 'r:gz') as bundle:
with bundle.extractfile('./jobtracker.db') as archived:
self.assertEqual(database.read_bytes(), archived.read())
def verify_archive(self, member_name, truncated_tar=False, member_type=tarfile.REGTYPE, contents=None):
source = (ROOT / 'deploy/deploy.sh').read_text(encoding='utf-8')
functions = []
for name in ['verify_backup_file_shape', 'verify_volume_backup']:
start = source.index(name + '() {')
functions.append(source[start:source.index('\n}\n', start) + 3])
with tempfile.TemporaryDirectory() as temporary:
folder = Path(temporary)
archive = folder / 'synthetic.tar.gz'
member = tarfile.TarInfo(member_name)
member.type = member_type
if member_type in (tarfile.SYMTYPE, tarfile.LNKTYPE):
member.linkname = 'missing-database'
if contents is None:
database = folder / 'fixture.db'
with sqlite3.connect(database) as connection:
connection.execute('CREATE TABLE synthetic (id INTEGER PRIMARY KEY, value TEXT)')
connection.execute("INSERT INTO synthetic(value) VALUES ('fixture only')")
connection.close()
contents = database.read_bytes()
if truncated_tar:
member.size = 4096
with gzip.open(archive, 'wb') as bundle:
bundle.write(member.tobuf())
else:
member.size = len(contents) if member_type == tarfile.REGTYPE else 0
with tarfile.open(archive, 'w:gz') as bundle:
bundle.addfile(member, io.BytesIO(contents) if member_type == tarfile.REGTYPE else None)
script = '\n'.join([
'set -euo pipefail',
'BACKUP_DIR=' + shlex.quote(folder.as_posix()),
'python3() { ' + shlex.quote(Path(sys.executable).as_posix()) + ' "$@"; }',
'report_backup() { echo "FORMAT VERIFIED"; }',
'docker() { echo "Unexpected Docker validation call" >&2; return 99; }',
*functions,
'if ! verify_volume_backup "$BACKUP_DIR/synthetic.tar.gz"; then exit 1; fi',
'echo "NEXT DEPLOY GATE WOULD START"',
])
return subprocess.run([BASH, '--noprofile', '--norc'], input=script,
text=True, encoding='utf-8', capture_output=True, cwd=ROOT,
timeout=10, env={key: value for key, value in os.environ.items()
if key.upper() in {'PATH', 'SYSTEMROOT', 'WINDIR', 'TEMP', 'TMP'}})
def test_sidecar_or_similar_name_is_not_the_database_member(self):
for name in ['jobtracker.db-wal', 'old-jobtracker.db', 'jobtracker.db.backup']:
with self.subTest(member=name):
result = self.verify_archive(name)
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
self.assertNotIn('FORMAT VERIFIED', result.stdout)
self.assertNotIn('NEXT DEPLOY GATE WOULD START', result.stdout)
def test_exact_database_member_at_root_or_in_a_directory_is_accepted(self):
for name in ['jobtracker.db', './jobtracker.db', 'data/jobtracker.db']:
with self.subTest(member=name):
result = self.verify_archive(name)
self.assertEqual(0, result.returncode, result.stdout + result.stderr)
self.assertIn('FORMAT VERIFIED', result.stdout)
self.assertIn('NEXT DEPLOY GATE WOULD START', result.stdout)
def test_valid_gzip_with_truncated_tar_database_member_is_rejected(self):
result = self.verify_archive('jobtracker.db', truncated_tar=True)
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
self.assertNotIn('FORMAT VERIFIED', result.stdout)
self.assertNotIn('NEXT DEPLOY GATE WOULD START', result.stdout)
def test_links_and_special_members_do_not_substitute_for_database_bytes(self):
for kind in [tarfile.SYMTYPE, tarfile.LNKTYPE, tarfile.FIFOTYPE, tarfile.CHRTYPE]:
with self.subTest(kind=kind):
result = self.verify_archive('jobtracker.db', member_type=kind)
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
self.assertNotIn('FORMAT VERIFIED', result.stdout)
self.assertNotIn('NEXT DEPLOY GATE WOULD START', result.stdout)
def test_empty_or_non_sqlite_database_bytes_block_the_next_gate(self):
for contents in [b'', b'unrelated application bytes', b'SQLite format 3X\x00']:
with self.subTest(contents=contents):
result = self.verify_archive('jobtracker.db', contents=contents)
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
self.assertNotIn('FORMAT VERIFIED', result.stdout)
self.assertNotIn('NEXT DEPLOY GATE WOULD START', result.stdout)
def test_unsafe_database_member_paths_block_the_next_gate(self):
for name in ['../jobtracker.db', '/jobtracker.db', 'data/../../jobtracker.db']:
with self.subTest(name=name):
result = self.verify_archive(name)
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
self.assertNotIn('FORMAT VERIFIED', result.stdout)
self.assertNotIn('NEXT DEPLOY GATE WOULD START', result.stdout)
if __name__ == '__main__':
unittest.main()