cesnimda
362 lines
20 KiB
Python
362 lines
20 KiB
Python
"""Exercise deployment backup functions with synthetic commands; never deploy."""
|
|
import os
|
|
from pathlib import Path
|
|
import shlex
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
BASH = 'C:/Program Files/Git/bin/bash.exe' if os.name == 'nt' else shutil.which('bash')
|
|
|
|
|
|
class BackupConnectionTests(unittest.TestCase):
|
|
def backup(self, connection):
|
|
source = (ROOT / 'deploy/deploy.sh').read_text(encoding='utf-8')
|
|
start = source.index('backup_database() {')
|
|
function = source[start:source.index('\n}\n', start) + 3]
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
folder = Path(temporary)
|
|
script = '\n'.join([
|
|
'set -euo pipefail',
|
|
'BACKUP_DIR=' + shlex.quote(folder.as_posix()),
|
|
'BACKUP_STAMP=synthetic',
|
|
'DATABASE_PROVIDER=mariadb',
|
|
'JOBTRACKER_CONNECTION_STRING=' + shlex.quote(connection),
|
|
'python3() { ' + shlex.quote(Path(sys.executable).as_posix()) + ' "$@"; }',
|
|
'docker() { echo "Unexpected Docker call" >&2; return 99; }',
|
|
'mariadb-dump() { printf "%s\\0" "$@" > "$BACKUP_DIR/args"; '
|
|
'printf "%s" "$MYSQL_PWD" > "$BACKUP_DIR/password"; '
|
|
'printf "CREATE TABLE synthetic (id int);\\n-- Dump completed\\n"; }',
|
|
'verify_sql_backup() { gzip -t "$1"; }',
|
|
function,
|
|
'backup_database',
|
|
])
|
|
result = subprocess.run([BASH, '--noprofile', '--norc'], input=script,
|
|
text=True, encoding='utf-8', capture_output=True, cwd=ROOT,
|
|
env={**os.environ, 'DEPLOY_SKIP_DB_BACKUP': 'false'})
|
|
arguments = (folder / 'args').read_bytes().split(b'\0')[:-1] if (folder / 'args').exists() else None
|
|
password = (folder / 'password').read_text(encoding='utf-8') if (folder / 'password').exists() else None
|
|
return result, arguments, password
|
|
|
|
def test_supported_host_alias_and_case_reach_dump_unchanged(self):
|
|
for host in ['Server', 'SERVER', 'Host', 'Data Source', 'Address', 'Addr', 'Network Address']:
|
|
with self.subTest(host=host):
|
|
result, args, password = self.backup(f'{host}=127.0.0.1;DATABASE=synthetic;UID=reader;PWD=dummy-secret')
|
|
self.assertEqual(0, result.returncode, result.stderr + result.stdout)
|
|
self.assertEqual([b'--host=127.0.0.1', b'--port=3306', b'--user=reader',
|
|
b'--single-transaction', b'--routines', b'--events', b'--quick', b'synthetic'], args)
|
|
self.assertEqual('dummy-secret', password)
|
|
self.assertNotIn('dummy-secret', result.stdout + result.stderr)
|
|
|
|
def test_quoted_separators_and_shell_characters_remain_data(self):
|
|
result, args, password = self.backup('Server=127.0.0.1;Port=3307;Database=synthetic;User ID=reader;Password="semi;eq=quote""$(false)"')
|
|
self.assertEqual(0, result.returncode, result.stderr + result.stdout)
|
|
self.assertIn(b'--port=3307', args)
|
|
self.assertEqual('semi;eq=quote"$(false)', password)
|
|
|
|
def test_empty_password_is_preserved(self):
|
|
result, args, password = self.backup('Server=127.0.0.1;Database=synthetic;User ID=reader;Password=')
|
|
self.assertEqual(0, result.returncode, result.stderr + result.stdout)
|
|
self.assertIsNotNone(args)
|
|
self.assertEqual('', password)
|
|
|
|
def test_single_quoted_unicode_password_and_database_alias_are_preserved(self):
|
|
result, args, password = self.backup("Host=127.0.0.1;Initial Catalog=synthetic;Username=reader;Password='søknad你好;it''s';SslMode=VerifyFull")
|
|
self.assertEqual(0, result.returncode, result.stderr + result.stdout)
|
|
self.assertEqual(b'synthetic', args[-1])
|
|
self.assertEqual("søknad你好;it's", password)
|
|
|
|
def test_invalid_or_ambiguous_input_never_invokes_dump_or_exposes_input(self):
|
|
for connection in [
|
|
'Host=127.0.0.1;Database=synthetic;Uid=reader;Port=0;Pwd=dummy-secret',
|
|
'Host=127.0.0.1;Database=synthetic;Uid=reader;Port=65536;Pwd=dummy-secret',
|
|
'Host=127.0.0.1;Server=other;Database=synthetic;Uid=reader;Pwd=dummy-secret',
|
|
'Host=127.0.0.1;Database=synthetic;Uid=reader;Pwd="dummy-secret',
|
|
'Host=127.0.0.1;Database=synthetic;Uid=reader;Pwd=dummy-secret\nline',
|
|
'Host=127.0.0.1;Uid=reader;Pwd=dummy-secret',
|
|
]:
|
|
with self.subTest(connection=connection.replace('dummy-secret', 'REDACTED')):
|
|
result, args, _ = self.backup(connection)
|
|
self.assertNotEqual(0, result.returncode)
|
|
self.assertIsNone(args)
|
|
self.assertNotIn('dummy-secret', result.stdout + result.stderr)
|
|
|
|
|
|
class BackupUploadTests(unittest.TestCase):
|
|
def upload(self, fail_operation=''):
|
|
source = (ROOT / 'deploy/deploy.sh').read_text(encoding='utf-8')
|
|
start = source.index('upload_current_backup_set() {')
|
|
function = source[start:source.index('\n}\n', start) + 3]
|
|
gate_start = source.index('if ! apply_backup_retention || ! upload_current_backup_set; then')
|
|
gate = source[gate_start:source.index('\nfi\n', gate_start) + 4]
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
folder = Path(temporary)
|
|
for filename in ['jobtracker-db-synthetic.sql.gz', 'jobtracker-files-synthetic.tar.gz',
|
|
'jobtracker-db-older.sql.gz']:
|
|
(folder / filename).touch()
|
|
(folder / 'monthly' / '2099-01').mkdir(parents=True)
|
|
script = '\n'.join([
|
|
'set -euo pipefail',
|
|
'BACKUP_DIR=' + shlex.quote(folder.as_posix()),
|
|
'BACKUP_STAMP=synthetic',
|
|
'BACKUP_RCLONE_REMOTE=fixture:encrypted/',
|
|
'FAIL_OPERATION=' + shlex.quote(fail_operation),
|
|
'date() { echo 2099-01; }',
|
|
'apply_backup_retention() { return 0; }',
|
|
'rclone() {',
|
|
' printf "UPLOAD %s %s %s\\n" "$1" "$3" "$4";',
|
|
' if [ "$1" = "$FAIL_OPERATION" ]; then return 23; fi;',
|
|
'}',
|
|
function,
|
|
gate,
|
|
'echo "DEPLOY WOULD CONTINUE"',
|
|
])
|
|
result = subprocess.run([BASH, '--noprofile', '--norc'], input=script,
|
|
text=True, encoding='utf-8', capture_output=True,
|
|
cwd=ROOT, timeout=10,
|
|
env={key: value for key, value in os.environ.items()
|
|
if key.upper() in {'PATH', 'SYSTEMROOT', 'WINDIR', 'TEMP', 'TMP'}})
|
|
calls = [line for line in result.stdout.splitlines() if line.startswith('UPLOAD ')]
|
|
return result, calls
|
|
|
|
def test_uploads_only_current_daily_files_then_monthly_snapshot(self):
|
|
result, calls = self.upload()
|
|
self.assertEqual(0, result.returncode, result.stdout + result.stderr)
|
|
self.assertEqual([
|
|
'UPLOAD copyto fixture:encrypted/daily/jobtracker-db-synthetic.sql.gz --immutable',
|
|
'UPLOAD copyto fixture:encrypted/daily/jobtracker-files-synthetic.tar.gz --immutable',
|
|
'UPLOAD copy fixture:encrypted/monthly/2099-01 --immutable',
|
|
], calls)
|
|
self.assertIn('DEPLOY WOULD CONTINUE', result.stdout)
|
|
self.assertIn('Current recovery set copied to configured off-host remote.', result.stdout)
|
|
|
|
def test_daily_copy_failure_stops_before_remaining_files_and_monthly_copy(self):
|
|
result, calls = self.upload('copyto')
|
|
self.assertEqual(1, result.returncode, result.stdout + result.stderr)
|
|
self.assertEqual([
|
|
'UPLOAD copyto fixture:encrypted/daily/jobtracker-db-synthetic.sql.gz --immutable',
|
|
], calls)
|
|
self.assertIn('Daily off-host backup upload failed', result.stdout)
|
|
self.assertNotIn('DEPLOY WOULD CONTINUE', result.stdout)
|
|
self.assertNotIn('Current recovery set copied', result.stdout)
|
|
|
|
def test_monthly_copy_failure_blocks_deploy(self):
|
|
result, calls = self.upload('copy')
|
|
self.assertEqual(1, result.returncode, result.stdout + result.stderr)
|
|
self.assertEqual(3, len(calls))
|
|
self.assertEqual('UPLOAD copy fixture:encrypted/monthly/2099-01 --immutable', calls[-1])
|
|
self.assertIn('Monthly off-host backup upload failed', result.stdout)
|
|
self.assertNotIn('DEPLOY WOULD CONTINUE', result.stdout)
|
|
self.assertNotIn('Current recovery set copied', result.stdout)
|
|
|
|
|
|
class BackupRetentionTests(unittest.TestCase):
|
|
def retain(self, fail_operation='', *, previous_snapshot=False):
|
|
source = (ROOT / 'deploy/deploy.sh').read_text(encoding='utf-8')
|
|
start = source.index('apply_backup_retention() {')
|
|
function = source[start:source.index('\n}\n', start) + 3]
|
|
gate_start = source.index('if ! apply_backup_retention || ! upload_current_backup_set; then')
|
|
gate = source[gate_start:source.index('\nfi\n', gate_start) + 4]
|
|
fixtures = {'jobtracker-db-synthetic.sql.gz': b'database fixture',
|
|
'jobtracker-files-synthetic.tar.gz': b'files fixture'}
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
folder = Path(temporary)
|
|
for name, contents in fixtures.items():
|
|
(folder / name).write_bytes(contents)
|
|
monthly = folder / 'monthly' / '2099-01'
|
|
if previous_snapshot:
|
|
monthly.mkdir(parents=True)
|
|
(monthly / 'jobtracker-db-earlier.sql.gz').write_bytes(b'earlier fixture')
|
|
(monthly / '.complete').write_bytes(b'earlier\n')
|
|
script = '\n'.join([
|
|
'set -euo pipefail',
|
|
'BACKUP_DIR=' + shlex.quote(folder.as_posix()),
|
|
'BACKUP_STAMP=synthetic',
|
|
'BACKUP_DAILY_RETENTION_DAYS=30',
|
|
'BACKUP_MONTHLY_RETENTION_COUNT=1',
|
|
'FAIL_OPERATION=' + shlex.quote(fail_operation),
|
|
'date() { echo 2099-01; }',
|
|
'upload_current_backup_set() { echo "UPLOAD WOULD START"; }',
|
|
'find() {',
|
|
' if [[ "$*" == *-delete* ]]; then',
|
|
' echo "PRUNING WOULD START";',
|
|
' [ "$FAIL_OPERATION" != daily-find ]; return;',
|
|
' fi;',
|
|
' echo "MONTHLY ENUMERATION" >&2;',
|
|
' if [ "$FAIL_OPERATION" = empty-enumeration ]; then return 0; fi;',
|
|
' printf "%s/monthly/%s\\n" "$BACKUP_DIR" 2098-12 "$BACKUP_DIR" 2099-01 "$BACKUP_DIR" 2098-11;',
|
|
' [ "$FAIL_OPERATION" != monthly-find ];',
|
|
'}',
|
|
'sort() { if [ "$FAIL_OPERATION" = sort ]; then return 25; fi; command sort "$@"; }',
|
|
'rm() {',
|
|
' echo "CLEANUP REQUEST $1 $2";',
|
|
' if [ "$1" = -rf ]; then',
|
|
' echo "MONTHLY PRUNE ${3##*/}";',
|
|
' [ "$FAIL_OPERATION" != monthly-rm ];',
|
|
' fi;',
|
|
'}',
|
|
'cp() {',
|
|
' if [ "$FAIL_OPERATION" = cp ]; then',
|
|
' printf partial > "${@: -1}/partial"; return 23;',
|
|
' fi;',
|
|
' command cp "$@";',
|
|
'}',
|
|
'mv() { if [ "$FAIL_OPERATION" = mv ]; then return 24; fi; command mv "$@"; }',
|
|
function,
|
|
gate,
|
|
'echo "DEPLOY WOULD CONTINUE"',
|
|
])
|
|
result = subprocess.run([BASH, '--noprofile', '--norc'], input=script,
|
|
text=True, encoding='utf-8', capture_output=True,
|
|
cwd=ROOT, timeout=10,
|
|
env={key: value for key, value in os.environ.items()
|
|
if key.upper() in {'PATH', 'SYSTEMROOT', 'WINDIR', 'TEMP', 'TMP'}})
|
|
artifacts = {file.name: file.read_bytes() for file in monthly.iterdir()} if monthly.exists() else {}
|
|
return result, artifacts
|
|
|
|
def test_completed_monthly_snapshot_contains_both_artifacts_and_stamp(self):
|
|
result, artifacts = self.retain()
|
|
self.assertEqual(0, result.returncode, result.stdout + result.stderr)
|
|
self.assertEqual({'jobtracker-db-synthetic.sql.gz': b'database fixture',
|
|
'jobtracker-files-synthetic.tar.gz': b'files fixture',
|
|
'.complete': b'synthetic\n'}, artifacts)
|
|
self.assertIn('PRUNING WOULD START', result.stdout)
|
|
self.assertEqual(['MONTHLY PRUNE 2098-11', 'MONTHLY PRUNE 2098-12'],
|
|
[line for line in result.stdout.splitlines() if line.startswith('MONTHLY PRUNE ')])
|
|
self.assertIn('UPLOAD WOULD START', result.stdout)
|
|
self.assertIn('DEPLOY WOULD CONTINUE', result.stdout)
|
|
|
|
def test_partial_monthly_copy_cannot_mark_snapshot_complete_or_proceed(self):
|
|
result, artifacts = self.retain('cp')
|
|
self.assertEqual(1, result.returncode, result.stdout + result.stderr)
|
|
self.assertEqual({'partial': b'partial'}, artifacts)
|
|
self.assertIn('Could not retain monthly recovery files', result.stdout)
|
|
self.assertNotIn('PRUNING WOULD START', result.stdout)
|
|
self.assertNotIn('UPLOAD WOULD START', result.stdout)
|
|
self.assertNotIn('DEPLOY WOULD CONTINUE', result.stdout)
|
|
|
|
def test_marker_publish_failure_does_not_expose_complete_marker_or_proceed(self):
|
|
result, artifacts = self.retain('mv')
|
|
self.assertEqual(1, result.returncode, result.stdout + result.stderr)
|
|
self.assertNotIn('.complete', artifacts)
|
|
self.assertIn('CLEANUP REQUEST -f --', result.stdout)
|
|
self.assertIn('Could not publish monthly recovery marker', result.stdout)
|
|
self.assertNotIn('PRUNING WOULD START', result.stdout)
|
|
self.assertNotIn('UPLOAD WOULD START', result.stdout)
|
|
self.assertNotIn('DEPLOY WOULD CONTINUE', result.stdout)
|
|
|
|
def test_preserves_the_existing_completed_monthly_snapshot(self):
|
|
result, artifacts = self.retain('cp', previous_snapshot=True)
|
|
self.assertEqual(0, result.returncode, result.stdout + result.stderr)
|
|
self.assertEqual({'jobtracker-db-earlier.sql.gz': b'earlier fixture', '.complete': b'earlier\n'}, artifacts)
|
|
self.assertIn('DEPLOY WOULD CONTINUE', result.stdout)
|
|
|
|
def test_daily_prune_failure_blocks_monthly_pruning_upload_and_deploy(self):
|
|
result, _ = self.retain('daily-find')
|
|
self.assertEqual(1, result.returncode, result.stdout + result.stderr)
|
|
self.assertIn('Daily backup retention failed', result.stdout)
|
|
self.assertNotIn('MONTHLY ENUMERATION', result.stderr)
|
|
self.assertNotIn('MONTHLY PRUNE', result.stdout)
|
|
self.assertNotIn('UPLOAD WOULD START', result.stdout)
|
|
self.assertNotIn('DEPLOY WOULD CONTINUE', result.stdout)
|
|
self.assertNotIn('Backup retention applied', result.stdout)
|
|
|
|
def test_monthly_enumeration_failure_blocks_pruning_upload_and_deploy(self):
|
|
for operation in ['monthly-find', 'sort']:
|
|
with self.subTest(operation=operation):
|
|
result, _ = self.retain(operation)
|
|
self.assertEqual(1, result.returncode, result.stdout + result.stderr)
|
|
self.assertIn('Could not enumerate monthly backups', result.stdout)
|
|
self.assertNotIn('MONTHLY PRUNE', result.stdout)
|
|
self.assertNotIn('UPLOAD WOULD START', result.stdout)
|
|
self.assertNotIn('DEPLOY WOULD CONTINUE', result.stdout)
|
|
self.assertNotIn('Backup retention applied', result.stdout)
|
|
|
|
def test_monthly_prune_failure_stops_before_remaining_months_upload_and_deploy(self):
|
|
result, _ = self.retain('monthly-rm')
|
|
self.assertEqual(1, result.returncode, result.stdout + result.stderr)
|
|
self.assertIn('Monthly backup retention failed', result.stdout)
|
|
self.assertIn('MONTHLY PRUNE 2098-11', result.stdout)
|
|
self.assertNotIn('MONTHLY PRUNE 2098-12', result.stdout)
|
|
self.assertNotIn('UPLOAD WOULD START', result.stdout)
|
|
self.assertNotIn('DEPLOY WOULD CONTINUE', result.stdout)
|
|
self.assertNotIn('Backup retention applied', result.stdout)
|
|
|
|
def test_empty_monthly_enumeration_does_not_request_removal(self):
|
|
result, _ = self.retain('empty-enumeration')
|
|
self.assertEqual(0, result.returncode, result.stdout + result.stderr)
|
|
self.assertNotIn('MONTHLY PRUNE', result.stdout)
|
|
self.assertIn('Backup retention applied', result.stdout)
|
|
self.assertIn('UPLOAD WOULD START', result.stdout)
|
|
self.assertIn('DEPLOY WOULD CONTINUE', result.stdout)
|
|
|
|
|
|
class BackupPermissionTests(unittest.TestCase):
|
|
def permission_gate(self, kind, host_status, fallback_status):
|
|
source = (ROOT / 'deploy/deploy.sh').read_text(encoding='utf-8')
|
|
functions = []
|
|
for name in ['report_backup', 'verify_sql_backup', 'verify_volume_backup']:
|
|
start = source.index(name + '() {')
|
|
functions.append(source[start:source.index('\n}\n', start) + 3])
|
|
caller = 'backup_database' if kind == 'database' else 'backup_recovery_volumes'
|
|
start = source.index('if ! ' + caller + '; then')
|
|
gate = source[start:source.index('\nfi\n', start) + 4]
|
|
with tempfile.TemporaryDirectory(prefix='backup-permission-') as directory:
|
|
target = Path(directory) / 'synthetic-backup.gz'
|
|
payload = b'synthetic permission fixture'
|
|
target.write_bytes(payload)
|
|
script = '\n'.join([
|
|
'set -euo pipefail',
|
|
'BACKUP_DIR=' + shlex.quote(Path(directory).as_posix()),
|
|
'TARGET=' + shlex.quote(target.as_posix()),
|
|
'chmod() { echo "HOST PERMISSIONS"; return ' + str(host_status) + '; }',
|
|
'docker() { if [[ "${@: -1}" == *chmod* ]]; then echo "FALLBACK PERMISSIONS"; return ' + str(fallback_status) + '; fi; return 0; }',
|
|
'du() { printf "1K\\tsynthetic\\n"; }',
|
|
'rm() { echo "UNEXPECTED REMOVAL"; return 99; }',
|
|
'gzip() { printf "CREATE TABLE synthetic (id int);\\n-- Dump completed\\n"; }',
|
|
'verify_backup_file_shape() { [ -s "$1" ]; }',
|
|
*functions,
|
|
'backup_database() { verify_sql_backup "$TARGET"; }',
|
|
'backup_recovery_volumes() { verify_volume_backup "$TARGET"; }',
|
|
gate,
|
|
'echo "NEXT DEPLOY GATE WOULD START"',
|
|
])
|
|
result = subprocess.run([BASH, '--noprofile', '--norc'], input=script, text=True,
|
|
encoding='utf-8', capture_output=True, cwd=ROOT)
|
|
self.assertEqual(payload, target.read_bytes(), 'Permission failure must not remove the backup artifact')
|
|
self.assertNotIn('UNEXPECTED REMOVAL', result.stdout)
|
|
return result
|
|
|
|
def test_host_permission_success_does_not_need_fallback(self):
|
|
for kind in ['database', 'recovery']:
|
|
with self.subTest(kind=kind):
|
|
result = self.permission_gate(kind, 0, 23)
|
|
self.assertEqual(0, result.returncode, result.stdout + result.stderr)
|
|
self.assertNotIn('FALLBACK PERMISSIONS', result.stdout)
|
|
self.assertIn('Backup verified', result.stdout)
|
|
self.assertIn('NEXT DEPLOY GATE WOULD START', result.stdout)
|
|
|
|
def test_fallback_permission_success_preserves_verification(self):
|
|
for kind in ['database', 'recovery']:
|
|
with self.subTest(kind=kind):
|
|
result = self.permission_gate(kind, 1, 0)
|
|
self.assertEqual(0, result.returncode, result.stdout + result.stderr)
|
|
self.assertIn('FALLBACK PERMISSIONS', result.stdout)
|
|
self.assertIn('Backup verified', result.stdout)
|
|
self.assertIn('NEXT DEPLOY GATE WOULD START', result.stdout)
|
|
|
|
def test_both_permission_failures_block_verification_and_next_gate(self):
|
|
for kind in ['database', 'recovery']:
|
|
with self.subTest(kind=kind):
|
|
result = self.permission_gate(kind, 1, 23)
|
|
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
|
|
self.assertNotIn('Backup verified', result.stdout)
|
|
self.assertNotIn('NEXT DEPLOY GATE WOULD START', result.stdout)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|