Files
jobtrackingapp/scripts/archive-sqlite-volume.py
T

76 lines
3.3 KiB
Python

"""Archive a volume with standalone SQLite snapshots and the remaining original files."""
from contextlib import closing
import os
from pathlib import Path
import shutil
import sqlite3
import stat
import subprocess
import sys
import tempfile
def snapshot(source, destination):
info = source.lstat()
if not stat.S_ISREG(info.st_mode):
raise ValueError('Expected a regular database file.')
with source.open('rb') as original:
if original.read(16) != b'SQLite format 3\x00':
raise ValueError('Expected a SQLite database header.')
with closing(sqlite3.connect(source.as_uri() + '?mode=ro', uri=True)) as reader:
# Establish one read transaction before copying. WAL writers may keep
# committing without moving the snapshot or restarting a large backup.
reader.execute('BEGIN')
reader.execute('PRAGMA schema_version').fetchone()
with closing(sqlite3.connect(destination)) as copied:
reader.backup(copied)
if copied.execute('PRAGMA journal_mode=DELETE').fetchone() != ('delete',):
raise ValueError('Snapshot still requires a journal.')
if hasattr(os, 'chown'):
os.chown(destination, info.st_uid, info.st_gid)
shutil.copystat(source, destination)
def archive_volume(volume, target):
def raise_walk_error(error):
raise error
# Keep uncompressed snapshots on the throwaway container filesystem.
with tempfile.TemporaryDirectory(prefix='sqlite-snapshot-') as temporary:
stage = Path(temporary)
excludes = []
snapshots = []
for folder, dirs, files in os.walk(volume, onerror=raise_walk_error):
if 'jobtracker.db' in dirs:
raise ValueError('Expected a database file.')
if 'jobtracker.db' not in files:
continue
source = Path(folder) / 'jobtracker.db'
name = source.relative_to(volume).as_posix()
staged = stage / 'snapshot' / name
staged.parent.mkdir(parents=True, exist_ok=True)
snapshot(source, staged)
snapshots.append('snapshot/' + name)
for suffix in ('', '-shm', '-wal', '-journal'):
excludes.append('--exclude=./' + name + suffix)
if not snapshots:
raise ValueError('No database found.')
# The original volume's names start with "./". Only staged members match
# this transform. Literal, anchored excludes preserve unrelated sidecars.
tar = shutil.which('tar')
if not tar:
raise FileNotFoundError('GNU tar is required.')
subprocess.run([tar, 'czf', target.name, '--anchored', '--no-wildcards', *excludes,
'--transform=flags=r;s|^snapshot/|./|', '-C', volume.as_posix(), '.',
'-C', stage.as_posix(), *snapshots],
cwd=target.parent, check=True, stderr=subprocess.DEVNULL)
if __name__ == '__main__':
try:
volume, target = map(Path, sys.argv[1:])
archive_volume(volume.resolve(strict=True), target.absolute())
except (OSError, ValueError, sqlite3.Error, subprocess.CalledProcessError):
# Member names and database contents may contain private information.
sys.exit('SQLite volume snapshot or archive creation failed.')