feat(ops): protect complete pre-deploy recovery set
This commit is contained in:
@@ -193,7 +193,11 @@ deploy if the backup fails**. Nothing else in the deploy runs without a restore
|
||||
- **Selection:** driven solely by `DATABASE_PROVIDER`, which must be set.
|
||||
- `mariadb` / `mysql` → SQL dump, `jobtracker-<database>-<UTC timestamp>.sql.gz`, e.g.
|
||||
`jobtracker-jobtracker-20260719T153759Z.sql.gz`
|
||||
- MariaDB deployments also archive `jobtracker_data`, which contains attachments, imported CVs,
|
||||
generated exports and Data Protection keys.
|
||||
- `sqlite` → data volume archive, `jobtracker-sqlite-<UTC timestamp>.tar.gz`
|
||||
- every deployment archives the separate account-deletion tombstone volume when it exists;
|
||||
restore it independently and never replace it with an older application-data snapshot
|
||||
- anything else → the deploy stops
|
||||
- **The filename tells you which path ran.** If you expect a MariaDB deploy and find a
|
||||
`jobtracker-sqlite-*.tar.gz`, the environment is wrong — that is the exact failure this check exists
|
||||
@@ -215,6 +219,7 @@ like a restore point.
|
||||
|---|---|
|
||||
| MariaDB | non-empty; valid gzip; contains `CREATE TABLE`; contains the `Dump completed` trailer that `mariadb-dump` writes last, so a dump that died partway through is rejected |
|
||||
| SQLite | non-empty; valid gzip; the archive actually contains `jobtracker.db` |
|
||||
| Recovery volumes | non-empty valid gzip archives; a missing required application-data volume aborts the deploy |
|
||||
|
||||
A failed check deletes the file rather than leaving something that looks like a backup.
|
||||
|
||||
|
||||
+52
-4
@@ -265,6 +265,7 @@ validate_deploy_config() {
|
||||
# exactly when a restore point matters. See deploy/README.md.
|
||||
# ---------------------------------------------------------------------------
|
||||
BACKUP_DIR="${BACKUP_DIR:-/opt/job-tracker/backups}"
|
||||
BACKUP_STAMP="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
|
||||
backup_database() {
|
||||
if [ "${DEPLOY_SKIP_DB_BACKUP:-false}" = "true" ]; then
|
||||
@@ -276,8 +277,6 @@ backup_database() {
|
||||
local provider
|
||||
provider="$(printf '%s' "$DATABASE_PROVIDER" | tr '[:upper:]' '[:lower:]')"
|
||||
|
||||
local stamp
|
||||
stamp="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
if [ ! -w "$BACKUP_DIR" ]; then
|
||||
echo "Backup directory ${BACKUP_DIR} is not writable; repairing ownership via Docker."
|
||||
@@ -304,7 +303,7 @@ backup_database() {
|
||||
return 1
|
||||
fi
|
||||
|
||||
local target="$BACKUP_DIR/jobtracker-sqlite-${stamp}.tar.gz"
|
||||
local target="$BACKUP_DIR/jobtracker-sqlite-${BACKUP_STAMP}.tar.gz"
|
||||
echo "Backing up SQLite data volume '${volume}' to ${target}"
|
||||
# The archive path is built inside the container: passing /backup/... as an argument
|
||||
# gets rewritten by MSYS path translation when the script is run from Git Bash.
|
||||
@@ -341,7 +340,7 @@ backup_database() {
|
||||
return 1
|
||||
fi
|
||||
|
||||
local target="$BACKUP_DIR/jobtracker-${db_name}-${stamp}.sql.gz"
|
||||
local target="$BACKUP_DIR/jobtracker-${db_name}-${BACKUP_STAMP}.sql.gz"
|
||||
echo "Backing up MariaDB database '${db_name}' on ${db_host}:${db_port} to ${target}"
|
||||
|
||||
# The password goes via MYSQL_PWD, never on the command line, so it cannot leak
|
||||
@@ -469,6 +468,51 @@ verify_volume_backup() {
|
||||
report_backup "$target"
|
||||
}
|
||||
|
||||
backup_named_volume() {
|
||||
local volume="$1" label="$2" required="$3"
|
||||
if ! docker volume inspect "$volume" >/dev/null 2>&1; then
|
||||
if [ "$required" = "true" ]; then
|
||||
echo "Required recovery volume '${volume}' does not exist. Aborting deploy."
|
||||
return 1
|
||||
fi
|
||||
echo "Recovery volume '${volume}' does not exist yet; nothing to preserve."
|
||||
return 0
|
||||
fi
|
||||
|
||||
local target="$BACKUP_DIR/jobtracker-${label}-${BACKUP_STAMP}.tar.gz"
|
||||
echo "Backing up recovery volume '${volume}' to ${target}"
|
||||
if ! docker run --rm \
|
||||
-v "$volume":/data:ro \
|
||||
-v "$BACKUP_DIR":/backup \
|
||||
-e ARCHIVE_NAME="$(basename "$target")" \
|
||||
alpine:3 sh -c 'tar czf "/backup/$ARCHIVE_NAME" -C /data .'; then
|
||||
echo "Recovery volume backup FAILED. Aborting deploy."
|
||||
rm -f "$target"
|
||||
return 1
|
||||
fi
|
||||
|
||||
verify_backup_file_shape "$target" "archive" || return 1
|
||||
report_backup "$target"
|
||||
}
|
||||
|
||||
backup_recovery_volumes() {
|
||||
local project_name="${COMPOSE_PROJECT_NAME:-$(basename "$PWD")}" provider
|
||||
provider="$(printf '%s' "$DATABASE_PROVIDER" | tr '[:upper:]' '[:lower:]')"
|
||||
|
||||
# With MariaDB, the database dump does not include attachments, imported CVs,
|
||||
# exports or Data Protection keys stored in /data. SQLite already archives this
|
||||
# exact volume in backup_database, so avoid writing the same archive twice.
|
||||
if [ "$provider" != "sqlite" ]; then
|
||||
backup_named_volume "${project_name}_jobtracker_data" "files-${project_name}" true || return 1
|
||||
fi
|
||||
|
||||
# Tombstones are restored independently from application data so an older data
|
||||
# restore cannot resurrect an erased identity. The volume may be absent on the
|
||||
# first deployment of the deletion lifecycle.
|
||||
backup_named_volume "${project_name}_jobtracker_deletion_tombstones" \
|
||||
"deletion-tombstones-${project_name}" false
|
||||
}
|
||||
|
||||
resolve_existing_web_proxy_subnet
|
||||
|
||||
if ! validate_deploy_config; then
|
||||
@@ -479,6 +523,10 @@ if ! backup_database; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! backup_recovery_volumes; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
previous_backend_ref=""
|
||||
previous_frontend_ref=""
|
||||
capture_previous_core_image() {
|
||||
|
||||
Reference in New Issue
Block a user