fix(deploy): load production environment before backup
deploy.sh symlinked /opt/job-tracker/shared/.env for docker compose but never loaded it into its own shell. Its own decisions therefore ran against an empty environment: DATABASE_PROVIDER fell back to sqlite on a MariaDB host, so the deploy tarred the data volume, printed "Backup verified" and continued with no database dump. The operator saw a green backup line and a new file in the backups directory, and had no restore point. Load the shared env before any decision. Parsed line by line rather than sourced, because a compose .env is not a shell script and an unquoted value containing spaces would execute as a command. Values already in the environment win, so CI-provided APP_VERSION and friends still override the file. No value is echoed. Remove the sqlite default. DATABASE_PROVIDER must be stated; missing or unrecognised aborts the deploy. Validate deployment configuration before the backup, and so before anything is built, stopped or replaced: the connection string when the provider needs one, AI_SERVICE_TOKEN (compose declares it with :?) and AUTH_JWT_KEY (the backend throws on a blank key). Names in the output, never values. Verify each backup against its own format. A dump must be valid gzip, contain CREATE TABLE, and carry the "Dump completed" trailer, so a dump that died partway through is rejected. An archive must contain jobtracker.db. A tar can no longer pass the dump check. Also resolve the SQLite volume by its project-prefixed name and fail if absent. The bare jobtracker_data name would have silently created an empty volume and backed that up -- the same class of bug, found while testing this fix. Verified against a seeded MariaDB 11 container and real Docker volumes: provider selection, all four validation failures, both backup formats and their failure paths, truncated and trailer-stripped dumps, and zero secret occurrences across every test's output. Docs updated for the drift: deploy/README.md, deploy/first-production- deployment.md, docs/release-candidate-review.md (B1 closed) and .env.example, which now names the two database variables. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
cesnimda
+211
-16
@@ -19,6 +19,48 @@ if [ ! -L "$ENV_TARGET" ] && [ ! -f "$ENV_TARGET" ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Load the shared environment into THIS shell.
|
||||
#
|
||||
# The symlink above is for docker compose, which reads .env itself. The script's
|
||||
# own decisions — which backup to take, which variables are missing — ran against
|
||||
# an empty environment until this existed, so DATABASE_PROVIDER defaulted to
|
||||
# sqlite and a MariaDB host silently got a volume tar instead of a dump, with a
|
||||
# "Backup verified" line to match. See docs/release-candidate-review.md (B1).
|
||||
#
|
||||
# Parsed line by line rather than sourced: a compose .env is not a shell script,
|
||||
# so an unquoted value containing spaces would execute as a command under `.`.
|
||||
# Values already present in the environment win, so CI-provided APP_VERSION and
|
||||
# friends still override the file. No value is ever echoed.
|
||||
# ---------------------------------------------------------------------------
|
||||
load_env_file() {
|
||||
local file="$1" line key value
|
||||
|
||||
while IFS= read -r line || [ -n "$line" ]; do
|
||||
line="${line%$'\r'}"
|
||||
case "$line" in ''|'#'*) continue ;; esac
|
||||
case "$line" in *=*) ;; *) continue ;; esac
|
||||
|
||||
key="${line%%=*}"
|
||||
value="${line#*=}"
|
||||
key="${key#export }"
|
||||
key="${key//[[:space:]]/}"
|
||||
case "$key" in ''|*[!A-Za-z0-9_]*) continue ;; esac
|
||||
|
||||
case "$value" in
|
||||
\"*\") value="${value#\"}"; value="${value%\"}" ;;
|
||||
\'*\') value="${value#\'}"; value="${value%\'}" ;;
|
||||
esac
|
||||
|
||||
# Already set (CI, or an explicit override on the command line) wins.
|
||||
[ -n "${!key+x}" ] && continue
|
||||
export "$key=$value"
|
||||
done < "$file"
|
||||
}
|
||||
|
||||
load_env_file "$ENV_TARGET"
|
||||
echo "Loaded deployment environment from ${ENV_SOURCE}"
|
||||
|
||||
export APP_VERSION="${APP_VERSION:-0.0.0}"
|
||||
export APP_COMMIT_SHA="${APP_COMMIT_SHA:-unknown}"
|
||||
export APP_BUILD_STAMP="${APP_BUILD_STAMP:-unknown}"
|
||||
@@ -28,6 +70,74 @@ compose() {
|
||||
docker compose "$@"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configuration validation, before anything is built, stopped or replaced.
|
||||
#
|
||||
# Everything checked here is fatal later anyway — compose declares AI_SERVICE_TOKEN
|
||||
# with `:?`, and the backend throws on a blank Auth__JwtKey. Failing here costs an
|
||||
# aborted deploy; failing there costs a half-replaced stack.
|
||||
#
|
||||
# Names only in the output. Never values.
|
||||
# ---------------------------------------------------------------------------
|
||||
require_var() {
|
||||
local name="$1" hint="$2"
|
||||
|
||||
if [ -z "${!name:-}" ]; then
|
||||
echo "Missing required deployment variable: ${name}"
|
||||
echo " ${hint}"
|
||||
echo " Set it in ${ENV_SOURCE}."
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
validate_deploy_config() {
|
||||
local failed=0
|
||||
|
||||
# Deliberately no default. Guessing this wrong means backing up the wrong
|
||||
# database and reporting success — the exact failure this block exists to stop.
|
||||
if [ -z "${DATABASE_PROVIDER:-}" ]; then
|
||||
echo "Missing required deployment variable: DATABASE_PROVIDER"
|
||||
echo " Set to 'mariadb' (or 'mysql') for a MariaDB deployment, or 'sqlite'."
|
||||
echo " There is no default: the wrong value backs up the wrong database."
|
||||
echo " Set it in ${ENV_SOURCE}."
|
||||
failed=1
|
||||
else
|
||||
case "$(printf '%s' "$DATABASE_PROVIDER" | tr '[:upper:]' '[:lower:]')" in
|
||||
mysql|mariadb)
|
||||
require_var JOBTRACKER_CONNECTION_STRING \
|
||||
"MariaDB connection string. Required to take a database dump." || failed=1
|
||||
;;
|
||||
sqlite)
|
||||
;;
|
||||
*)
|
||||
echo "DATABASE_PROVIDER is not a recognised value. Use mariadb, mysql or sqlite."
|
||||
failed=1
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
require_var AI_SERVICE_TOKEN \
|
||||
"Shared secret between backend and ai-service. docker compose refuses to start without it." || failed=1
|
||||
|
||||
# docker-compose.yml sets Auth__Require=true unconditionally, and the backend
|
||||
# throws at startup rather than issuing tokens signed with a blank key.
|
||||
require_var AUTH_JWT_KEY \
|
||||
"JWT signing key. With Auth__Require=true the backend refuses to start without it." || failed=1
|
||||
|
||||
if [ -z "${APP_PUBLIC_BASE_URL:-}" ]; then
|
||||
echo "Note: APP_PUBLIC_BASE_URL is not set — the post-deploy public smoke check will be skipped."
|
||||
fi
|
||||
|
||||
if [ "$failed" -ne 0 ]; then
|
||||
echo "Deployment configuration is incomplete. Nothing was built, stopped or replaced."
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Deployment configuration validated (database provider: ${DATABASE_PROVIDER})."
|
||||
return 0
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Database backup, taken BEFORE anything is stopped, built or replaced.
|
||||
#
|
||||
@@ -44,27 +154,43 @@ backup_database() {
|
||||
return 0
|
||||
fi
|
||||
|
||||
local provider="${DATABASE_PROVIDER:-sqlite}"
|
||||
# validate_deploy_config has already established this is set and recognised.
|
||||
local provider
|
||||
provider="$(printf '%s' "$DATABASE_PROVIDER" | tr '[:upper:]' '[:lower:]')"
|
||||
|
||||
local stamp
|
||||
stamp="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
|
||||
if [ "$provider" != "mysql" ] && [ "$provider" != "mariadb" ]; then
|
||||
if [ "$provider" = "sqlite" ]; then
|
||||
# SQLite lives in the jobtracker_data volume. Tar it from a throwaway container
|
||||
# so the host needs no sqlite tooling and no knowledge of the volume layout.
|
||||
#
|
||||
# docker compose prefixes volume names with the project name, which defaults to
|
||||
# the directory name. Naming the bare volume here would silently CREATE an empty
|
||||
# one and back that up instead, so resolve it and fail if it is not there.
|
||||
local volume="${COMPOSE_PROJECT_NAME:-$(basename "$PWD")}_jobtracker_data"
|
||||
if ! docker volume inspect "$volume" >/dev/null 2>&1; then
|
||||
echo "SQLite data volume '${volume}' does not exist. Aborting deploy."
|
||||
echo " Volumes are prefixed with the compose project name; set COMPOSE_PROJECT_NAME if it differs."
|
||||
echo " Available: $(docker volume ls -q | grep jobtracker_data | tr '\n' ' ')"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local target="$BACKUP_DIR/jobtracker-sqlite-${stamp}.tar.gz"
|
||||
echo "Backing up SQLite data volume to ${target}"
|
||||
echo "Backing up SQLite data volume '${volume}' to ${target}"
|
||||
# The archive path is built inside the container: passing /backup/... as an argument
|
||||
# gets rewritten by MSYS path translation when the script is run from Git Bash.
|
||||
if ! docker run --rm \
|
||||
-v jobtracker_data:/data:ro \
|
||||
-v "$volume":/data:ro \
|
||||
-v "$BACKUP_DIR":/backup \
|
||||
-e ARCHIVE_NAME="$(basename "$target")" \
|
||||
alpine:3 sh -c 'tar czf "/backup/$ARCHIVE_NAME" -C /data .'; then
|
||||
echo "SQLite volume backup FAILED. Aborting deploy."
|
||||
rm -f "$target"
|
||||
return 1
|
||||
fi
|
||||
verify_backup "$target"
|
||||
verify_volume_backup "$target"
|
||||
return $?
|
||||
fi
|
||||
|
||||
@@ -120,14 +246,20 @@ backup_database() {
|
||||
return 1
|
||||
fi
|
||||
|
||||
verify_backup "$target" "CREATE TABLE"
|
||||
verify_sql_backup "$target"
|
||||
}
|
||||
|
||||
# A dump that exists but is empty or truncated is worse than none, because it
|
||||
# looks like a restore point. Check size, and content when we know what to expect.
|
||||
verify_backup() {
|
||||
local target="$1"
|
||||
local expect="${2:-}"
|
||||
# A backup that exists but is empty, truncated, or is the wrong KIND of backup is
|
||||
# worse than none, because it looks like a restore point. Each provider gets the
|
||||
# check that proves its own format — a tar cannot pass the dump check and a dump
|
||||
# cannot pass the archive check.
|
||||
#
|
||||
# Note on `set +o pipefail` below: `grep -q` exits at the first match, which
|
||||
# SIGPIPEs the decompressor upstream. Under pipefail that fails the pipeline and
|
||||
# would reject a perfectly good backup.
|
||||
|
||||
verify_backup_file_shape() {
|
||||
local target="$1" kind="$2"
|
||||
|
||||
if [ ! -s "$target" ]; then
|
||||
echo "Backup file ${target} is missing or empty. Aborting deploy."
|
||||
@@ -135,20 +267,83 @@ verify_backup() {
|
||||
return 1
|
||||
fi
|
||||
|
||||
local size
|
||||
size="$(du -h "$target" | cut -f1)"
|
||||
|
||||
if [ -n "$expect" ] && ! gzip -dc "$target" | grep -q "$expect"; then
|
||||
echo "Backup ${target} does not contain '${expect}' — it is not a usable dump. Aborting deploy."
|
||||
if ! gzip -t "$target" 2>/dev/null; then
|
||||
echo "Backup ${target} is not a valid gzip archive (truncated ${kind}?). Aborting deploy."
|
||||
rm -f "$target"
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
report_backup() {
|
||||
local target="$1" size
|
||||
size="$(du -h "$target" | cut -f1)"
|
||||
echo "Backup verified: ${target} (${size})"
|
||||
echo "Retention is manual — old backups in ${BACKUP_DIR} are never deleted automatically."
|
||||
return 0
|
||||
}
|
||||
|
||||
verify_sql_backup() {
|
||||
local target="$1"
|
||||
|
||||
verify_backup_file_shape "$target" "dump" || return 1
|
||||
|
||||
local has_schema=1 has_trailer=1
|
||||
set +o pipefail
|
||||
if gzip -dc "$target" | grep -q "CREATE TABLE"; then has_schema=0; fi
|
||||
# mariadb-dump / mysqldump write "-- Dump completed on ..." as the last line.
|
||||
# Its absence means the dump died partway through.
|
||||
if gzip -dc "$target" | grep -q "Dump completed"; then has_trailer=0; fi
|
||||
set -o pipefail
|
||||
|
||||
if [ "$has_schema" -ne 0 ]; then
|
||||
echo "Backup ${target} contains no CREATE TABLE statement — it is not a usable dump. Aborting deploy."
|
||||
rm -f "$target"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ "$has_trailer" -ne 0 ]; then
|
||||
echo "Backup ${target} has no dump trailer — it is truncated. Aborting deploy."
|
||||
rm -f "$target"
|
||||
return 1
|
||||
fi
|
||||
|
||||
report_backup "$target"
|
||||
}
|
||||
|
||||
verify_volume_backup() {
|
||||
local target="$1"
|
||||
|
||||
verify_backup_file_shape "$target" "archive" || return 1
|
||||
|
||||
# Prove this is the DATABASE volume, not merely a non-empty tar. This is what
|
||||
# catches an archive taken from the wrong volume, or from a freshly created
|
||||
# empty one.
|
||||
# Listed in the same container that wrote it, not with the host's tar: GNU tar
|
||||
# reads a leading "C:/" as an rsh host and fails outright when this script runs
|
||||
# from Git Bash. Docker is already required to have produced the archive.
|
||||
local has_db=1
|
||||
if docker run --rm \
|
||||
-v "$(dirname "$target")":/backup:ro \
|
||||
-e ARCHIVE_NAME="$(basename "$target")" \
|
||||
alpine:3 sh -c 'tar tzf "/backup/$ARCHIVE_NAME" | grep -q "jobtracker\.db"' >/dev/null 2>&1; then
|
||||
has_db=0
|
||||
fi
|
||||
|
||||
if [ "$has_db" -ne 0 ]; then
|
||||
echo "Archive ${target} contains no jobtracker.db — it is not a SQLite data backup. Aborting deploy."
|
||||
rm -f "$target"
|
||||
return 1
|
||||
fi
|
||||
|
||||
report_backup "$target"
|
||||
}
|
||||
|
||||
if ! validate_deploy_config; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! backup_database; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user