fix(deploy): restore previous release on failure

This commit is contained in:
cesnimda
2026-08-31 11:57:03 +02:00
parent b9e703ee7c
commit 5f0c0b0549
5 changed files with 62 additions and 8 deletions
+48 -8
View File
@@ -65,6 +65,8 @@ export APP_VERSION="${APP_VERSION:-0.0.0}"
export APP_COMMIT_SHA="${APP_COMMIT_SHA:-unknown}"
export APP_BUILD_STAMP="${APP_BUILD_STAMP:-unknown}"
export DEPLOY_BUILD_AI_SERVICE="${DEPLOY_BUILD_AI_SERVICE:-false}"
export BACKEND_IMAGE="${BACKEND_IMAGE:-jobtracker-backend:${APP_COMMIT_SHA}}"
export FRONTEND_IMAGE="${FRONTEND_IMAGE:-jobtracker-frontend:${APP_COMMIT_SHA}}"
compose() {
docker compose -f docker-compose.yml "$@"
@@ -477,6 +479,50 @@ if ! backup_database; then
exit 1
fi
previous_backend_ref=""
previous_frontend_ref=""
capture_previous_core_image() {
local service="$1" previous_ref="$2" container_id image_id
container_id="$(compose ps -q "$service" 2>/dev/null || true)"
[ -n "$container_id" ] || return 0
image_id="$(docker inspect -f '{{.Image}}' "$container_id" 2>/dev/null || true)"
[ -n "$image_id" ] || return 0
docker tag "$image_id" "$previous_ref"
printf '%s' "$previous_ref"
}
previous_backend_ref="$(capture_previous_core_image backend jobtracker-backend:previous)"
previous_frontend_ref="$(capture_previous_core_image frontend jobtracker-frontend:previous)"
replacement_started=false
auth_config_body_file=""
auth_config_headers_file=""
finish_deploy() {
local status=$?
trap - EXIT
[ -z "$auth_config_body_file" ] || rm -f "$auth_config_body_file"
[ -z "$auth_config_headers_file" ] || rm -f "$auth_config_headers_file"
if [ "$status" -ne 0 ] && [ "$replacement_started" = true ]; then
if [ -n "$previous_backend_ref" ] && [ -n "$previous_frontend_ref" ]; then
echo "Deployment failed after core replacement; restoring previous healthy images."
if BACKEND_IMAGE="$previous_backend_ref" FRONTEND_IMAGE="$previous_frontend_ref" \
compose up -d --force-recreate --remove-orphans backend frontend; then
echo "Previous core release restored. Deployment remains failed; inspect the logs above."
else
echo "Automatic core rollback failed. Manual recovery is required."
compose ps || true
compose logs --tail=200 backend frontend || true
fi
else
echo "Deployment failed, but no complete previous core release was available for automatic rollback."
fi
fi
exit "$status"
}
trap finish_deploy EXIT
build_core_with_recovery() {
if compose build backend frontend; then
return 0
@@ -500,7 +546,6 @@ build_ai_with_recovery() {
compose build --no-cache ai-service
}
compose pull || true
build_core_with_recovery
if [ "$DEPLOY_BUILD_AI_SERVICE" = "true" ]; then
build_ai_with_recovery
@@ -508,6 +553,7 @@ else
echo "Skipping ai-service rebuild during deploy (set DEPLOY_BUILD_AI_SERVICE=true to rebuild it)."
fi
# Force recreation so updated port mappings, env vars, and container config always apply on deploy.
replacement_started=true
compose up -d --force-recreate --remove-orphans backend frontend
if [ "$DEPLOY_BUILD_AI_SERVICE" = "true" ]; then
# Ollama is opt-in (compose "bundled-ollama" profile). Deploys reuse an
@@ -539,10 +585,6 @@ fi
public_base="${APP_PUBLIC_BASE_URL%/}"
auth_config_body_file="$(mktemp)"
auth_config_headers_file="$(mktemp)"
cleanup_public_check() {
rm -f "$auth_config_body_file" "$auth_config_headers_file"
}
trap cleanup_public_check EXIT
echo "Running public smoke check against ${public_base}"
if ! curl -fsS "${public_base}/" >/dev/null; then
@@ -579,10 +621,8 @@ if ! grep -q 'requireAuth' "$auth_config_body_file"; then
exit 1
fi
trap - EXIT
cleanup_public_check
# Clean up old legacy container name if it still exists from pre-rename deployments.
docker rm -f app-summarizer-1 2>/dev/null || true
replacement_started=false
echo "Deployment complete: ${APP_VERSION} ${APP_COMMIT_SHA}"