From a39ac847298e1fbe7667cf42671e7ec265500fcd Mon Sep 17 00:00:00 2001 From: cesnimda Date: Wed, 11 Nov 2026 07:19:45 +0100 Subject: [PATCH] fix(release): verify core health after automatic image rollback --- deploy/deploy.sh | 12 ++++++++++-- scripts/test-deploy-rollback.py | 31 +++++++++++++++++++++++++++---- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/deploy/deploy.sh b/deploy/deploy.sh index 84d6e8ef..dbe1921c 100644 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -660,10 +660,18 @@ finish_deploy() { if [ "$status" -ne 0 ] && [ "$replacement_started" = true ]; then if [ -n "$previous_backend_ref" ] && [ -n "$previous_frontend_ref" ]; then - echo "Deployment failed after core replacement; restoring previous healthy images." + echo "Deployment failed after core replacement; restarting previous core images." if BACKEND_IMAGE="$previous_backend_ref" FRONTEND_IMAGE="$previous_frontend_ref" \ compose up -d --force-recreate --remove-orphans backend frontend; then - echo "Previous core release restored. Deployment remains failed; inspect the logs above." + if curl -fsS --connect-timeout 5 --max-time 5 --retry 5 --retry-delay 2 --retry-connrefused --retry-max-time 30 "${APP_PUBLIC_BASE_URL%/}/" >/dev/null && + curl -fsS --connect-timeout 5 --max-time 5 --retry 5 --retry-delay 2 --retry-connrefused --retry-max-time 30 "${APP_PUBLIC_BASE_URL%/}/health" >/dev/null && + curl -fsS --connect-timeout 5 --max-time 5 --retry 5 --retry-delay 2 --retry-connrefused --retry-max-time 30 "${APP_PUBLIC_BASE_URL%/}/ready" >/dev/null; then + echo "Previous core images restarted and public core checks passed. Deployment remains failed; inspect the logs above." + else + echo "Previous core images restarted, but rollback health verification failed. Manual recovery is required." + compose ps || true + compose logs --tail=200 backend frontend || true + fi else echo "Automatic core rollback failed. Manual recovery is required." compose ps || true diff --git a/scripts/test-deploy-rollback.py b/scripts/test-deploy-rollback.py index b522ef6c..37bdcd1d 100644 --- a/scripts/test-deploy-rollback.py +++ b/scripts/test-deploy-rollback.py @@ -10,12 +10,14 @@ BASH = 'C:/Program Files/Git/bin/bash.exe' if os.name == 'nt' else shutil.which( class DeployRollbackTests(unittest.TestCase): - def exit_deploy(self, status, *, replaced=True, backend=True, frontend=True, compose_status=0): + def exit_deploy(self, status, *, replaced=True, backend=True, frontend=True, compose_status=0, failed_probe='', diagnostics_status=0): source = (ROOT / 'deploy/deploy.sh').read_text(encoding='utf-8') start = source.index('finish_deploy() {') handler = source[start:source.index('\ntrap finish_deploy EXIT\n', start) + len('\ntrap finish_deploy EXIT\n')] script = '\n'.join([ 'set -euo pipefail', + 'exec 3>&1', + 'APP_PUBLIC_BASE_URL=https://synthetic.invalid/', f'replacement_started={str(replaced).lower()}', 'previous_backend_ref=' + ('synthetic/backend:previous' if backend else ''), 'previous_frontend_ref=' + ('synthetic/frontend:previous' if frontend else ''), @@ -23,11 +25,15 @@ class DeployRollbackTests(unittest.TestCase): 'auth_config_headers_file=synthetic-headers', 'rm() { printf "CALL rm %s\\n" "$*"; }', 'docker() { echo "CALL unexpected docker"; return 99; }', - 'curl() { echo "CALL unexpected curl"; return 99; }', + 'curl() {', + ' printf "CALL curl %s\\n" "$*" >&3;', + f' if [ "${{@: -1}}" = "https://synthetic.invalid{failed_probe}" ]; then return 22; fi;', + ' return 0;', + '}', 'compose() {', ' printf "CALL compose %s [%s] [%s]\\n" "$*" "${BACKEND_IMAGE:-}" "${FRONTEND_IMAGE:-}";', f' if [ "$1" = up ]; then return {compose_status}; fi;', - ' return 0;', + f' return {diagnostics_status};', '}', handler, f'exit {status}', @@ -49,11 +55,28 @@ class DeployRollbackTests(unittest.TestCase): self.assertEqual([ 'CALL compose up -d --force-recreate --remove-orphans backend frontend ' '[synthetic/backend:previous] [synthetic/frontend:previous]', + *self.probes(), ], calls) + self.assertIn('public core checks passed', stdout) self.assertIn('Deployment remains failed', stdout) + @staticmethod + def probes(): + flags = '-fsS --connect-timeout 5 --max-time 5 --retry 5 --retry-delay 2 --retry-connrefused --retry-max-time 30' + return [f'CALL curl {flags} https://synthetic.invalid{path}' for path in ['/', '/health', '/ready']] + + def test_each_failed_health_probe_preserves_failure_and_requires_manual_recovery(self): + for index, path in enumerate(['/', '/health', '/ready']): + with self.subTest(path=path): + stdout, calls = self.exit_deploy(41, failed_probe=path, diagnostics_status=17) + self.assertEqual(self.probes()[:index + 1], calls[1:-2]) + self.assertEqual(['CALL compose ps [] []', 'CALL compose logs --tail=200 backend frontend [] []'], calls[-2:]) + self.assertIn('rollback health verification failed', stdout) + self.assertIn('Manual recovery is required', stdout) + self.assertNotIn('public core checks passed', stdout) + def test_rollback_failure_retains_original_status_and_collects_diagnostics(self): - stdout, calls = self.exit_deploy(41, compose_status=19) + stdout, calls = self.exit_deploy(41, compose_status=19, diagnostics_status=17) self.assertEqual([ 'CALL compose up -d --force-recreate --remove-orphans backend frontend ' '[synthetic/backend:previous] [synthetic/frontend:previous]',