fix(account): close deletion cache gap
CI and Deploy / test (pull_request) Successful in 5m22s
CI and Deploy / deploy (pull_request) Has been skipped

Require authenticated sidecar cache purge before a deletion can complete and keep failures retryable. Mount tombstones outside restored application data while leaving deletion disabled by default.
This commit is contained in:
cesnimda
2026-08-15 19:40:07 +02:00
parent c0e190d5b5
commit 7185491a05
14 changed files with 134 additions and 20 deletions
+3
View File
@@ -20,6 +20,9 @@ AUTH_ADMIN_PASSWORD=CHANGE_ME_STRONG_PASSWORD
AUTH_ALLOW_REGISTRATION=false
# Require local accounts to confirm ownership of their email address before signing in.
AUTH_REQUIRE_EMAIL_VERIFICATION=true
# Destructive account deletion stays dark-launched until the retention and restore runbook is
# approved and rehearsed. The tombstone path is mounted separately by docker-compose.yml.
ACCOUNT_DELETION_ENABLED=false
TURNSTILE_SITE_KEY=
TURNSTILE_SECRET_KEY=
# Optional hosted Stripe Checkout. Configure all three values and the customer portal before enabling billing.
+2 -2
View File
@@ -38,9 +38,9 @@ Updated: 2026-08-15
## Account deletion retention and restore policy
- **Blocked:** Enabling SEC-009 self-service deletion in production and declaring backup erasure complete.
- **Why:** The readable export and idempotent live-data deletion coordinator are implemented at `842e793` behind an explicit disabled gate. Repository code cannot truthfully choose legal retention periods, backup expiry, provider obligations or the tombstone lifetime needed to prevent restoration from resurrecting an erased account. Production inventory also confirms current backups are database-only and no protected tombstone volume exists yet.
- **Why:** The readable export and idempotent live-data deletion coordinator are implemented behind an explicit disabled gate. Repository code cannot truthfully choose legal retention periods, backup expiry, provider obligations or the tombstone lifetime needed to prevent restoration from resurrecting an erased account. Production inventory confirms current backups are database-only and no protected tombstone volume is deployed yet; the release branch now defines the separate volume and retryable authenticated sidecar-cache purge.
- **Required:** Decide retention periods for operational backups, audit/security records and deletion tombstones; identify any legal hold/export obligations; approve the restore behavior for deleted identities.
- **Recommended:** Keep production self-service and admin deletion disabled. Decide retention, mount the tombstone store outside restored application data, build a complete DB/files/keys backup set, then use a disposable account to prove provider/cache cleanup and restored-backup tombstone replay before staged activation.
- **Recommended:** Keep production self-service and admin deletion disabled. Decide retention, deploy/protect the configured tombstone volume, build a complete DB/files/keys backup set, then use a disposable account to prove remote-provider cleanup, sidecar purge across restart and restored-backup tombstone replay before staged activation.
## Legacy job/application column cutover
+39 -3
View File
@@ -152,6 +152,35 @@ public sealed class AccountDeletionTests
Assert.Equal(1, await fixture.Service.ProcessPendingAsync(CancellationToken.None));
Assert.False(await fixture.Db.Users.AsNoTracking().AnyAsync(item => item.Id == owner.Id));
Assert.Equal(2, (await fixture.Tombstones.ReadAsync(CancellationToken.None)).Count);
fixture.CachePurger.Verify(item => item.PurgeAsync(It.IsAny<CancellationToken>()), Times.Exactly(2));
});
}
[Fact]
public async Task Sidecar_cache_failure_keeps_deleted_account_retryable_until_purge_succeeds()
{
await InFixtureAsync(enabled: true, async fixture =>
{
fixture.Db.Users.Add(User("owner"));
await fixture.Db.SaveChangesAsync();
fixture.CachePurger.Setup(item => item.PurgeAsync(It.IsAny<CancellationToken>()))
.ThrowsAsync(new HttpRequestException("synthetic sidecar outage"));
var accepted = await fixture.Service.RequestAsync("owner", "owner", CancellationToken.None);
Assert.NotNull(accepted);
Assert.False(await fixture.Service.ProcessAsync(accepted!.RequestId, CancellationToken.None));
fixture.Db.ChangeTracker.Clear();
Assert.False(await fixture.Db.Users.AsNoTracking().AnyAsync(item => item.Id == "owner"));
var retry = await fixture.Db.AccountDeletionRequests.SingleAsync(item => item.Id == accepted.RequestId);
Assert.Equal(AccountDeletionRequestStatuses.RetryRequired, retry.Status);
Assert.Equal(AccountDeletionStages.PurgingFiles, retry.Stage);
Assert.Empty(await fixture.Tombstones.ReadAsync(CancellationToken.None));
fixture.CachePurger.Setup(item => item.PurgeAsync(It.IsAny<CancellationToken>()))
.Returns(Task.CompletedTask);
Assert.True(await fixture.Service.ProcessAsync(accepted.RequestId, CancellationToken.None));
Assert.Single(await fixture.Tombstones.ReadAsync(CancellationToken.None));
});
}
@@ -270,8 +299,10 @@ public sealed class AccountDeletionTests
using var cache = new MemoryCache(new MemoryCacheOptions());
var inventory = new AccountOwnedFileInventory(db, paths, new AttachmentStorage(paths));
var tombstones = new AccountDeletionTombstoneStore(paths);
var service = new AccountDeletionService(db, inventory, tombstones, configuration, cache, TimeProvider.System, NullLogger<AccountDeletionService>.Instance);
await test(new Fixture(db, paths, tombstones, service));
var cachePurger = new Mock<IAiSidecarCachePurger>();
cachePurger.Setup(item => item.PurgeAsync(It.IsAny<CancellationToken>())).Returns(Task.CompletedTask);
var service = new AccountDeletionService(db, inventory, tombstones, cachePurger.Object, configuration, cache, TimeProvider.System, NullLogger<AccountDeletionService>.Instance);
await test(new Fixture(db, paths, tombstones, cachePurger, service));
}
finally
{
@@ -280,5 +311,10 @@ public sealed class AccountDeletionTests
}
}
private sealed record Fixture(JobTrackerContext Db, AppPaths Paths, AccountDeletionTombstoneStore Tombstones, AccountDeletionService Service);
private sealed record Fixture(
JobTrackerContext Db,
AppPaths Paths,
AccountDeletionTombstoneStore Tombstones,
Mock<IAiSidecarCachePurger> CachePurger,
AccountDeletionService Service);
}
+1
View File
@@ -49,6 +49,7 @@ builder.Services.AddScoped<EmailSendAttemptStore>();
builder.Services.AddScoped<AccountOwnedFileInventory>();
builder.Services.AddScoped<AccountDataExportService>();
builder.Services.AddSingleton<AccountDeletionTombstoneStore>();
builder.Services.AddScoped<IAiSidecarCachePurger, AiSidecarCachePurger>();
builder.Services.AddScoped<AccountDeletionService>();
builder.Services.AddScoped<AiOperationAdmission>();
builder.Services.AddScoped<StrategySnapshotService>();
@@ -13,6 +13,7 @@ public sealed class AccountDeletionService(
JobTrackerContext db,
AccountOwnedFileInventory fileInventory,
AccountDeletionTombstoneStore tombstones,
IAiSidecarCachePurger aiSidecarCache,
IConfiguration configuration,
IMemoryCache memoryCache,
TimeProvider timeProvider,
@@ -313,7 +314,10 @@ public sealed class AccountDeletionService(
file.Status = "purged";
}
if (memoryCache is MemoryCache cache) cache.Compact(1.0);
AppendWarning(request, "The local AI sidecar cache is content-keyed and ages out under its configured TTL; production deletion remains disabled until cache purge/restart is rehearsed.");
// The sidecar cache is content-keyed rather than owner-keyed, so deletion clears it
// globally. Keep this inside the durable stage: an unavailable sidecar leaves the request
// retryable and prevents a false completion/tombstone acknowledgement.
await aiSidecarCache.PurgeAsync(cancellationToken);
request.Stage = AccountDeletionStages.RecordingTombstone;
await db.SaveChangesAsync(cancellationToken);
}
@@ -0,0 +1,16 @@
namespace JobTrackerApi.Services;
public interface IAiSidecarCachePurger
{
Task PurgeAsync(CancellationToken cancellationToken);
}
public sealed class AiSidecarCachePurger(IHttpClientFactory clients) : IAiSidecarCachePurger
{
public async Task PurgeAsync(CancellationToken cancellationToken)
{
using var response = await clients.CreateClient("ai-service")
.DeleteAsync("/maintenance/cache", cancellationToken);
response.EnsureSuccessStatusCode();
}
}
+6
View File
@@ -6,10 +6,15 @@ services:
dockerfile: JobTrackerApi/Dockerfile
volumes:
- jobtracker_data:/data
# Kept outside restored application data so an older database/data backup cannot erase the
# deletion ledger used to detect and re-delete resurrected accounts.
- jobtracker_deletion_tombstones:/account-lifecycle/tombstones
environment:
- ASPNETCORE_URLS=http://+:8080
- Data__Root=/data
- Exports__DailyFolder=/data/exports
- AccountLifecycle__DeletionEnabled=${ACCOUNT_DELETION_ENABLED:-false}
- AccountLifecycle__TombstonesRoot=/account-lifecycle/tombstones
- Database__Provider=${DATABASE_PROVIDER:-sqlite}
- ConnectionStrings__JobTracker=${JOBTRACKER_CONNECTION_STRING}
# If you enable HTTPS at a reverse proxy (recommended), handle redirects there.
@@ -230,6 +235,7 @@ services:
volumes:
jobtracker_data:
jobtracker_deletion_tombstones:
ollama_data:
networks:
+2
View File
@@ -209,3 +209,5 @@ Output was reduced to filenames and commit counts. The token artifact appears un
| V-175 | Account-deletion threat-path review; real-SQLite lifecycle/controller tests; focused/full backend/frontend; optimized build; EF parity and MariaDB script generation; disposable Chromium startup/application suite | Repository root / `job-tracker-ui` | Deliver a production-inert, owner-isolated, retryable live-account deletion lifecycle without implying backup/provider erasure | PASS — focused backend 21/21, backend 657/657, frontend focused 8/8 and full 58 suites/237 tests, optimized build, no pending EF changes, bounded MariaDB script and Chromium 9/9. Tests prove disabled gate, exact/recent confirmation, immediate lockout, idempotence, owner-isolated row/file purge, quarantine failure safety, tombstone creation and restored-account replay; `842e793` pushed | Synthetic local rows/files only. No live deletion, provider revocation, sidecar restart, backup restore, production migration or retention decision occurred. Self/admin requests remain disabled by default. Full every-provider/disposable-production rehearsal remains external | SEC-009 repository scope implemented; production activation remains blocked |
| V-176 | Authorized sanitized read-only SSH inventory of OS/CPU/RAM/GPU/storage/Docker/Ollama/app health/networks/selected non-secret configuration and backup metadata/integrity | Production host / repository report | Replace stale guessed hardware/access assumptions and define evidence-based rollout stops without changing production | PASS/PARTIAL — confirmed Ubuntu 24.04.4, i5-8600 6C, 31 GiB RAM, GTX 1060 6GB, 1.4 TiB Docker-volume headroom, Ollama 0.31.1 with `qwen2.5:7b`, four healthy app containers and exact deployed version. Also confirmed all-interface 11434/3000 listeners, unlimited container resources, direct Gemini selection in the old sidecar, dirty deploy-script mode, and 21 gzip-valid database-only backups ending 2026-08-02 | No secret values, logs, prompts, private rows/content, provider call, inference, model pull, file read beyond metadata/integrity, service restart, backup creation, restore or mutation. Internet/NAT exposure, logical restore and complete file/key recovery remain unverified | PROD-001 inventory/report complete; safety acceptance blocked by measured network/backup/deployment gaps |
| V-177 | Python unit tests and plan-only execution against the checked-in synthetic fixture; script/content/SSRF/report review | Repository root | Prepare reproducible privacy-safe PROD-003 evaluation without authorizing a model or network call | PASS — harness tests 4/4; plan validates four Strategy cases across 4K/8K for eight future requests. Default performs no HTTP; exact model, `--execute` and explicit output are required; model pull/delete is absent; report excludes raw fixture/prompt/output content | No Ollama request, provider/internet access, model metadata query through the harness, candidate pull, inference or production change. Candidate metadata/licenses/results and model decision remain unmeasured | PROD-003 repository harness complete; execution remains blocked |
| V-178 | Benchmark harness safety tests, plan-only execution and CI workflow inspection | Repository root | Prevent an approved private Ollama origin from escaping through proxy settings or redirects and make the boundary a release gate | PASS — 5/5 standard-library tests; proxy discovery is disabled, redirects are refused, plan-only output remains eight future Strategy requests, and CI now runs the suite without dependencies or network execution | No network, Ollama, provider, package or production call occurred | Benchmark request boundary corrected and CI-enforced |
| V-179 | Account-deletion real-SQLite failure/retry tests; sidecar token/cache tests; full backend; Compose validation | Repository root / `tools/summarizer` | Remove the live sidecar-cache and shared tombstone-path gaps without enabling deletion | PASS — lifecycle 6/6, backend 658/658, sidecar 23/23 and Compose config pass. Sidecar failure withholds completion/tombstone until retry; maintenance purge is token-protected; tombstones map to a separate named volume; activation defaults false | Synthetic rows/cache only; no production volume, deletion, restart, provider revocation, backup restore or retention decision | SEC-009 repository cache/storage boundary complete; production activation remains blocked |
+9 -4
View File
@@ -85,11 +85,12 @@ MYSQL_PWD='<pw>' mariadb --host=<host> --user=<user> --default-character-set=utf
## Production access
**This environment has no route to the production database** — no `/opt/job-tracker`, no production
connection string, and the local stack runs SQLite. Per the task constraints, **no credential
discovery and no SSH guessing were attempted.**
A sanitized read-only host inventory was completed on 2026-08-15. It verified the integrity of the
existing compressed database dumps without reading private rows or restoring data. The newest
observed dump was dated 2026-08-02, and the set is database-only: it does not prove recovery of owned
files, data-protection keys, configuration, or account-deletion tombstones.
**Manual step the owner must perform** (only the owner has production access):
**Authorized production steps still required:**
1. On the production host, run one out-of-band backup: `deploy/deploy.sh` takes one automatically, or
dump by hand with the command in `deploy/README.md`.
@@ -97,6 +98,10 @@ discovery and no SSH guessing were attempted.**
- table count (~42) and row counts for `AspNetUsers`, `JobApplications`, `Companies`,
`CareerProfiles` match production;
- a real record containing `æ`/`ø`/`å` reads back correctly (the check above).
3. Build and restore a complete recovery bundle covering the `jobtracker_data` volume and deployment
data-protection keys as well as MariaDB. Preserve the separately mounted
`jobtracker_deletion_tombstones` volume across application-data restores; never overwrite it with
an older backup capable of resurrecting a deleted identity.
Until that is done, backup/restore is proven **on the mechanism and on synthetic Norwegian data**, not
on the production dataset.
@@ -37,15 +37,18 @@ No existing generated file is moved or guessed. Legacy shared-date outputs stay
- Self-service requires an exact server-provided `DELETE <email>` phrase and a session created within 15 minutes. Last-administrator protection remains enforced. Admin deletion uses the same coordinator and exact-email confirmation header.
- One managed-root inventory covers attachments, CV artifacts, file-backed avatars, generated CVs, daily exports, and previously generated account-export ZIPs. Files move to same-volume quarantine markers before any database delete; partial file failure restores them and leaves rows untouched.
- Database deletion is explicit and transactional across all owned application, Career, CV, correspondence, provider-credential, queue/notification, security, and Identity rows. Request/file ledgers survive for retry and audit. Commit-acknowledgement ambiguity leaves files quarantined and replays deletion instead of risking data resurrection.
- Purge clears backend in-memory caches, removes quarantined files, then writes a minimal pseudonymous tombstone to a separate append-only JSONL root. Invalid ledger records fail closed.
- Purge clears backend in-memory caches, removes quarantined files, and calls the authenticated AI-sidecar maintenance endpoint before it can write a minimal pseudonymous tombstone. A sidecar failure leaves the durable request at `purging_files` for retry rather than falsely completing. Invalid ledger records fail closed.
- Production Compose maps the tombstone root to the separate `jobtracker_deletion_tombstones` named volume and exposes only the disabled-by-default `ACCOUNT_DELETION_ENABLED` switch. This is repository configuration evidence, not proof that the volume exists or is protected on production.
- Startup stages restored identities matching tombstones before readiness, and the background reconciler resumes all durable non-completed requests even while new deletion requests remain disabled.
- Settings explains the disabled production gate; when enabled it uses the reusable prompt dialog and exact phrase. Admin user deletion supplies the matching account email.
## Verification
- Owner-storage focused CV/export/controller/background tests: 77/77.
- Account lifecycle/export/auth/admin focused backend/API tests: 21/21. Five real-SQLite deletion tests cover disabled requests, exact confirmation/recent authentication, immediate lockout/idempotency, two-owner row/file isolation, quarantine failure, repeat reconciliation, tombstone creation, and restored-backup replay.
- Full backend: 657/657.
- Account lifecycle focused backend/API tests: 6/6. The new failure test proves a sidecar outage prevents completion/tombstone creation and a later retry succeeds. The broader lifecycle/export/auth/admin slice remains 21/21 from the prior checkpoint.
- Full backend: 658/658.
- AI sidecar: 23/23, including authenticated cache purge; cache access is lock-protected.
- Production Compose configuration: valid with a distinct tombstone volume and deletion disabled by default.
- Frontend export/Settings/admin tests: 8/8; full frontend 58 suites/237 tests.
- Backend build: pass, zero warnings/errors.
- Optimized frontend build/TypeScript: pass.
@@ -56,8 +59,8 @@ No existing generated file is moved or guessed. Legacy shared-date outputs stay
## Remaining external/production work
1. Decide backup, audit/security-log, quarantine, and tombstone retention plus any legal-hold obligations.
2. Mount/protect the tombstone root outside restored application data and rehearse a pre-deletion backup restore with tombstone replay.
3. Rehearse sidecar cache purge/restart and remote provider-revocation semantics using a disposable synthetic account.
2. Deploy and protect the separately configured tombstone volume, then rehearse a pre-deletion backup restore with tombstone replay.
3. Rehearse the implemented sidecar cache purge across container restart and define remote provider-revocation semantics using a disposable synthetic account.
4. Only then enable admin deletion, observe it, and separately approve self-service activation.
Production retention, legal hold and restored-backup decisions remain recorded in `BLOCKERS.md`.
+10
View File
@@ -769,3 +769,13 @@
- **Consequences:** candidate selection remains blocked and no model decision is claimed. Approved runs can compare 4K/8K timing, token rate, JSON/constraint quality and Ollama VRAM metadata reproducibly while host GPU/RAM metrics are captured separately.
- **User approval required:** Yes before any model pull, production inference, or production report execution. No approval is required for plan-only fixture validation.
- **Reversible:** Remove the script/tests/template; no dependency, model, application, production, schema or configuration state changed.
## DEC-078 — Make cache erasure a durable deletion stage
- **Date:** 2026-08-15
- **Decision:** Clear the content-keyed AI-sidecar cache through an authenticated maintenance endpoint inside the existing `purging_files` stage. Treat sidecar failure as retryable and withhold the deletion tombstone/completion acknowledgement until purge succeeds. Mount tombstones on a separate named Compose volume while keeping new deletion requests disabled by default.
- **Reason/evidence:** Expiry after one hour did not satisfy immediate live-cache erasure, and the default tombstone path shared `/data` with the application restore boundary. Backend 6/6 and sidecar 23/23 prove failure/retry and token-protected purge; Compose validation proves the separate repository configuration.
- **Alternatives considered:** wait for TTL; restart the whole sidecar; remove caching; allow deletion to complete with a warning. These retain deleted content temporarily, disrupt unrelated work, regress performance, or falsely acknowledge completion.
- **Consequences:** a rare account deletion globally clears the shared summary cache because entries are not owner-keyed. An unavailable sidecar delays completion but does not restore deleted live rows/files. Production still needs protected volume custody, provider semantics, retention decisions and a disposable restore rehearsal.
- **User approval required:** Production activation and operational mutation only. Repository implementation remains inert behind `ACCOUNT_DELETION_ENABLED=false`.
- **Reversible:** Keep deletion disabled, drain any in-flight request, then revert the endpoint/client and Compose volume mapping. Never remove a deployed tombstone ledger while older restorable backups exist.
+2 -2
View File
@@ -270,9 +270,9 @@ This queue records the highest-value work that can proceed without production cr
- **Required production verification:** backup retention/tombstone rehearsal before self-service enablement.
- **Status:** `IMPLEMENTED — NOT VERIFIED`.
- **Blocker:** legal/operator retention and production restore decisions block activation, not the repository-side disabled/dark launch.
- **Evidence:** audit JT-009 inventory/design; `docs/verification/sec-009-account-lifecycle.md`; V-173/V-174/V-175. Owner-scoped generated storage, complete redacted readable ZIP, immediate lockout, transactional owner-isolated row/file purge, retry, fail-closed tombstones and restored-backup replay pass real-SQLite, focused API/UI, full backend/frontend, build and Chromium checks.
- **Evidence:** audit JT-009 inventory/design; `docs/verification/sec-009-account-lifecycle.md`; V-173/V-174/V-175/V-179. Owner-scoped generated storage, complete redacted readable ZIP, immediate lockout, transactional owner-isolated row/file/cache purge, retry, fail-closed tombstones and restored-backup replay pass real-SQLite, sidecar, focused API/UI, full backend/frontend, Compose, build and Chromium checks.
- **Commit:** `842e793`.
- **Remaining work:** repository scope is complete. Production activation remains blocked by retention/legal decisions, protected tombstone custody, restored-backup rehearsal, provider/cache semantics and staged disposable-account rollout.
- **Remaining work:** repository scope is complete, including retryable authenticated sidecar-cache purge and a separately mounted Compose tombstone volume. Production activation remains blocked by retention/legal decisions, deployed/protected tombstone custody, complete restored-backup rehearsal, remote-provider semantics and staged disposable-account rollout.
### CORE-001 — Restore default SQLite/MariaDB behavior parity
+15 -3
View File
@@ -175,6 +175,7 @@ if EAGER_MODEL_LOAD and not SKIP_MODEL_LOAD:
_ensure_runtime_loaded()
cache = TTLCache(maxsize=1024, ttl=60 * 60)
cache_lock = threading.Lock()
class SummarizeRequest(BaseModel):
@@ -959,8 +960,10 @@ async def summarize(req: SummarizeRequest):
raise HTTPException(status_code=400, detail="min_length must be smaller than max_length.")
key = _key(req.text, req.max_length, req.min_length, req.top_skills)
if key in cache:
return {"summary": cache[key], "cached": True}
with cache_lock:
cached_summary = cache.get(key)
if cached_summary is not None:
return {"summary": cached_summary, "cached": True}
info = _role_focused_excerpt(req.text)
summary = _model_summarize(info["focused_input"], req.max_length, req.min_length)
@@ -1028,10 +1031,19 @@ async def summarize(req: SummarizeRequest):
lines.append("- Prepare examples showing relevant impact, collaboration, and delivery.")
out = "\n".join(lines).strip()
cache[key] = out
with cache_lock:
cache[key] = out
return {"summary": out, "cached": False}
@app.delete("/maintenance/cache")
async def purge_cache():
with cache_lock:
cleared = len(cache)
cache.clear()
return {"cleared": cleared}
def _normalize_text(value: str) -> str:
value = value.replace("\x00", " ")
return re.sub(r"\s+", " ", value).strip()
+16
View File
@@ -510,6 +510,22 @@ def test_health_stays_open_so_probes_and_healthchecks_work(monkeypatch):
assert client.get("/health").status_code == 200
def test_cache_purge_requires_service_token_and_clears_content(monkeypatch):
module = load_app_module(monkeypatch, service_token="s3cret")
module.cache["synthetic-key"] = "synthetic-summary"
client = TestClient(module.app)
assert client.delete("/maintenance/cache").status_code == 401
response = client.delete(
"/maintenance/cache",
headers={"X-Ai-Service-Token": "s3cret"},
)
assert response.status_code == 200
assert response.json() == {"cleared": 1}
assert len(module.cache) == 0
def test_endpoints_stay_open_when_no_token_is_configured(monkeypatch):
module = load_app_module(monkeypatch)
client = TestClient(module.app)