From c094800c69ca203a0289ab39709824d338147e3c Mon Sep 17 00:00:00 2001 From: cesnimda Date: Sun, 30 Aug 2026 16:44:48 +0200 Subject: [PATCH] refactor(db): migrate user rule settings --- JobTrackerApi.Tests/MigrationChainTests.cs | 38 +++++++++++++ ...60830121000_AdoptUserRuleSettingsSchema.cs | 54 +++++++++++++++++++ .../StartupInitializationExtensions.cs | 35 ------------ .../Services/StartupSchemaOwnership.cs | 2 +- docs/audits/audit-remediation-backlog.md | 2 +- docs/audits/verification-log.md | 1 + docs/infrastructure/database-ownership.md | 7 ++- docs/verification/jt-019-schema-ownership.md | 7 ++- docs/work-programmes/master-progress.md | 3 +- 9 files changed, 107 insertions(+), 42 deletions(-) create mode 100644 JobTrackerApi/Migrations/20260830121000_AdoptUserRuleSettingsSchema.cs diff --git a/JobTrackerApi.Tests/MigrationChainTests.cs b/JobTrackerApi.Tests/MigrationChainTests.cs index b29550c..1b8c5fb 100644 --- a/JobTrackerApi.Tests/MigrationChainTests.cs +++ b/JobTrackerApi.Tests/MigrationChainTests.cs @@ -100,6 +100,7 @@ public sealed class MigrationChainTests Assert.Contains("CREATE TABLE IF NOT EXISTS `AspNetUsers`", script, StringComparison.Ordinal); Assert.Contains("CREATE TABLE IF NOT EXISTS `AiInteractions`", script, StringComparison.Ordinal); Assert.Contains("CREATE TABLE IF NOT EXISTS `SystemEmailSettings`", script, StringComparison.Ordinal); + Assert.Contains("CREATE TABLE IF NOT EXISTS `UserRuleSettings`", script, StringComparison.Ordinal); Assert.Contains("`UiLanguage` varchar(16)", script, StringComparison.Ordinal); Assert.All( Regex.Matches(script, "CONSTRAINT `([^`]+)`").Select(match => match.Groups[1].Value), @@ -148,6 +149,43 @@ public sealed class MigrationChainTests Assert.Empty(await db.Database.GetPendingMigrationsAsync()); } + [Fact] + public async Task User_rule_settings_adoption_preserves_legacy_rows() + { + await using var connection = new SqliteConnection("Data Source=:memory:"); + await connection.OpenAsync(); + await using var db = Context(connection); + var migrator = db.GetService(); + await migrator.MigrateAsync("20260830120000_AdoptSystemEmailSettingsSchema"); + await ExecuteAsync(connection, """ + CREATE TABLE "UserRuleSettings" ( + "OwnerUserId" TEXT NOT NULL CONSTRAINT "PK_UserRuleSettings" PRIMARY KEY, + "AppliedFollowUpDays" INTEGER NOT NULL, + "AppliedGhostDays" INTEGER NOT NULL, + "OfferFollowUpDays" INTEGER NOT NULL, + "OfferGhostDays" INTEGER NOT NULL, + "FeedbackFollowUpDays" INTEGER NOT NULL, + "FeedbackGhostDays" INTEGER NOT NULL + ); + INSERT INTO "UserRuleSettings" + ("OwnerUserId", "AppliedFollowUpDays", "AppliedGhostDays", "OfferFollowUpDays", + "OfferGhostDays", "FeedbackFollowUpDays", "FeedbackGhostDays") + VALUES ('owner-fixture', 3, 9, 4, 10, 5, 11); + """); + + await migrator.MigrateAsync(); + Assert.Equal(3L, await ScalarAsync(connection, + "SELECT AppliedFollowUpDays FROM UserRuleSettings WHERE OwnerUserId = 'owner-fixture';")); + + await migrator.MigrateAsync("20260830120000_AdoptSystemEmailSettingsSchema"); + Assert.Equal(3L, await ScalarAsync(connection, + "SELECT AppliedFollowUpDays FROM UserRuleSettings WHERE OwnerUserId = 'owner-fixture';")); + await migrator.MigrateAsync(); + Assert.Equal(3L, await ScalarAsync(connection, + "SELECT AppliedFollowUpDays FROM UserRuleSettings WHERE OwnerUserId = 'owner-fixture';")); + Assert.Empty(await db.Database.GetPendingMigrationsAsync()); + } + private static JobTrackerContext Context(SqliteConnection connection) { var currentUser = new Mock(); diff --git a/JobTrackerApi/Migrations/20260830121000_AdoptUserRuleSettingsSchema.cs b/JobTrackerApi/Migrations/20260830121000_AdoptUserRuleSettingsSchema.cs new file mode 100644 index 0000000..292f8fc --- /dev/null +++ b/JobTrackerApi/Migrations/20260830121000_AdoptUserRuleSettingsSchema.cs @@ -0,0 +1,54 @@ +using System; +using JobTrackerApi.Data; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace JobTrackerApi.Migrations; + +/// +/// Transfers creation ownership of per-user rule settings from startup reconciliation to the +/// migration chain without replacing tables created by older deployments. +/// +[DbContext(typeof(JobTrackerContext))] +[Migration("20260830121000_AdoptUserRuleSettingsSchema")] +public sealed class AdoptUserRuleSettingsSchema : Migration +{ + protected override void Up(MigrationBuilder migrationBuilder) + { + if (ActiveProvider.Contains("MySql", StringComparison.OrdinalIgnoreCase)) + { + migrationBuilder.Sql(""" + CREATE TABLE IF NOT EXISTS `UserRuleSettings` ( + `OwnerUserId` varchar(255) NOT NULL, + `AppliedFollowUpDays` int NOT NULL, + `AppliedGhostDays` int NOT NULL, + `OfferFollowUpDays` int NOT NULL, + `OfferGhostDays` int NOT NULL, + `FeedbackFollowUpDays` int NOT NULL, + `FeedbackGhostDays` int NOT NULL, + PRIMARY KEY (`OwnerUserId`) + ) CHARACTER SET=utf8mb4; + """); + return; + } + + migrationBuilder.Sql(""" + CREATE TABLE IF NOT EXISTS "UserRuleSettings" ( + "OwnerUserId" TEXT NOT NULL CONSTRAINT "PK_UserRuleSettings" PRIMARY KEY, + "AppliedFollowUpDays" INTEGER NOT NULL, + "AppliedGhostDays" INTEGER NOT NULL, + "OfferFollowUpDays" INTEGER NOT NULL, + "OfferGhostDays" INTEGER NOT NULL, + "FeedbackFollowUpDays" INTEGER NOT NULL, + "FeedbackGhostDays" INTEGER NOT NULL + ); + """); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + // Preserve settings that may have been created before migration ownership was introduced. + } +} diff --git a/JobTrackerApi/Services/StartupInitializationExtensions.cs b/JobTrackerApi/Services/StartupInitializationExtensions.cs index f8fe746..cf1441a 100644 --- a/JobTrackerApi/Services/StartupInitializationExtensions.cs +++ b/JobTrackerApi/Services/StartupInitializationExtensions.cs @@ -536,25 +536,6 @@ public static class StartupInitializationExtensions // UiLanguage is migration-owned (AddUiLanguagePreference). Adding it here before // the per-migration loop makes a fresh database fail when that migration runs. - static void EnsureUserRuleSettingsTable(DbConnection c) - { - if (HasTable(c, "UserRuleSettings")) return; - - Exec(c, """ - CREATE TABLE IF NOT EXISTS "UserRuleSettings" ( - "OwnerUserId" TEXT NOT NULL CONSTRAINT "PK_UserRuleSettings" PRIMARY KEY, - "AppliedFollowUpDays" INTEGER NOT NULL, - "AppliedGhostDays" INTEGER NOT NULL, - "OfferFollowUpDays" INTEGER NOT NULL, - "OfferGhostDays" INTEGER NOT NULL, - "FeedbackFollowUpDays" INTEGER NOT NULL, - "FeedbackGhostDays" INTEGER NOT NULL - ); - """); - } - - EnsureUserRuleSettingsTable(conn); - static void EnsureGmailConnectionsTable(DbConnection c) { Exec(c, """ @@ -1453,22 +1434,6 @@ public static class StartupInitializationExtensions seedRuleSettings.ExecuteNonQuery(); } - if (!HasMySqlTable(conn, "UserRuleSettings")) - { - using var cmd = conn.CreateCommand(); - cmd.CommandText = @"CREATE TABLE IF NOT EXISTS `UserRuleSettings` ( - `OwnerUserId` varchar(255) NOT NULL, - `AppliedFollowUpDays` int NOT NULL, - `AppliedGhostDays` int NOT NULL, - `OfferFollowUpDays` int NOT NULL, - `OfferGhostDays` int NOT NULL, - `FeedbackFollowUpDays` int NOT NULL, - `FeedbackGhostDays` int NOT NULL, - PRIMARY KEY (`OwnerUserId`) - );"; - cmd.ExecuteNonQuery(); - } - if (!HasMySqlTable(conn, "CvUploadArtifacts")) { using var cmd = conn.CreateCommand(); diff --git a/JobTrackerApi/Services/StartupSchemaOwnership.cs b/JobTrackerApi/Services/StartupSchemaOwnership.cs index b544feb..38231c0 100644 --- a/JobTrackerApi/Services/StartupSchemaOwnership.cs +++ b/JobTrackerApi/Services/StartupSchemaOwnership.cs @@ -24,6 +24,7 @@ internal static class StartupSchemaOwnership "SystemEmailSettings", "UserNotifications", "UserOperations", + "UserRuleSettings", }; internal static readonly IReadOnlySet ReconcilerOwnedTables = new HashSet(StringComparer.Ordinal) @@ -60,7 +61,6 @@ internal static class StartupSchemaOwnership "TailoredCvDrafts", "TrustedDevices", "TwoFactorRecoveryCodes", - "UserRuleSettings", "UserSessions", }; diff --git a/docs/audits/audit-remediation-backlog.md b/docs/audits/audit-remediation-backlog.md index 0b38304..40ad255 100644 --- a/docs/audits/audit-remediation-backlog.md +++ b/docs/audits/audit-remediation-backlog.md @@ -558,7 +558,7 @@ SEC-008 implements the same durable state machine with `.uploading` and ` ### P3-1 — Reduce dual schema ownership incrementally -**Status (2026-08-30): in progress.** All 49 model tables now have an executable, disjoint creation-owner classification. `SystemEmailSettings` is the first completed transfer: additive provider-aware migration, legacy-row preservation, startup-DDL removal, fresh SQLite runtime proof, generated MariaDB SQL, and backend 722/722. Thirty-four startup-created tables remain; see `docs/verification/jt-019-schema-ownership.md` and V-194. +**Status (2026-08-30): in progress.** All 49 model tables now have an executable, disjoint creation-owner classification. `SystemEmailSettings` and `UserRuleSettings` have moved through additive provider-aware migrations with legacy-row preservation, downgrade/retry safety, startup-DDL removal, fresh SQLite runtime proof and generated MariaDB SQL. Thirty-three startup-created tables remain; see `docs/verification/jt-019-schema-ownership.md` and V-194/V-195. - **Findings/scope:** JT-019; inventory reconciler operations and move stable schema ownership to EF migrations in small steps. - **Dependencies:** provider upgrade fixtures and P2-2 restore safety. diff --git a/docs/audits/verification-log.md b/docs/audits/verification-log.md index 8234eca..9436075 100644 --- a/docs/audits/verification-log.md +++ b/docs/audits/verification-log.md @@ -226,3 +226,4 @@ Output was reduced to filenames and commit counts. The token artifact appears un | V-192 | Exact SDK resolution; generated API/test NuGet locks; locked restore; full backend; transitive NuGet vulnerability audit | Repository root | Make .NET toolchain and transitive package resolution reproducible and fail CI on dependency drift | PASS — SDK 9.0.317 selected by `global.json`; both lock files restore in locked mode; backend 719/719; NuGet reports no known vulnerable direct or transitive package | The first test attempt observed the deliberately started local API holding the apphost, then passed after shutdown; the clean rerun passed without warnings. CI action SHAs, image digests, installer hashing, SBOM and container scanning remain JT-017 work | .NET provenance gap closed; broader build provenance remains partial | | V-193 | Clean `npm ci`; lint; focused/full Jest; Next build/TypeScript; locked NuGet restores; full backend; stale-claim/path inspection | Repository root / `job-tracker-ui` | Rebuild current developer/operator documentation and prove its setup, quality and provider claims | PASS — install succeeds without the obsolete peer override; lint zero warnings; CV Builder 12/12 and frontend 64 suites/272 tests; optimized build/TypeScript; locked restores and backend 719/719; every documented path exists | User-local SDK 9.0.317 is shadowed by a runtime-only system host on this workstation, so backend proof invoked the installed host explicitly. No second clean machine or production rehearsal | JT-018 repository documentation complete; external operator rehearsal remains deployment evidence | | V-194 | Executable 49-table ownership partition; focused ownership/migration chain; legacy-row upgrade; MariaDB script; full backend; fresh application SQLite startup | Repository root / disposable local SQLite | Establish one creation owner per model table and transfer the independent SystemEmailSettings table from startup DDL to EF migration ownership | PASS — ownership/migration 6/6; blank and repeated migration succeed; representative legacy SMTP row survives; provider SQL is present; backend 722/722; fresh runtime applies `20260830120000_AdoptSystemEmailSettingsSchema` and starts healthy | MariaDB SQL is generated but the new migration was not executed on a live MariaDB server; downgrade deliberately preserves ambiguous pre-migration data. Thirty-four reconciler-owned tables remain | JT-019 first migration-backed ownership transfer complete; provider runtime and remaining dependency groups continue incrementally | +| V-195 | Ownership/migration chain; populated adoption/downgrade/retry; MariaDB script; full backend | Repository root / disposable local SQLite | Transfer the independent UserRuleSettings table from both startup-DDL paths to migration ownership without losing per-user settings | PASS — focused ownership/migration 7/7; representative owner row survives adoption, downgrade and re-upgrade; generated MariaDB SQL contains provider-safe DDL; startup no longer contains the table create; full backend 723/723 | MariaDB SQL generated only; no production migration. Down preserves ambiguous legacy data by design. Thirty-three reconciler-owned tables remain | JT-019 second leaf transfer complete; dependency groups continue incrementally | diff --git a/docs/infrastructure/database-ownership.md b/docs/infrastructure/database-ownership.md index 72b67bc..aff59bf 100644 --- a/docs/infrastructure/database-ownership.md +++ b/docs/infrastructure/database-ownership.md @@ -68,12 +68,15 @@ Created by EF migrations, never by the reconciler: `AccountDeletionFiles`, `AccountDeletionRequests`, `AiUsageRecords`, `Attachments`, `Companies`, `Correspondences`, `EmailDrafts`, `EmailSendAttempts`, `JobApplications`, `JobEvents`, `Jobs`, -`RuleSettings`, `SystemEmailSettings`, `UserNotifications`, and `UserOperations`. +`RuleSettings`, `SystemEmailSettings`, `UserNotifications`, `UserOperations`, and `UserRuleSettings`. `SystemEmailSettings` is the first completed ownership transfer: migration `20260830120000_AdoptSystemEmailSettingsSchema` creates it for both supported providers and preserves an existing reconciler-created MariaDB table. Startup no longer creates it. +`UserRuleSettings` followed in `20260830121000_AdoptUserRuleSettingsSchema`; its owner-keyed rows are +preserved across adoption, downgrade and retry, and startup no longer creates it. + The reconciler may **repair** these (add a missing column, add an index, fix a non-`AUTO_INCREMENT` primary key) and may seed the default `RuleSettings` row — but it must never `CREATE TABLE` them. It used to create `RuleSettings`, which is precisely why a clean install failed with @@ -83,7 +86,7 @@ It used to create `RuleSettings`, which is precisely why a clean install failed Created by `StartupInitializationExtensions`, with a **no-op migration** holding the model snapshot: -`UserRuleSettings`, `CvUploadArtifacts`, `CvExtractionRuns`, +`CvUploadArtifacts`, `CvExtractionRuns`, `GmailConnections`, `MicrosoftGraphConnections`, `ImapConnections`, `TailoredCvDrafts`, `CareerProfiles`, `CareerProfileVersions`, the six CareerProfile children (`CareerExperiences`, `CareerEducations`, `CareerSkills`, `CareerProjects`, `CareerCertifications`, `CareerLanguages`), diff --git a/docs/verification/jt-019-schema-ownership.md b/docs/verification/jt-019-schema-ownership.md index 4d85173..b0bfaad 100644 --- a/docs/verification/jt-019-schema-ownership.md +++ b/docs/verification/jt-019-schema-ownership.md @@ -15,6 +15,8 @@ table. It does not attempt to delete the legacy reconciler wholesale. and MariaDB/MySQL. - Removed the MariaDB startup `CREATE TABLE` block for `SystemEmailSettings` and moved the table to the migration-owned set. +- Added `20260830121000_AdoptUserRuleSettingsSchema` and removed both provider startup-create paths + for the independent per-user rule-settings table. - Corrected the ownership runbook: new tables now default to migrations; the reconciler is legacy compatibility code to retire one dependency group at a time. @@ -32,14 +34,15 @@ migration. `SystemEmailSettings` columns. - A database stopped immediately before the adoption migration, seeded with a representative SMTP settings row, upgrades without changing that row. +- A representative per-user rules row survives adoption, downgrade, and re-upgrade. - Generated MariaDB SQL contains the provider-correct `SystemEmailSettings` DDL. -- Full backend: 722/722 passed. +- Full backend: 723/723 passed after both leaf-table transfers. - Fresh application startup over a new disposable SQLite database applied `20260830120000_AdoptSystemEmailSettingsSchema` and reached the healthy listening state. ## Remaining JT-019 work -Thirty-four model tables remain startup-created, including the Identity group and several tables +Thirty-three model tables remain startup-created, including the Identity group and several tables with parent dependencies. Transfer them in small dependency-aware migrations with blank, populated, retry and MariaDB runtime proof. Column/index repairs must remain until historical upgrade fixtures prove each one redundant. diff --git a/docs/work-programmes/master-progress.md b/docs/work-programmes/master-progress.md index 8784f33..c4865bf 100644 --- a/docs/work-programmes/master-progress.md +++ b/docs/work-programmes/master-progress.md @@ -44,6 +44,7 @@ Updated: 2026-08-30 - Pinned the repository to .NET SDK 9.0.317, generated content-hashed transitive NuGet locks for the API and test project, and made CI restores fail on lock drift. Locked restore and backend 719/719 pass; the current NuGet graph has no known vulnerable packages. - Rebuilt the active developer/operator documentation around the actual Next.js 16/.NET 9 application, replaced CRA and `npm start` guidance, separated normal and Playwright ports, corrected React Router 7 and the SQLite/MariaDB provider matrix, removed the obsolete npm peer override, and verified the documented clean install, lint, test, build and locked-restore commands. - Began the JT-019 schema-ownership retirement with an executable 49-table ownership partition and transferred the leaf `SystemEmailSettings` table from MariaDB-only startup DDL to an additive provider-aware migration. Fresh SQLite now receives the table; legacy rows are preserved and startup no longer creates it. +- Transferred the independent `UserRuleSettings` table from both provider startup paths to its own provider-aware migration; owner-keyed settings survive adoption, downgrade and retry. ### In progress @@ -121,7 +122,7 @@ Updated: 2026-08-30 - **Deferred work:** None. Conditional multi-replica coordination, model deletion, realtime operation delivery and unrelated production changes remain outside current packages. - **Immediate order:** all sixteen immediate repository items are complete locally, including the original UI/release queue plus SEC-009 cache/tombstone safety, worker restart clocks, universal AI accounting, email-token/Stripe lifecycle tests, exhaustive Job email selectors, the repaired migration chain, CV/public-edge hardening and measured admin/mail scaling. PROD-001 read-only evidence and the PROD-003 plan-only harness are also complete. The final audit is checking tooling/documentation before declaring only external blockers remain. - **Status counts:** 8 `VERIFIED LOCALLY`; 27 `IMPLEMENTED — NOT VERIFIED`; 0 `IN PROGRESS`; 0 `NOT STARTED`; 4 `BLOCKED`; 0 `DONE`; 0 `DEFERRED`. -- **Test status:** backend 722/722; frontend 64/64 suites and 272/272 tests; ESLint zero findings; AI sidecar 37/37 without warnings; Ollama benchmark harness 5/5 plus safe dry-run; optimized production build/TypeScript; EF model parity; SQLite/MariaDB migration scripts; blank/idempotent/populated SQLite migration-chain tests; fresh SQLite application startup through the SystemEmailSettings ownership migration; disposable fresh/restarted MariaDB 11.8 application startup; Docker Compose config; safe-failure deployment preflight; and Playwright 9/9. npm audit remains at zero. The new ownership migration has generated MariaDB SQL but still needs a MariaDB runtime rehearsal; the parser image build is pending because Docker Desktop's daemon is offline. Jest's slow/open-handle behavior remains recorded. +- **Test status:** backend 723/723; frontend 64/64 suites and 272/272 tests; ESLint zero findings; AI sidecar 37/37 without warnings; Ollama benchmark harness 5/5 plus safe dry-run; optimized production build/TypeScript; EF model parity; SQLite/MariaDB migration scripts; blank/idempotent/populated SQLite migration-chain tests; fresh SQLite application startup through the SystemEmailSettings ownership migration; disposable fresh/restarted MariaDB 11.8 application startup; Docker Compose config; safe-failure deployment preflight; and Playwright 9/9. npm audit remains at zero. The new ownership migrations have generated MariaDB SQL but still need a MariaDB runtime rehearsal; the parser image build is pending because Docker Desktop's daemon is offline. Jest's slow/open-handle behavior remains recorded. - **Deployment status:** Gitea pull-request run 609 passed the complete CI job in 4m20s. Deploy was intentionally skipped because the workflow deploys only a `push` to `main`; live remains unchanged. No merge/deployment was performed directly, no production migrations were run and the AI operation worker remains disabled by default. - **Production status:** State unchanged. Sanitized read-only SSH inventory was performed; no logs, prompts, private rows/content or secret values were read, and no provider/model call, model pull, service restart, file/config change, backup, restore, migration or deployment occurred. It confirmed all-interface Ollama/frontend listeners and stale database-only backups as rollout blockers. - **Known regressions:** None found by automated/local browser checks. Jest still needs `--forceExit` and reports its existing open-handle notice. Email-provider/send tests are fake/local only; real delivery is not claimed. Current MAIL browser evidence is 1280×720 only because the browser surface could not resize or perform native Tab traversal. Interrupted attempts are aged after 15 minutes and notified without retry; the five-minute scan is unmeasured on a large ledger. The direct clean EF-only SQLite defect and synchronous AI accounting gap are closed; migration/reconciler dual ownership remains architectural debt.