From b425f1edc5b95bba68035cfbe5041e1b781e1d9f Mon Sep 17 00:00:00 2001 From: cesnimda Date: Sun, 30 Aug 2026 16:48:32 +0200 Subject: [PATCH] refactor(db): migrate Gmail review decisions Move creation ownership from startup reconciliation to an additive provider-aware migration. Preserve legacy decisions during downgrade and add the previously missing MariaDB table path. --- JobTrackerApi.Tests/MigrationChainTests.cs | 37 +++++++++++++ ...0122000_AdoptGmailReviewDecisionsSchema.cs | 54 +++++++++++++++++++ .../StartupInitializationExtensions.cs | 12 ----- .../Services/StartupSchemaOwnership.cs | 2 +- docs/audits/audit-remediation-backlog.md | 2 +- docs/audits/verification-log.md | 1 + docs/infrastructure/database-ownership.md | 5 +- docs/verification/jt-019-schema-ownership.md | 7 ++- docs/work-programmes/master-progress.md | 4 +- 9 files changed, 106 insertions(+), 18 deletions(-) create mode 100644 JobTrackerApi/Migrations/20260830122000_AdoptGmailReviewDecisionsSchema.cs diff --git a/JobTrackerApi.Tests/MigrationChainTests.cs b/JobTrackerApi.Tests/MigrationChainTests.cs index 1b8c5fb..335420a 100644 --- a/JobTrackerApi.Tests/MigrationChainTests.cs +++ b/JobTrackerApi.Tests/MigrationChainTests.cs @@ -101,6 +101,7 @@ public sealed class MigrationChainTests Assert.Contains("CREATE TABLE IF NOT EXISTS `AiInteractions`", script, StringComparison.Ordinal); Assert.Contains("CREATE TABLE IF NOT EXISTS `SystemEmailSettings`", script, StringComparison.Ordinal); Assert.Contains("CREATE TABLE IF NOT EXISTS `UserRuleSettings`", script, StringComparison.Ordinal); + Assert.Contains("CREATE TABLE IF NOT EXISTS `GmailReviewDecisions`", script, StringComparison.Ordinal); Assert.Contains("`UiLanguage` varchar(16)", script, StringComparison.Ordinal); Assert.All( Regex.Matches(script, "CONSTRAINT `([^`]+)`").Select(match => match.Groups[1].Value), @@ -186,6 +187,42 @@ public sealed class MigrationChainTests Assert.Empty(await db.Database.GetPendingMigrationsAsync()); } + [Fact] + public async Task Gmail_review_decision_adoption_preserves_legacy_rows() + { + await using var connection = new SqliteConnection("Data Source=:memory:"); + await connection.OpenAsync(); + await using var db = Context(connection); + var migrator = db.GetService(); + await migrator.MigrateAsync("20260830121000_AdoptUserRuleSettingsSchema"); + await ExecuteAsync(connection, """ + CREATE TABLE "GmailReviewDecisions" ( + "Id" INTEGER NOT NULL CONSTRAINT "PK_GmailReviewDecisions" PRIMARY KEY AUTOINCREMENT, + "OwnerUserId" TEXT NOT NULL, + "ThreadId" TEXT NOT NULL, + "JobApplicationId" INTEGER NULL, + "Decision" TEXT NOT NULL, + "Note" TEXT NULL, + "UpdatedAt" TEXT NOT NULL + ); + INSERT INTO "GmailReviewDecisions" + ("OwnerUserId", "ThreadId", "JobApplicationId", "Decision", "Note", "UpdatedAt") + VALUES ('owner-fixture', 'thread-fixture', NULL, 'linked', 'Preserve me', '2026-08-30T09:00:00+00:00'); + """); + + await migrator.MigrateAsync(); + Assert.Equal("linked", await ScalarAsync(connection, + "SELECT Decision FROM GmailReviewDecisions WHERE ThreadId = 'thread-fixture';")); + + await migrator.MigrateAsync("20260830121000_AdoptUserRuleSettingsSchema"); + Assert.Equal("Preserve me", await ScalarAsync(connection, + "SELECT Note FROM GmailReviewDecisions WHERE ThreadId = 'thread-fixture';")); + await migrator.MigrateAsync(); + Assert.Equal("Preserve me", await ScalarAsync(connection, + "SELECT Note FROM GmailReviewDecisions WHERE ThreadId = 'thread-fixture';")); + Assert.Empty(await db.Database.GetPendingMigrationsAsync()); + } + private static JobTrackerContext Context(SqliteConnection connection) { var currentUser = new Mock(); diff --git a/JobTrackerApi/Migrations/20260830122000_AdoptGmailReviewDecisionsSchema.cs b/JobTrackerApi/Migrations/20260830122000_AdoptGmailReviewDecisionsSchema.cs new file mode 100644 index 0000000..949882a --- /dev/null +++ b/JobTrackerApi/Migrations/20260830122000_AdoptGmailReviewDecisionsSchema.cs @@ -0,0 +1,54 @@ +using System; +using JobTrackerApi.Data; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace JobTrackerApi.Migrations; + +/// +/// Moves Gmail review-decision creation into the migration chain and supplies the previously +/// missing MariaDB table while retaining SQLite rows created by startup reconciliation. +/// +[DbContext(typeof(JobTrackerContext))] +[Migration("20260830122000_AdoptGmailReviewDecisionsSchema")] +public sealed class AdoptGmailReviewDecisionsSchema : Migration +{ + protected override void Up(MigrationBuilder migrationBuilder) + { + if (ActiveProvider.Contains("MySql", StringComparison.OrdinalIgnoreCase)) + { + migrationBuilder.Sql(""" + CREATE TABLE IF NOT EXISTS `GmailReviewDecisions` ( + `Id` int NOT NULL AUTO_INCREMENT, + `OwnerUserId` varchar(255) NOT NULL, + `ThreadId` varchar(512) NOT NULL, + `JobApplicationId` int NULL, + `Decision` varchar(32) NOT NULL, + `Note` longtext NULL, + `UpdatedAt` datetime(6) NOT NULL, + PRIMARY KEY (`Id`) + ) CHARACTER SET=utf8mb4; + """); + return; + } + + migrationBuilder.Sql(""" + CREATE TABLE IF NOT EXISTS "GmailReviewDecisions" ( + "Id" INTEGER NOT NULL CONSTRAINT "PK_GmailReviewDecisions" PRIMARY KEY AUTOINCREMENT, + "OwnerUserId" TEXT NOT NULL, + "ThreadId" TEXT NOT NULL, + "JobApplicationId" INTEGER NULL, + "Decision" TEXT NOT NULL, + "Note" TEXT NULL, + "UpdatedAt" TEXT NOT NULL + ); + """); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + // Preserve decisions that may have been created before migration ownership was introduced. + } +} diff --git a/JobTrackerApi/Services/StartupInitializationExtensions.cs b/JobTrackerApi/Services/StartupInitializationExtensions.cs index cf1441a..b7c3b8a 100644 --- a/JobTrackerApi/Services/StartupInitializationExtensions.cs +++ b/JobTrackerApi/Services/StartupInitializationExtensions.cs @@ -556,18 +556,6 @@ public static class StartupInitializationExtensions "LastSyncStatus" TEXT NULL, "LastSyncError" TEXT NULL ); - """); - - Exec(c, """ - CREATE TABLE IF NOT EXISTS "GmailReviewDecisions" ( - "Id" INTEGER NOT NULL CONSTRAINT "PK_GmailReviewDecisions" PRIMARY KEY AUTOINCREMENT, - "OwnerUserId" TEXT NOT NULL, - "ThreadId" TEXT NOT NULL, - "JobApplicationId" INTEGER NULL, - "Decision" TEXT NOT NULL, - "Note" TEXT NULL, - "UpdatedAt" TEXT NOT NULL - ); """); EnsureColumn(c, "GmailConnections", "LastSyncAttemptedAt", "ALTER TABLE GmailConnections ADD COLUMN LastSyncAttemptedAt TEXT NULL;"); diff --git a/JobTrackerApi/Services/StartupSchemaOwnership.cs b/JobTrackerApi/Services/StartupSchemaOwnership.cs index 38231c0..3b0a2e0 100644 --- a/JobTrackerApi/Services/StartupSchemaOwnership.cs +++ b/JobTrackerApi/Services/StartupSchemaOwnership.cs @@ -17,6 +17,7 @@ internal static class StartupSchemaOwnership "Correspondences", "EmailDrafts", "EmailSendAttempts", + "GmailReviewDecisions", "JobApplications", "JobEvents", "Jobs", @@ -53,7 +54,6 @@ internal static class StartupSchemaOwnership "CvVariants", "CvVariantVersions", "GmailConnections", - "GmailReviewDecisions", "ImapConnections", "InterviewPrepItems", "InterviewPrepNotes", diff --git a/docs/audits/audit-remediation-backlog.md b/docs/audits/audit-remediation-backlog.md index 40ad255..587594b 100644 --- a/docs/audits/audit-remediation-backlog.md +++ b/docs/audits/audit-remediation-backlog.md @@ -558,7 +558,7 @@ SEC-008 implements the same durable state machine with `.uploading` and ` ### P3-1 — Reduce dual schema ownership incrementally -**Status (2026-08-30): in progress.** All 49 model tables now have an executable, disjoint creation-owner classification. `SystemEmailSettings` and `UserRuleSettings` have moved through additive provider-aware migrations with legacy-row preservation, downgrade/retry safety, startup-DDL removal, fresh SQLite runtime proof and generated MariaDB SQL. Thirty-three startup-created tables remain; see `docs/verification/jt-019-schema-ownership.md` and V-194/V-195. +**Status (2026-08-30): in progress.** All 49 model tables now have an executable, disjoint creation-owner classification. `SystemEmailSettings`, `UserRuleSettings`, and `GmailReviewDecisions` have moved through additive provider-aware migrations with legacy-row preservation, downgrade/retry safety, startup-DDL removal, fresh SQLite runtime proof and generated MariaDB SQL. The Gmail decision migration also closes its missing MariaDB creation path. Thirty-two startup-created tables remain; see `docs/verification/jt-019-schema-ownership.md` and V-194–V-196. - **Findings/scope:** JT-019; inventory reconciler operations and move stable schema ownership to EF migrations in small steps. - **Dependencies:** provider upgrade fixtures and P2-2 restore safety. diff --git a/docs/audits/verification-log.md b/docs/audits/verification-log.md index 9436075..e07dd09 100644 --- a/docs/audits/verification-log.md +++ b/docs/audits/verification-log.md @@ -227,3 +227,4 @@ Output was reduced to filenames and commit counts. The token artifact appears un | V-193 | Clean `npm ci`; lint; focused/full Jest; Next build/TypeScript; locked NuGet restores; full backend; stale-claim/path inspection | Repository root / `job-tracker-ui` | Rebuild current developer/operator documentation and prove its setup, quality and provider claims | PASS — install succeeds without the obsolete peer override; lint zero warnings; CV Builder 12/12 and frontend 64 suites/272 tests; optimized build/TypeScript; locked restores and backend 719/719; every documented path exists | User-local SDK 9.0.317 is shadowed by a runtime-only system host on this workstation, so backend proof invoked the installed host explicitly. No second clean machine or production rehearsal | JT-018 repository documentation complete; external operator rehearsal remains deployment evidence | | V-194 | Executable 49-table ownership partition; focused ownership/migration chain; legacy-row upgrade; MariaDB script; full backend; fresh application SQLite startup | Repository root / disposable local SQLite | Establish one creation owner per model table and transfer the independent SystemEmailSettings table from startup DDL to EF migration ownership | PASS — ownership/migration 6/6; blank and repeated migration succeed; representative legacy SMTP row survives; provider SQL is present; backend 722/722; fresh runtime applies `20260830120000_AdoptSystemEmailSettingsSchema` and starts healthy | MariaDB SQL is generated but the new migration was not executed on a live MariaDB server; downgrade deliberately preserves ambiguous pre-migration data. Thirty-four reconciler-owned tables remain | JT-019 first migration-backed ownership transfer complete; provider runtime and remaining dependency groups continue incrementally | | V-195 | Ownership/migration chain; populated adoption/downgrade/retry; MariaDB script; full backend | Repository root / disposable local SQLite | Transfer the independent UserRuleSettings table from both startup-DDL paths to migration ownership without losing per-user settings | PASS — focused ownership/migration 7/7; representative owner row survives adoption, downgrade and re-upgrade; generated MariaDB SQL contains provider-safe DDL; startup no longer contains the table create; full backend 723/723 | MariaDB SQL generated only; no production migration. Down preserves ambiguous legacy data by design. Thirty-three reconciler-owned tables remain | JT-019 second leaf transfer complete; dependency groups continue incrementally | +| V-196 | Ownership/migration chain; populated adoption/downgrade/retry; MariaDB script; full backend | Repository root / disposable local SQLite | Move GmailReviewDecisions to migration ownership and close the missing MariaDB table path | PASS — focused ownership/migration 8/8; representative decision survives adoption, downgrade and re-upgrade; generated MariaDB SQL contains provider-safe DDL; SQLite startup create is removed; full backend 724/724 | MariaDB SQL generated only; no provider account or production migration. Down preserves ambiguous legacy data by design. Thirty-two reconciler-owned tables remain | JT-019 third leaf transfer complete; provider runtime and dependency groups continue incrementally | diff --git a/docs/infrastructure/database-ownership.md b/docs/infrastructure/database-ownership.md index aff59bf..7d6a8e6 100644 --- a/docs/infrastructure/database-ownership.md +++ b/docs/infrastructure/database-ownership.md @@ -67,7 +67,7 @@ already-correct database. Two consequences worth knowing: Created by EF migrations, never by the reconciler: `AccountDeletionFiles`, `AccountDeletionRequests`, `AiUsageRecords`, `Attachments`, `Companies`, -`Correspondences`, `EmailDrafts`, `EmailSendAttempts`, `JobApplications`, `JobEvents`, `Jobs`, +`Correspondences`, `EmailDrafts`, `EmailSendAttempts`, `GmailReviewDecisions`, `JobApplications`, `JobEvents`, `Jobs`, `RuleSettings`, `SystemEmailSettings`, `UserNotifications`, `UserOperations`, and `UserRuleSettings`. `SystemEmailSettings` is the first completed ownership transfer: migration @@ -77,6 +77,9 @@ an existing reconciler-created MariaDB table. Startup no longer creates it. `UserRuleSettings` followed in `20260830121000_AdoptUserRuleSettingsSchema`; its owner-keyed rows are preserved across adoption, downgrade and retry, and startup no longer creates it. +`GmailReviewDecisions` moved in `20260830122000_AdoptGmailReviewDecisionsSchema`. This also supplies +the table on MariaDB, where the old reconciler had no creation path. + The reconciler may **repair** these (add a missing column, add an index, fix a non-`AUTO_INCREMENT` primary key) and may seed the default `RuleSettings` row — but it must never `CREATE TABLE` them. It used to create `RuleSettings`, which is precisely why a clean install failed with diff --git a/docs/verification/jt-019-schema-ownership.md b/docs/verification/jt-019-schema-ownership.md index b0bfaad..77239a2 100644 --- a/docs/verification/jt-019-schema-ownership.md +++ b/docs/verification/jt-019-schema-ownership.md @@ -17,6 +17,8 @@ table. It does not attempt to delete the legacy reconciler wholesale. the migration-owned set. - Added `20260830121000_AdoptUserRuleSettingsSchema` and removed both provider startup-create paths for the independent per-user rule-settings table. +- Added `20260830122000_AdoptGmailReviewDecisionsSchema`, removed the SQLite startup-create path, + and supplied the previously absent MariaDB table definition. - Corrected the ownership runbook: new tables now default to migrations; the reconciler is legacy compatibility code to retire one dependency group at a time. @@ -35,14 +37,15 @@ migration. - A database stopped immediately before the adoption migration, seeded with a representative SMTP settings row, upgrades without changing that row. - A representative per-user rules row survives adoption, downgrade, and re-upgrade. +- A representative Gmail review decision survives adoption, downgrade, and re-upgrade. - Generated MariaDB SQL contains the provider-correct `SystemEmailSettings` DDL. -- Full backend: 723/723 passed after both leaf-table transfers. +- Full backend: 724/724 passed after all three leaf-table transfers. - Fresh application startup over a new disposable SQLite database applied `20260830120000_AdoptSystemEmailSettingsSchema` and reached the healthy listening state. ## Remaining JT-019 work -Thirty-three model tables remain startup-created, including the Identity group and several tables +Thirty-two model tables remain startup-created, including the Identity group and several tables with parent dependencies. Transfer them in small dependency-aware migrations with blank, populated, retry and MariaDB runtime proof. Column/index repairs must remain until historical upgrade fixtures prove each one redundant. diff --git a/docs/work-programmes/master-progress.md b/docs/work-programmes/master-progress.md index c4865bf..5d83dae 100644 --- a/docs/work-programmes/master-progress.md +++ b/docs/work-programmes/master-progress.md @@ -45,6 +45,7 @@ Updated: 2026-08-30 - Rebuilt the active developer/operator documentation around the actual Next.js 16/.NET 9 application, replaced CRA and `npm start` guidance, separated normal and Playwright ports, corrected React Router 7 and the SQLite/MariaDB provider matrix, removed the obsolete npm peer override, and verified the documented clean install, lint, test, build and locked-restore commands. - Began the JT-019 schema-ownership retirement with an executable 49-table ownership partition and transferred the leaf `SystemEmailSettings` table from MariaDB-only startup DDL to an additive provider-aware migration. Fresh SQLite now receives the table; legacy rows are preserved and startup no longer creates it. - Transferred the independent `UserRuleSettings` table from both provider startup paths to its own provider-aware migration; owner-keyed settings survive adoption, downgrade and retry. +- Moved `GmailReviewDecisions` into a provider-aware migration, preserving existing SQLite decisions and closing the previously missing MariaDB table path. ### In progress @@ -73,8 +74,9 @@ Updated: 2026-08-30 ### Verification - Focused frontend: 2 suites, 6 tests passed. -- Full frontend: 62 suites, 256 tests passed. +- Full frontend: 64 suites, 272 tests passed. - Next production build and TypeScript: passed. +- Full backend: 724/724 tests passed after the third JT-019 ownership transfer. - Portable Playwright launcher: resolved the user-local .NET 9 SDK; backend Release build passed with 0 warnings/errors. - Playwright: initial full run 9/10 exposed the intentional mobile Settings control change; updated focused rerun passed 1/1. A final complete browser rerun remains in the end-of-batch gate. - Focused backend match/intelligence verification: 34/34 passed.