From 2b26953d6f3c3fdacdffefa2c0a696c9084f1afd Mon Sep 17 00:00:00 2001 From: cesnimda Date: Sat, 24 Oct 2026 14:00:26 +0200 Subject: [PATCH] fix(db): normalize document index names across MariaDB collations --- .../DocumentsIndexTests.cs | 172 ++++++++++++++++++ .../Services/DocumentsIndexService.cs | 16 +- 2 files changed, 181 insertions(+), 7 deletions(-) create mode 100644 JobTrackerApi.MariaDb.Tests/DocumentsIndexTests.cs diff --git a/JobTrackerApi.MariaDb.Tests/DocumentsIndexTests.cs b/JobTrackerApi.MariaDb.Tests/DocumentsIndexTests.cs new file mode 100644 index 0000000..e18c662 --- /dev/null +++ b/JobTrackerApi.MariaDb.Tests/DocumentsIndexTests.cs @@ -0,0 +1,172 @@ +using System.Text.Json; +using JobTrackerApi.Data; +using JobTrackerApi.Models; +using JobTrackerApi.Services; +using Microsoft.EntityFrameworkCore; +using Xunit; + +namespace JobTrackerApi.MariaDb.Tests; + +public sealed class DocumentsIndexTests +{ + private const string Owner = "rd21-owner"; + private const string Other = "documents-other"; + private const string PrivateContent = "PRIVATE_DOCUMENT_CONTENT"; + + [Fact] + public async Task Startup_mixed_collations_support_all_sources_counts_pagination_and_read_only_metadata() + { + await using var database = await MariaDbDatabase.CreateAsync(); + await SeedAsync(database); + await using var db = database.Context(); + var collations = await db.Database.SqlQueryRaw(""" + SELECT DISTINCT COLLATION_NAME AS Value FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND + ((TABLE_NAME = 'CvVariants' AND COLUMN_NAME = 'Name') + OR (TABLE_NAME = 'JobApplications' AND COLUMN_NAME = 'JobTitle') + OR (TABLE_NAME = 'Attachments' AND COLUMN_NAME = 'FileName') + OR (TABLE_NAME = 'SubmittedApplicationPackages' AND COLUMN_NAME = 'CvVariantName') + OR (TABLE_NAME = 'EmailDrafts' AND COLUMN_NAME = 'Subject')) + """).ToListAsync(); + Assert.True(collations.Count > 1, "The startup-created schema must reproduce mixed source collations."); + var before = await SnapshotAsync(database); + var all = await GetAsync(db); + Assert.Equal(7, all.Total); + Assert.Equal(new[] { "application-material", "attachment", "cover-letter", "cv", "email-draft", "submitted-package", "tailored-cv" }, + all.Items.Select(item => item.Type).Order()); + var first = await GetAsync(db, pageSize: 3); + var second = await GetAsync(db, page: 2, pageSize: 3); + var third = await GetAsync(db, page: 3, pageSize: 3); + Assert.Equal(7, first.Total); + Assert.Equal(3, first.TotalPages); + Assert.Equal(new[] { 3, 3, 1 }, new[] { first.Items.Count, second.Items.Count, third.Items.Count }); + Assert.Equal(all.Items.Select(item => item.Id), first.Items.Concat(second.Items).Concat(third.Items).Select(item => item.Id)); + var letter = Assert.Single(all.Items, item => item.Type == "cover-letter"); + Assert.Equal(1, letter.HistoryCount); + Assert.Equal(2L, letter.Version); + Assert.True(letter.HasCurrentContent); + Assert.Equal("2026-01-02T00:00:00.0000000Z", letter.RecordedAt); + Assert.Equal(1, Assert.Single(all.Items, item => item.Type == "submitted-package").AttachmentCount); + Assert.Equal("/api/attachments/download/1", Assert.Single(all.Items, item => item.Type == "attachment").DownloadHref); + Assert.DoesNotContain(PrivateContent, JsonSerializer.Serialize(all)); + Assert.Empty(db.ChangeTracker.Entries()); + Assert.Equal(before, await SnapshotAsync(database)); + } + + [Fact] + public async Task Name_search_preserves_unicode_accents_literal_wildcards_and_binary_ties() + { + await using var database = await MariaDbDatabase.CreateAsync(); + await SeedAsync(database); + await using var db = database.Context(); + var names = new[] { "ÆØÅ résumé", "resume", "100%", "under_score", "bang!", "decoy", "same", "same" }; + db.CvVariants.AddRange(names.Select((name, index) => Variant(10 + index, Owner, name))); + await db.SaveChangesAsync(); + db.ChangeTracker.Clear(); + foreach (var (query, expected) in new[] { ("æøå", "cv:10"), ("RÉSUMÉ", "cv:10"), ("resume", "cv:11"), ("%", "cv:12"), ("_", "cv:13"), ("!", "cv:14") }) + { + var result = await GetAsync(db, q: query, type: "cv", pageSize: 1); + Assert.Equal(1, result.Total); + Assert.Equal(expected, Assert.Single(result.Items).Id); + } + var ties = await GetAsync(db, q: "same", type: "cv"); + Assert.Equal(new[] { "cv:16", "cv:17" }, ties.Items.Select(item => item.Id)); + Assert.Empty((await GetAsync(db, q: PrivateContent)).Items); + Assert.Empty(db.ChangeTracker.Entries()); + } + + [Fact] + public async Task Tenant_joins_and_filters_exclude_foreign_documents_and_damaged_associations() + { + await using var database = await MariaDbDatabase.CreateAsync(); + await SeedAsync(database); + await using var db = database.Context(); + var damaged = Variant(3, Owner, "Damaged association"); + damaged.JobApplicationId = 2; + db.CvVariants.Add(damaged); + db.EmailDrafts.Add(Draft(Owner, 2, "Foreign linked draft")); + db.SubmittedApplicationPackages.Add(new() { OwnerUserId = Owner, JobApplicationId = 2, Version = 99, CvVariantName = "Foreign linked package" }); + db.CoverLetterVersions.Add(new() { OwnerUserId = Other, JobApplicationId = 1, Version = 99, Text = PrivateContent }); + await db.SaveChangesAsync(); + db.ChangeTracker.Clear(); + var before = await SnapshotAsync(database); + var own = await GetAsync(db); + Assert.Equal(8, own.Total); + Assert.DoesNotContain(own.Items, item => item.Name.StartsWith("Foreign", StringComparison.Ordinal)); + Assert.DoesNotContain(own.Items, item => item.CompanyName == "Foreign company"); + var orphan = Assert.Single(own.Items, item => item.Id == "cv:3"); + Assert.Null(orphan.JobApplicationId); + Assert.Null(orphan.CompanyName); + Assert.Null(orphan.JobTitle); + Assert.Equal(1, Assert.Single(own.Items, item => item.Type == "cover-letter").HistoryCount); + Assert.Empty((await GetAsync(db, job: 2)).Items); + Assert.Equal(7, (await GetAsync(db, job: 1)).Total); + Assert.Equal(1, (await GetAsync(db, type: "cv", visibility: "public")).Total); + Assert.Equal(4, (await GetAsync(db, state: "draft")).Total); + var foreign = await GetAsync(db, owner: Other); + Assert.Equal(7, foreign.Total); + Assert.All(foreign.Items, item => Assert.Equal(2, item.JobApplicationId)); + Assert.Empty(db.ChangeTracker.Entries()); + Assert.Equal(before, await SnapshotAsync(database)); + } + + private static Task GetAsync(JobTrackerContext db, int page = 1, int pageSize = 100, + string? q = null, string type = "all", string state = "all", string visibility = "all", int? job = null, string owner = Owner) + => new DocumentsIndexService(db).GetAsync(owner, page, pageSize, q, type, state, visibility, job, "name-asc", default); + + private static CvVariant Variant(int id, string owner, string name) => new() + { + Id = id, OwnerUserId = owner, Name = name, PublicSlug = $"documents-{id}", SettingsJson = PrivateContent, + Version = 1, CreatedAtUtc = DateTimeOffset.Parse("2026-01-01T00:00:00Z"), UpdatedAtUtc = DateTimeOffset.Parse("2026-01-01T00:00:00Z"), + }; + + private static EmailDraft Draft(string owner, int job, string subject) => new() + { + Id = Guid.NewGuid(), OwnerUserId = owner, JobApplicationId = job, Subject = subject, Provider = "gmail", + To = "private@example.test", BodyText = PrivateContent, ClientRequestId = Guid.NewGuid().ToString(), + CreatedAtUtc = new DateTime(2026, 1, 1), UpdatedAtUtc = new DateTime(2026, 1, 1), + }; + + private static async Task SeedAsync(MariaDbDatabase database) + { + await using var db = database.Context(); + foreach (var (id, owner) in new[] { (1, Owner), (2, Other) }) + { + db.Users.Add(new() { Id = owner, UserName = owner }); + db.Companies.Add(new() { Id = id, OwnerUserId = owner, Name = id == 1 ? "Nordlys" : "Foreign company" }); + db.JobApplications.Add(new() + { + Id = id, OwnerUserId = owner, CompanyId = id, JobTitle = id == 1 ? "Owned role" : "Foreign role", + CoverLetterText = PrivateContent, TailoredCvText = PrivateContent, RecruiterMessageDraft = PrivateContent, + }); + var cv = Variant(id, owner, id == 1 ? "Public CV" : "Foreign CV"); + cv.JobApplicationId = id; + cv.IsPublic = true; + db.CvVariants.Add(cv); + db.CoverLetterVersions.Add(new() { OwnerUserId = owner, JobApplicationId = id, Version = 2, Text = PrivateContent, CreatedAtUtc = DateTimeOffset.Parse("2026-01-02T00:00:00Z") }); + db.Attachments.Add(new() { Id = id, JobApplicationId = id, FileName = "Portfolio.pdf", FilePath = PrivateContent, FileType = "application/pdf", FileSize = 41, UploadDate = new DateTime(2026, 1, 1) }); + db.SubmittedApplicationPackages.Add(new() + { + OwnerUserId = owner, JobApplicationId = id, Version = 1, CvVariantName = "Captured CV", CoverLetterText = PrivateContent, + Attachments = [new() { FileName = "Captured.pdf", FilePath = PrivateContent, FileType = "application/pdf", Sha256 = new string('a', 64) }], + }); + db.EmailDrafts.Add(Draft(owner, id, "Saved subject")); + } + await db.SaveChangesAsync(); + } + + private static async Task SnapshotAsync(MariaDbDatabase database) + { + await using var db = database.Context(); + return JsonSerializer.Serialize(new + { + Jobs = await db.JobApplications.IgnoreQueryFilters().AsNoTracking().OrderBy(x => x.Id).ToListAsync(), + Variants = await db.CvVariants.IgnoreQueryFilters().AsNoTracking().OrderBy(x => x.Id).ToListAsync(), + Letters = await db.CoverLetterVersions.IgnoreQueryFilters().AsNoTracking().OrderBy(x => x.Id).ToListAsync(), + Attachments = await db.Attachments.IgnoreQueryFilters().AsNoTracking().OrderBy(x => x.Id).ToListAsync(), + Packages = await db.SubmittedApplicationPackages.IgnoreQueryFilters().AsNoTracking().OrderBy(x => x.Id).ToListAsync(), + Archived = await db.SubmittedPackageAttachments.IgnoreQueryFilters().AsNoTracking().OrderBy(x => x.Id).ToListAsync(), + Drafts = await db.EmailDrafts.IgnoreQueryFilters().AsNoTracking().OrderBy(x => x.Id).ToListAsync(), + }); + } +} diff --git a/JobTrackerApi/Services/DocumentsIndexService.cs b/JobTrackerApi/Services/DocumentsIndexService.cs index 3b7ccaa..3ea1b97 100644 --- a/JobTrackerApi/Services/DocumentsIndexService.cs +++ b/JobTrackerApi/Services/DocumentsIndexService.cs @@ -59,8 +59,10 @@ public sealed class DocumentsIndexService(JobTrackerContext db) var answers = HasAnswers(sqlite); var recruiter = HasContent("j.RecruiterMessageDraft"); var tailored = HasContent("j.TailoredCvText"); + // Startup-created tables can have different collations; keep the index's folded, accent-sensitive names compatible. + string Name(string value) => sqlite ? value : $"CONVERT({value} USING utf8mb4) COLLATE utf8mb4_bin"; var sources = $$""" - SELECT 'cv' AS Type, CAST(v.Id AS CHAR) AS ResourceId, v.Name AS Name, 'current' AS State, + SELECT 'cv' AS Type, CAST(v.Id AS CHAR) AS ResourceId, {{Name("v.Name")}} AS Name, 'current' AS State, CASE WHEN v.IsPublic = 1 THEN 'public' ELSE 'private' END AS Visibility, j.Id AS JobApplicationId, j.JobTitle AS JobTitle, c.Name AS CompanyName, j.IsDeleted AS JobIsDeleted, CAST(v.UpdatedAtUtc AS CHAR) AS RawRecordedAt, 'updated-utc' AS DateKind, @@ -73,7 +75,7 @@ public sealed class DocumentsIndexService(JobTrackerContext db) LEFT JOIN `Companies` c ON c.Id = j.CompanyId AND c.OwnerUserId = {0} WHERE v.OwnerUserId = {0} UNION ALL - SELECT 'cover-letter', CAST(j.Id AS CHAR), j.JobTitle, 'draft', NULL, + SELECT 'cover-letter', CAST(j.Id AS CHAR), {{Name("j.JobTitle")}}, 'draft', NULL, j.Id, j.JobTitle, c.Name, j.IsDeleted, {{letterDate}}, CASE WHEN {{letterDate}} IS NULL THEN 'unknown' ELSE 'history-recorded-utc' END, {{letterVersion}}, {{letterCount}}, NULL, NULL, NULL, NULL, @@ -82,14 +84,14 @@ public sealed class DocumentsIndexService(JobTrackerContext db) LEFT JOIN `Companies` c ON c.Id = j.CompanyId AND c.OwnerUserId = {0} WHERE j.OwnerUserId = {0} AND ({{letterContent}} OR {{letterCount}} > 0) UNION ALL - SELECT 'application-material', CAST(j.Id AS CHAR), j.JobTitle, 'draft', NULL, + SELECT 'application-material', CAST(j.Id AS CHAR), {{Name("j.JobTitle")}}, 'draft', NULL, j.Id, j.JobTitle, c.Name, j.IsDeleted, NULL, 'unknown', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL FROM `JobApplications` j LEFT JOIN `Companies` c ON c.Id = j.CompanyId AND c.OwnerUserId = {0} WHERE j.OwnerUserId = {0} AND ({{answers}} OR {{recruiter}}) UNION ALL - SELECT 'tailored-cv', CAST(j.Id AS CHAR), j.JobTitle, 'draft', NULL, + SELECT 'tailored-cv', CAST(j.Id AS CHAR), {{Name("j.JobTitle")}}, 'draft', NULL, j.Id, j.JobTitle, c.Name, j.IsDeleted, CAST(COALESCE(d.LastEditedAtUtc, d.LastGeneratedAtUtc, j.TailoredCvUpdatedAt) AS CHAR), CASE WHEN COALESCE(d.LastEditedAtUtc, d.LastGeneratedAtUtc, j.TailoredCvUpdatedAt) IS NULL THEN 'unknown' ELSE 'updated-utc' END, @@ -99,14 +101,14 @@ public sealed class DocumentsIndexService(JobTrackerContext db) LEFT JOIN `TailoredCvDrafts` d ON d.JobApplicationId = j.Id AND d.OwnerUserId = {0} WHERE j.OwnerUserId = {0} AND (d.Id IS NOT NULL OR {{tailored}}) UNION ALL - SELECT 'attachment', CAST(a.Id AS CHAR), a.FileName, 'current', NULL, + SELECT 'attachment', CAST(a.Id AS CHAR), {{Name("a.FileName")}}, 'current', NULL, j.Id, j.JobTitle, c.Name, j.IsDeleted, CAST(a.UploadDate AS CHAR), 'uploaded-local', NULL, NULL, NULL, NULL, NULL, NULL, a.FileType, a.FileSize, a.Purpose, a.UseForAi, NULL, NULL, NULL FROM `Attachments` a JOIN `JobApplications` j ON j.Id = a.JobApplicationId AND j.OwnerUserId = {0} LEFT JOIN `Companies` c ON c.Id = j.CompanyId AND c.OwnerUserId = {0} UNION ALL - SELECT 'submitted-package', CAST(p.Id AS CHAR), COALESCE(p.CvVariantName, j.JobTitle), 'frozen', NULL, + SELECT 'submitted-package', CAST(p.Id AS CHAR), {{Name("COALESCE(p.CvVariantName, j.JobTitle)")}}, 'frozen', NULL, j.Id, j.JobTitle, c.Name, j.IsDeleted, CAST(p.CreatedAtUtc AS CHAR), 'captured-utc', p.Version, NULL, (SELECT COUNT(*) FROM `SubmittedPackageAttachments` a WHERE a.SubmittedApplicationPackageId = p.Id), p.CvVariantId, p.CvVariantName, p.CvVariantVersion, NULL, NULL, NULL, NULL, NULL, NULL, NULL @@ -115,7 +117,7 @@ public sealed class DocumentsIndexService(JobTrackerContext db) LEFT JOIN `Companies` c ON c.Id = j.CompanyId AND c.OwnerUserId = {0} WHERE p.OwnerUserId = {0} UNION ALL - SELECT 'email-draft', CAST(d.Id AS CHAR), d.Subject, 'draft', NULL, + SELECT 'email-draft', CAST(d.Id AS CHAR), {{Name("d.Subject")}}, 'draft', NULL, j.Id, j.JobTitle, c.Name, j.IsDeleted, CAST(d.UpdatedAtUtc AS CHAR), 'updated-utc', d.Revision, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, d.Provider, NULL, NULL FROM `EmailDrafts` d