From a9be48daee9cdccf73dd9bef0e33b21b1d3fcc84 Mon Sep 17 00:00:00 2001 From: cesnimda Date: Wed, 1 Jul 2026 21:08:53 +0200 Subject: [PATCH] feat(search): relevance ranking + websearch_to_tsquery (#13) --- .../Controllers/WidgetLayoutController.cs | 11 +++- .../Analytics/AnalyticsService.cs | 62 +++++++++---------- .../Search/SearchService.cs | 33 +++++++--- .../Sync/HeuristicClassifier.cs | 2 +- .../Sync/SyncService.cs | 15 +++-- .../SearchRankingTests.cs | 53 ++++++++++++++++ 6 files changed, 128 insertions(+), 48 deletions(-) create mode 100644 tests/InboxIntel.IntegrationTests/SearchRankingTests.cs diff --git a/src/InboxIntel.Api/Controllers/WidgetLayoutController.cs b/src/InboxIntel.Api/Controllers/WidgetLayoutController.cs index 0dc1dec..fc3ee67 100644 --- a/src/InboxIntel.Api/Controllers/WidgetLayoutController.cs +++ b/src/InboxIntel.Api/Controllers/WidgetLayoutController.cs @@ -42,8 +42,15 @@ public class WidgetLayoutController : ApiControllerBase { _db.WidgetLayouts.Add(new WidgetLayout { - UserId = UserId, WidgetKey = dto.WidgetKey, X = dto.X, Y = dto.Y, W = dto.W, H = dto.H, - Visible = dto.Visible, SortOrder = dto.SortOrder, SettingsJson = dto.SettingsJson + UserId = UserId, + WidgetKey = dto.WidgetKey, + X = dto.X, + Y = dto.Y, + W = dto.W, + H = dto.H, + Visible = dto.Visible, + SortOrder = dto.SortOrder, + SettingsJson = dto.SettingsJson }); } } diff --git a/src/InboxIntel.Infrastructure/Analytics/AnalyticsService.cs b/src/InboxIntel.Infrastructure/Analytics/AnalyticsService.cs index 17a0e2e..7cb9fe2 100644 --- a/src/InboxIntel.Infrastructure/Analytics/AnalyticsService.cs +++ b/src/InboxIntel.Infrastructure/Analytics/AnalyticsService.cs @@ -133,13 +133,13 @@ public class AnalyticsService : IAnalyticsService var emails = _db.Emails.Where(e => e.UserId == userId); var allMail = await emails.CountAsync(ct); - var inbox = await emails.CountAsync(e => e.IsInInbox, ct); - var unread = await emails.CountAsync(e => e.IsUnread, ct); + var inbox = await emails.CountAsync(e => e.IsInInbox, ct); + var unread = await emails.CountAsync(e => e.IsUnread, ct); var starred = await emails.CountAsync(e => e.IsStarred, ct); - var trash = await emails.CountAsync(e => e.IsTrashed, ct); - var large = await emails.CountAsync(e => e.SizeEstimateBytes > 5_000_000, ct); - var cutoff = DateTimeOffset.UtcNow.AddYears(-1); - var old = await emails.CountAsync(e => e.SentAtUtc < cutoff, ct); + var trash = await emails.CountAsync(e => e.IsTrashed, ct); + var large = await emails.CountAsync(e => e.SizeEstimateBytes > 5_000_000, ct); + var cutoff = DateTimeOffset.UtcNow.AddYears(-1); + var old = await emails.CountAsync(e => e.SentAtUtc < cutoff, ct); // Label-backed counts (SENT / DRAFT / SPAM are system Gmail labels) async Task LabelCount(string gmailId) @@ -153,9 +153,9 @@ public class AnalyticsService : IAnalyticsService : 0; } - var sent = await LabelCount("SENT"); + var sent = await LabelCount("SENT"); var drafts = await LabelCount("DRAFT"); - var spam = await LabelCount("SPAM"); + var spam = await LabelCount("SPAM"); // Category → smart-folder slug mapping var catCounts = await emails @@ -167,31 +167,31 @@ public class AnalyticsService : IAnalyticsService var smartFolders = new Dictionary { - ["automated"] = Cat(EmailCategory.Notification), - ["finance"] = Cat(EmailCategory.Finance), - ["social"] = Cat(EmailCategory.Social), - ["shopping"] = Cat(EmailCategory.Shopping) + Cat(EmailCategory.Promotional), - ["noreply"] = Cat(EmailCategory.Notification), - ["gaming"] = Cat(EmailCategory.Gaming), - ["sales"] = Cat(EmailCategory.SeasonalSales), - ["ridesharing"] = Cat(EmailCategory.RideSharing), - ["food"] = Cat(EmailCategory.FoodDelivery), - ["wellness"] = Cat(EmailCategory.Wellness), + ["automated"] = Cat(EmailCategory.Notification), + ["finance"] = Cat(EmailCategory.Finance), + ["social"] = Cat(EmailCategory.Social), + ["shopping"] = Cat(EmailCategory.Shopping) + Cat(EmailCategory.Promotional), + ["noreply"] = Cat(EmailCategory.Notification), + ["gaming"] = Cat(EmailCategory.Gaming), + ["sales"] = Cat(EmailCategory.SeasonalSales), + ["ridesharing"] = Cat(EmailCategory.RideSharing), + ["food"] = Cat(EmailCategory.FoodDelivery), + ["wellness"] = Cat(EmailCategory.Wellness), // New categories - ["travel"] = Cat(EmailCategory.Travel), + ["travel"] = Cat(EmailCategory.Travel), ["subscriptions"] = Cat(EmailCategory.Subscriptions), - ["parcels"] = Cat(EmailCategory.Parcels), - ["recruitment"] = Cat(EmailCategory.Recruitment), - ["events"] = Cat(EmailCategory.Events), - ["security"] = Cat(EmailCategory.SecurityAlerts), - ["healthcare"] = Cat(EmailCategory.Healthcare), - ["education"] = Cat(EmailCategory.Education), - ["news"] = Cat(EmailCategory.NewsMedia), - ["property"] = Cat(EmailCategory.PropertyUtilities), - ["charity"] = Cat(EmailCategory.Charity), - ["government"] = Cat(EmailCategory.Government), - ["crypto"] = Cat(EmailCategory.CryptoInvesting), - ["family"] = Cat(EmailCategory.FamilySchool), + ["parcels"] = Cat(EmailCategory.Parcels), + ["recruitment"] = Cat(EmailCategory.Recruitment), + ["events"] = Cat(EmailCategory.Events), + ["security"] = Cat(EmailCategory.SecurityAlerts), + ["healthcare"] = Cat(EmailCategory.Healthcare), + ["education"] = Cat(EmailCategory.Education), + ["news"] = Cat(EmailCategory.NewsMedia), + ["property"] = Cat(EmailCategory.PropertyUtilities), + ["charity"] = Cat(EmailCategory.Charity), + ["government"] = Cat(EmailCategory.Government), + ["crypto"] = Cat(EmailCategory.CryptoInvesting), + ["family"] = Cat(EmailCategory.FamilySchool), }; return new SidebarCountsDto(inbox, allMail, unread, starred, sent, drafts, trash, spam, large, old, smartFolders); diff --git a/src/InboxIntel.Infrastructure/Search/SearchService.cs b/src/InboxIntel.Infrastructure/Search/SearchService.cs index ef3b0b4..178b3e4 100644 --- a/src/InboxIntel.Infrastructure/Search/SearchService.cs +++ b/src/InboxIntel.Infrastructure/Search/SearchService.cs @@ -10,7 +10,9 @@ namespace InboxIntel.Infrastructure.Search; /// /// Structured + full-text search. Structured filters compose as SQL WHERE /// clauses; free text uses PostgreSQL FTS via the generated SearchVector column -/// (EF.Functions.ToTsVector/Matches translate to @@ / to_tsquery). +/// (websearch_to_tsquery / @@ / ts_rank_cd). Results are relevance-ranked when a +/// free-text query is present, date-ordered otherwise. See +/// docs/discovery/05-search-redesign.md for the full multi-layer search design. /// public class SearchService : ISearchService { @@ -58,16 +60,24 @@ public class SearchService : ISearchService return new PagedResult { Items = [], Page = r.Page, PageSize = r.PageSize, TotalCount = 0 }; } - if (!string.IsNullOrWhiteSpace(r.Query)) - { - // PostgreSQL full-text match against the generated tsvector. - var term = r.Query.Trim(); - q = q.Where(e => e.SearchVector!.Matches(EF.Functions.PlainToTsQuery("english", term))); - } + // websearch_to_tsquery (vs. plainto_tsquery) understands quotes ("exact phrase"), + // OR, and -exclusions — the syntax users already expect from web search boxes. + var hasFreeTextQuery = !string.IsNullOrWhiteSpace(r.Query); + var term = r.Query?.Trim() ?? string.Empty; + if (hasFreeTextQuery) + q = q.Where(e => e.SearchVector!.Matches(EF.Functions.WebSearchToTsQuery("english", term))); var total = await q.CountAsync(ct); - var items = await q - .OrderByDescending(e => e.SentAtUtc) + + // Relevance-ranked when there's a free-text query (ts_rank_cd via RankCoverDensity, + // recency as a tiebreaker); date-only otherwise — matches the existing browse + // behaviour when the user isn't searching for anything in particular. + var ranked = hasFreeTextQuery + ? q.OrderByDescending(e => e.SearchVector!.RankCoverDensity(EF.Functions.WebSearchToTsQuery("english", term))) + .ThenByDescending(e => e.SentAtUtc) + : q.OrderByDescending(e => e.SentAtUtc); + + var items = await ranked .Skip((r.Page - 1) * r.PageSize) .Take(r.PageSize) .Select(e => new EmailSummaryDto( @@ -79,7 +89,10 @@ public class SearchService : ISearchService return new PagedResult { - Items = items, Page = r.Page, PageSize = r.PageSize, TotalCount = total + Items = items, + Page = r.Page, + PageSize = r.PageSize, + TotalCount = total }; } } diff --git a/src/InboxIntel.Infrastructure/Sync/HeuristicClassifier.cs b/src/InboxIntel.Infrastructure/Sync/HeuristicClassifier.cs index 445deb5..bf8d8a4 100644 --- a/src/InboxIntel.Infrastructure/Sync/HeuristicClassifier.cs +++ b/src/InboxIntel.Infrastructure/Sync/HeuristicClassifier.cs @@ -412,7 +412,7 @@ public static class HeuristicClassifier public static EmailCategory Classify(GmailMessageDetail d) { var subject = (d.Subject ?? string.Empty).ToLowerInvariant(); - var from = d.FromAddress.ToLowerInvariant(); + var from = d.FromAddress.ToLowerInvariant(); // 1. Security alerts — highest priority, overrides everything if (SecuritySubjectHints.Any(h => subject.Contains(h))) diff --git a/src/InboxIntel.Infrastructure/Sync/SyncService.cs b/src/InboxIntel.Infrastructure/Sync/SyncService.cs index 7b7f15a..27798e9 100644 --- a/src/InboxIntel.Infrastructure/Sync/SyncService.cs +++ b/src/InboxIntel.Infrastructure/Sync/SyncService.cs @@ -268,8 +268,12 @@ public class SyncService : ISyncService { _db.Attachments.Add(new Attachment { - UserId = userId, EmailId = email.Id, FileName = Trunc(fileName, 512) ?? string.Empty, - MimeType = Trunc(mime, 255), SizeBytes = size, GmailAttachmentId = attId + UserId = userId, + EmailId = email.Id, + FileName = Trunc(fileName, 512) ?? string.Empty, + MimeType = Trunc(mime, 255), + SizeBytes = size, + GmailAttachmentId = attId }); } @@ -338,8 +342,11 @@ public class SyncService : ISyncService if (thread is not null) return thread; thread = new MailThread { - UserId = userId, GmailThreadId = gmailThreadId, Subject = Trunc(subject, 1024), - Snippet = Trunc(snippet, 2048), FirstMessageUtc = sentAt + UserId = userId, + GmailThreadId = gmailThreadId, + Subject = Trunc(subject, 1024), + Snippet = Trunc(snippet, 2048), + FirstMessageUtc = sentAt }; _db.Threads.Add(thread); return thread; diff --git a/tests/InboxIntel.IntegrationTests/SearchRankingTests.cs b/tests/InboxIntel.IntegrationTests/SearchRankingTests.cs new file mode 100644 index 0000000..9706171 --- /dev/null +++ b/tests/InboxIntel.IntegrationTests/SearchRankingTests.cs @@ -0,0 +1,53 @@ +using FluentAssertions; +using InboxIntel.Application.Abstractions; +using InboxIntel.Application.Search; +using InboxIntel.Domain.Entities; +using InboxIntel.Infrastructure.Persistence; +using InboxIntel.Infrastructure.Search; +using Microsoft.EntityFrameworkCore; +using Xunit; + +namespace InboxIntel.IntegrationTests; + +/// +/// SearchService now relevance-ranks (ts_rank_cd via RankCoverDensity) when a free-text +/// query is present, falling back to date order otherwise. The ranked path requires a +/// live PostgreSQL instance to exercise (EF's InMemory provider cannot translate +/// websearch_to_tsquery/RankCoverDensity) — this project deliberately avoids a hard +/// Postgres/Testcontainers dependency for tests (see AuthEndpointsTests.TestAppFactory), +/// so ranking correctness itself is verified manually/in staging, not here. What IS +/// covered: the date-order fallback, which is plain LINQ and must not regress. +/// +public class SearchRankingTests +{ + private sealed class FakeCurrentUser : ICurrentUser + { + public Guid UserId { get; set; } + public bool IsAuthenticated => UserId != Guid.Empty; + } + + [Fact] + public async Task No_query_falls_back_to_date_descending_order() + { + var user = Guid.NewGuid(); + var opts = new DbContextOptionsBuilder() + .UseInMemoryDatabase(nameof(No_query_falls_back_to_date_descending_order)).Options; + + using (var seed = new AppDbContext(opts, new FakeCurrentUser())) + { + var sender = new Sender { UserId = user, Address = "sender@example.com", DisplayName = "Sender" }; + seed.Senders.Add(sender); + var baseline = DateTimeOffset.UtcNow; + seed.Emails.Add(new Email { UserId = user, GmailMessageId = "oldest", SentAtUtc = baseline.AddDays(-2), Sender = sender }); + seed.Emails.Add(new Email { UserId = user, GmailMessageId = "newest", SentAtUtc = baseline, Sender = sender }); + seed.Emails.Add(new Email { UserId = user, GmailMessageId = "middle", SentAtUtc = baseline.AddDays(-1), Sender = sender }); + await seed.SaveChangesAsync(); + } + + using var ctx = new AppDbContext(opts, new FakeCurrentUser { UserId = user }); + var request = GmailQueryParser.Parse(null, page: 1, pageSize: 50); + var result = await new SearchService(ctx).SearchAsync(user, request); + + result.Items.Select(i => i.GmailMessageId).Should().ContainInOrder("newest", "middle", "oldest"); + } +}