feat(search): relevance ranking + websearch_to_tsquery #13

Merged
cesnimda merged 2 commits from feature/search-relevance-ranking into develop 2026-07-01 21:08:54 +02:00
6 changed files with 128 additions and 48 deletions
@@ -42,8 +42,15 @@ public class WidgetLayoutController : ApiControllerBase
{ {
_db.WidgetLayouts.Add(new WidgetLayout _db.WidgetLayouts.Add(new WidgetLayout
{ {
UserId = UserId, WidgetKey = dto.WidgetKey, X = dto.X, Y = dto.Y, W = dto.W, H = dto.H, UserId = UserId,
Visible = dto.Visible, SortOrder = dto.SortOrder, SettingsJson = dto.SettingsJson WidgetKey = dto.WidgetKey,
X = dto.X,
Y = dto.Y,
W = dto.W,
H = dto.H,
Visible = dto.Visible,
SortOrder = dto.SortOrder,
SettingsJson = dto.SettingsJson
}); });
} }
} }
@@ -133,13 +133,13 @@ public class AnalyticsService : IAnalyticsService
var emails = _db.Emails.Where(e => e.UserId == userId); var emails = _db.Emails.Where(e => e.UserId == userId);
var allMail = await emails.CountAsync(ct); var allMail = await emails.CountAsync(ct);
var inbox = await emails.CountAsync(e => e.IsInInbox, ct); var inbox = await emails.CountAsync(e => e.IsInInbox, ct);
var unread = await emails.CountAsync(e => e.IsUnread, ct); var unread = await emails.CountAsync(e => e.IsUnread, ct);
var starred = await emails.CountAsync(e => e.IsStarred, ct); var starred = await emails.CountAsync(e => e.IsStarred, ct);
var trash = await emails.CountAsync(e => e.IsTrashed, ct); var trash = await emails.CountAsync(e => e.IsTrashed, ct);
var large = await emails.CountAsync(e => e.SizeEstimateBytes > 5_000_000, ct); var large = await emails.CountAsync(e => e.SizeEstimateBytes > 5_000_000, ct);
var cutoff = DateTimeOffset.UtcNow.AddYears(-1); var cutoff = DateTimeOffset.UtcNow.AddYears(-1);
var old = await emails.CountAsync(e => e.SentAtUtc < cutoff, ct); var old = await emails.CountAsync(e => e.SentAtUtc < cutoff, ct);
// Label-backed counts (SENT / DRAFT / SPAM are system Gmail labels) // Label-backed counts (SENT / DRAFT / SPAM are system Gmail labels)
async Task<int> LabelCount(string gmailId) async Task<int> LabelCount(string gmailId)
@@ -153,9 +153,9 @@ public class AnalyticsService : IAnalyticsService
: 0; : 0;
} }
var sent = await LabelCount("SENT"); var sent = await LabelCount("SENT");
var drafts = await LabelCount("DRAFT"); var drafts = await LabelCount("DRAFT");
var spam = await LabelCount("SPAM"); var spam = await LabelCount("SPAM");
// Category → smart-folder slug mapping // Category → smart-folder slug mapping
var catCounts = await emails var catCounts = await emails
@@ -167,31 +167,31 @@ public class AnalyticsService : IAnalyticsService
var smartFolders = new Dictionary<string, int> var smartFolders = new Dictionary<string, int>
{ {
["automated"] = Cat(EmailCategory.Notification), ["automated"] = Cat(EmailCategory.Notification),
["finance"] = Cat(EmailCategory.Finance), ["finance"] = Cat(EmailCategory.Finance),
["social"] = Cat(EmailCategory.Social), ["social"] = Cat(EmailCategory.Social),
["shopping"] = Cat(EmailCategory.Shopping) + Cat(EmailCategory.Promotional), ["shopping"] = Cat(EmailCategory.Shopping) + Cat(EmailCategory.Promotional),
["noreply"] = Cat(EmailCategory.Notification), ["noreply"] = Cat(EmailCategory.Notification),
["gaming"] = Cat(EmailCategory.Gaming), ["gaming"] = Cat(EmailCategory.Gaming),
["sales"] = Cat(EmailCategory.SeasonalSales), ["sales"] = Cat(EmailCategory.SeasonalSales),
["ridesharing"] = Cat(EmailCategory.RideSharing), ["ridesharing"] = Cat(EmailCategory.RideSharing),
["food"] = Cat(EmailCategory.FoodDelivery), ["food"] = Cat(EmailCategory.FoodDelivery),
["wellness"] = Cat(EmailCategory.Wellness), ["wellness"] = Cat(EmailCategory.Wellness),
// New categories // New categories
["travel"] = Cat(EmailCategory.Travel), ["travel"] = Cat(EmailCategory.Travel),
["subscriptions"] = Cat(EmailCategory.Subscriptions), ["subscriptions"] = Cat(EmailCategory.Subscriptions),
["parcels"] = Cat(EmailCategory.Parcels), ["parcels"] = Cat(EmailCategory.Parcels),
["recruitment"] = Cat(EmailCategory.Recruitment), ["recruitment"] = Cat(EmailCategory.Recruitment),
["events"] = Cat(EmailCategory.Events), ["events"] = Cat(EmailCategory.Events),
["security"] = Cat(EmailCategory.SecurityAlerts), ["security"] = Cat(EmailCategory.SecurityAlerts),
["healthcare"] = Cat(EmailCategory.Healthcare), ["healthcare"] = Cat(EmailCategory.Healthcare),
["education"] = Cat(EmailCategory.Education), ["education"] = Cat(EmailCategory.Education),
["news"] = Cat(EmailCategory.NewsMedia), ["news"] = Cat(EmailCategory.NewsMedia),
["property"] = Cat(EmailCategory.PropertyUtilities), ["property"] = Cat(EmailCategory.PropertyUtilities),
["charity"] = Cat(EmailCategory.Charity), ["charity"] = Cat(EmailCategory.Charity),
["government"] = Cat(EmailCategory.Government), ["government"] = Cat(EmailCategory.Government),
["crypto"] = Cat(EmailCategory.CryptoInvesting), ["crypto"] = Cat(EmailCategory.CryptoInvesting),
["family"] = Cat(EmailCategory.FamilySchool), ["family"] = Cat(EmailCategory.FamilySchool),
}; };
return new SidebarCountsDto(inbox, allMail, unread, starred, sent, drafts, trash, spam, large, old, smartFolders); return new SidebarCountsDto(inbox, allMail, unread, starred, sent, drafts, trash, spam, large, old, smartFolders);
@@ -10,7 +10,9 @@ namespace InboxIntel.Infrastructure.Search;
/// <summary> /// <summary>
/// Structured + full-text search. Structured filters compose as SQL WHERE /// Structured + full-text search. Structured filters compose as SQL WHERE
/// clauses; free text uses PostgreSQL FTS via the generated SearchVector column /// clauses; free text uses PostgreSQL FTS via the generated SearchVector column
/// (EF.Functions.ToTsVector/Matches translate to @@ / to_tsquery). /// (websearch_to_tsquery / @@ / ts_rank_cd). Results are relevance-ranked when a
/// free-text query is present, date-ordered otherwise. See
/// docs/discovery/05-search-redesign.md for the full multi-layer search design.
/// </summary> /// </summary>
public class SearchService : ISearchService public class SearchService : ISearchService
{ {
@@ -58,16 +60,24 @@ public class SearchService : ISearchService
return new PagedResult<EmailSummaryDto> { Items = [], Page = r.Page, PageSize = r.PageSize, TotalCount = 0 }; return new PagedResult<EmailSummaryDto> { Items = [], Page = r.Page, PageSize = r.PageSize, TotalCount = 0 };
} }
if (!string.IsNullOrWhiteSpace(r.Query)) // websearch_to_tsquery (vs. plainto_tsquery) understands quotes ("exact phrase"),
{ // OR, and -exclusions — the syntax users already expect from web search boxes.
// PostgreSQL full-text match against the generated tsvector. var hasFreeTextQuery = !string.IsNullOrWhiteSpace(r.Query);
var term = r.Query.Trim(); var term = r.Query?.Trim() ?? string.Empty;
q = q.Where(e => e.SearchVector!.Matches(EF.Functions.PlainToTsQuery("english", term))); if (hasFreeTextQuery)
} q = q.Where(e => e.SearchVector!.Matches(EF.Functions.WebSearchToTsQuery("english", term)));
var total = await q.CountAsync(ct); var total = await q.CountAsync(ct);
var items = await q
.OrderByDescending(e => e.SentAtUtc) // Relevance-ranked when there's a free-text query (ts_rank_cd via RankCoverDensity,
// recency as a tiebreaker); date-only otherwise — matches the existing browse
// behaviour when the user isn't searching for anything in particular.
var ranked = hasFreeTextQuery
? q.OrderByDescending(e => e.SearchVector!.RankCoverDensity(EF.Functions.WebSearchToTsQuery("english", term)))
.ThenByDescending(e => e.SentAtUtc)
: q.OrderByDescending(e => e.SentAtUtc);
var items = await ranked
.Skip((r.Page - 1) * r.PageSize) .Skip((r.Page - 1) * r.PageSize)
.Take(r.PageSize) .Take(r.PageSize)
.Select(e => new EmailSummaryDto( .Select(e => new EmailSummaryDto(
@@ -79,7 +89,10 @@ public class SearchService : ISearchService
return new PagedResult<EmailSummaryDto> return new PagedResult<EmailSummaryDto>
{ {
Items = items, Page = r.Page, PageSize = r.PageSize, TotalCount = total Items = items,
Page = r.Page,
PageSize = r.PageSize,
TotalCount = total
}; };
} }
} }
@@ -412,7 +412,7 @@ public static class HeuristicClassifier
public static EmailCategory Classify(GmailMessageDetail d) public static EmailCategory Classify(GmailMessageDetail d)
{ {
var subject = (d.Subject ?? string.Empty).ToLowerInvariant(); var subject = (d.Subject ?? string.Empty).ToLowerInvariant();
var from = d.FromAddress.ToLowerInvariant(); var from = d.FromAddress.ToLowerInvariant();
// 1. Security alerts — highest priority, overrides everything // 1. Security alerts — highest priority, overrides everything
if (SecuritySubjectHints.Any(h => subject.Contains(h))) if (SecuritySubjectHints.Any(h => subject.Contains(h)))
@@ -268,8 +268,12 @@ public class SyncService : ISyncService
{ {
_db.Attachments.Add(new Attachment _db.Attachments.Add(new Attachment
{ {
UserId = userId, EmailId = email.Id, FileName = Trunc(fileName, 512) ?? string.Empty, UserId = userId,
MimeType = Trunc(mime, 255), SizeBytes = size, GmailAttachmentId = attId EmailId = email.Id,
FileName = Trunc(fileName, 512) ?? string.Empty,
MimeType = Trunc(mime, 255),
SizeBytes = size,
GmailAttachmentId = attId
}); });
} }
@@ -338,8 +342,11 @@ public class SyncService : ISyncService
if (thread is not null) return thread; if (thread is not null) return thread;
thread = new MailThread thread = new MailThread
{ {
UserId = userId, GmailThreadId = gmailThreadId, Subject = Trunc(subject, 1024), UserId = userId,
Snippet = Trunc(snippet, 2048), FirstMessageUtc = sentAt GmailThreadId = gmailThreadId,
Subject = Trunc(subject, 1024),
Snippet = Trunc(snippet, 2048),
FirstMessageUtc = sentAt
}; };
_db.Threads.Add(thread); _db.Threads.Add(thread);
return thread; return thread;
@@ -0,0 +1,53 @@
using FluentAssertions;
using InboxIntel.Application.Abstractions;
using InboxIntel.Application.Search;
using InboxIntel.Domain.Entities;
using InboxIntel.Infrastructure.Persistence;
using InboxIntel.Infrastructure.Search;
using Microsoft.EntityFrameworkCore;
using Xunit;
namespace InboxIntel.IntegrationTests;
/// <summary>
/// SearchService now relevance-ranks (ts_rank_cd via RankCoverDensity) when a free-text
/// query is present, falling back to date order otherwise. The ranked path requires a
/// live PostgreSQL instance to exercise (EF's InMemory provider cannot translate
/// websearch_to_tsquery/RankCoverDensity) — this project deliberately avoids a hard
/// Postgres/Testcontainers dependency for tests (see AuthEndpointsTests.TestAppFactory),
/// so ranking correctness itself is verified manually/in staging, not here. What IS
/// covered: the date-order fallback, which is plain LINQ and must not regress.
/// </summary>
public class SearchRankingTests
{
private sealed class FakeCurrentUser : ICurrentUser
{
public Guid UserId { get; set; }
public bool IsAuthenticated => UserId != Guid.Empty;
}
[Fact]
public async Task No_query_falls_back_to_date_descending_order()
{
var user = Guid.NewGuid();
var opts = new DbContextOptionsBuilder<AppDbContext>()
.UseInMemoryDatabase(nameof(No_query_falls_back_to_date_descending_order)).Options;
using (var seed = new AppDbContext(opts, new FakeCurrentUser()))
{
var sender = new Sender { UserId = user, Address = "sender@example.com", DisplayName = "Sender" };
seed.Senders.Add(sender);
var baseline = DateTimeOffset.UtcNow;
seed.Emails.Add(new Email { UserId = user, GmailMessageId = "oldest", SentAtUtc = baseline.AddDays(-2), Sender = sender });
seed.Emails.Add(new Email { UserId = user, GmailMessageId = "newest", SentAtUtc = baseline, Sender = sender });
seed.Emails.Add(new Email { UserId = user, GmailMessageId = "middle", SentAtUtc = baseline.AddDays(-1), Sender = sender });
await seed.SaveChangesAsync();
}
using var ctx = new AppDbContext(opts, new FakeCurrentUser { UserId = user });
var request = GmailQueryParser.Parse(null, page: 1, pageSize: 50);
var result = await new SearchService(ctx).SearchAsync(user, request);
result.Items.Select(i => i.GmailMessageId).Should().ContainInOrder("newest", "middle", "oldest");
}
}