diff --git a/src/InboxIntel.Infrastructure/Search/SearchService.cs b/src/InboxIntel.Infrastructure/Search/SearchService.cs index 402ca20..c93b8f7 100644 --- a/src/InboxIntel.Infrastructure/Search/SearchService.cs +++ b/src/InboxIntel.Infrastructure/Search/SearchService.cs @@ -5,6 +5,7 @@ using InboxIntel.Domain.Entities; using InboxIntel.Domain.Enums; using InboxIntel.Infrastructure.Persistence; using Microsoft.EntityFrameworkCore; +using Pgvector.EntityFrameworkCore; namespace InboxIntel.Infrastructure.Search; @@ -18,7 +19,13 @@ namespace InboxIntel.Infrastructure.Search; public class SearchService : ISearchService { private readonly AppDbContext _db; - public SearchService(AppDbContext db) => _db = db; + private readonly IEmbeddingProvider? _embeddings; + + public SearchService(AppDbContext db, IEmbeddingProvider? embeddings = null) + { + _db = db; + _embeddings = embeddings; + } public async Task> SearchAsync(Guid userId, SearchRequestDto r, CancellationToken ct = default) { @@ -73,6 +80,18 @@ public class SearchService : ISearchService var total = await matched.CountAsync(ct); + // Hybrid semantic fusion (docs/discovery/05): when embeddings are available, fuse + // lexical top-K with vector top-K via Reciprocal Rank Fusion. Runs BEFORE the fuzzy + // fallback so a query with ZERO lexical hits (pure semantic recall — "gym receipt" + // phrased differently) still surfaces results. Exact lexical hits keep winning (they + // rank in both lists). Deeper pages fall through to lexical paging; any failure + // (Ollama down, nothing embedded yet) silently degrades to the lexical/fuzzy path. + if (hasFreeTextQuery && _embeddings is { IsAvailable: true }) + { + var hybrid = await TryHybridAsync(structured, matched, term, total, r, ct); + if (hybrid is not null) return hybrid; + } + // Fuzzy/typo fallback: ONLY when a free-text search found nothing exact. word_similarity // with an explicit 0.3 threshold — pg_trgm's default 0.6 misses real typos // ("recieved" -> "received" scores ~0.39). Rare path, so the (non-indexed) scan over the @@ -137,4 +156,73 @@ public class SearchService : ISearchService TotalCount = total }; } + + private const int HybridK = 50; // candidates taken from each layer + private const int RrfConstant = 60; // standard RRF dampening constant + + /// + /// RRF fusion of lexical and vector candidates. Returns null when the requested page + /// lies beyond the fused window or anything fails — caller falls back to lexical. + /// + private async Task?> TryHybridAsync( + IQueryable structured, IQueryable lexical, string term, int lexicalTotal, + SearchRequestDto r, CancellationToken ct) + { + try + { + var lexIds = await lexical + .OrderByDescending(e => e.SearchVector!.RankCoverDensity(EF.Functions.WebSearchToTsQuery("english", term))) + .ThenByDescending(e => e.SentAtUtc) + .Take(HybridK).Select(e => e.Id).ToListAsync(ct); + + var queryVec = await _embeddings!.EmbedAsync(term, ct); + List vecIds = new(); + if (queryVec.Length > 0) + { + var qv = new Pgvector.Vector(queryVec); + vecIds = await structured + .Where(e => e.Embedding != null) + .OrderBy(e => e.Embedding!.CosineDistance(qv)) + .Take(HybridK).Select(e => e.Id).ToListAsync(ct); + } + if (vecIds.Count == 0) return null; // nothing embedded yet → lexical path + + var scores = new Dictionary(); + for (var i = 0; i < lexIds.Count; i++) + scores[lexIds[i]] = scores.GetValueOrDefault(lexIds[i]) + 1.0 / (RrfConstant + i + 1); + for (var i = 0; i < vecIds.Count; i++) + scores[vecIds[i]] = scores.GetValueOrDefault(vecIds[i]) + 1.0 / (RrfConstant + i + 1); + + var fused = scores.OrderByDescending(kv => kv.Value).Select(kv => kv.Key).ToList(); + var pageIds = fused.Skip((r.Page - 1) * r.PageSize).Take(r.PageSize).ToList(); + if (pageIds.Count == 0 && r.Page > 1) return null; // deep page → lexical paging + + var headlineOpts = + $"StartSel={(char)0xE000},StopSel={(char)0xE001},MaxWords=16,MinWords=5,ShortWord=2,HighlightAll=false"; + var rows = await _db.Emails.AsNoTracking() + .Where(e => pageIds.Contains(e.Id)) + .Select(e => new EmailSummaryDto( + e.Id, e.GmailMessageId, e.Subject, e.Snippet, + e.Sender!.Address, e.Sender.DisplayName, e.SentAtUtc, + e.IsUnread, e.IsStarred, e.HasAttachments, e.SizeEstimateBytes, e.Category, + e.HasListUnsubscribe, e.SupportsOneClickUnsubscribe, + EF.Functions.WebSearchToTsQuery("english", term).GetResultHeadline("english", e.BodyText ?? "", headlineOpts))) + .ToListAsync(ct); + var byId = rows.ToDictionary(x => x.Id); + var items = pageIds.Where(byId.ContainsKey).Select(id => byId[id]).ToList(); + + return new PagedResult + { + Items = items, + Page = r.Page, + PageSize = r.PageSize, + // Semantic recall can exceed the lexical match count. + TotalCount = Math.Max(lexicalTotal, fused.Count) + }; + } + catch + { + return null; // AI must never break search — degrade to lexical + } + } } diff --git a/tests/InboxIntel.IntegrationTests/LiveDbSearchTests.cs b/tests/InboxIntel.IntegrationTests/LiveDbSearchTests.cs index 4793b15..674f740 100644 --- a/tests/InboxIntel.IntegrationTests/LiveDbSearchTests.cs +++ b/tests/InboxIntel.IntegrationTests/LiveDbSearchTests.cs @@ -123,4 +123,54 @@ public class LiveDbSearchTests } finally { await CleanupAsync(opts, uid); } } + + private sealed class DirectionalFakeEmbeddings : IEmbeddingProvider + { + public bool IsAvailable => true; + public Task EmbedAsync(string text, CancellationToken ct = default) + { + // Deterministic "semantics": anything fruit-flavoured points one way, else the other. + var v = new float[768]; + if (text.Contains("banana") || text.Contains("tropical")) v[0] = 1; else v[1] = 1; + return Task.FromResult(v); + } + public async Task> EmbedBatchAsync(IReadOnlyList texts, CancellationToken ct = default) + { + var list = new List(); + foreach (var t in texts) list.Add(await EmbedAsync(t, ct)); + return list; + } + } + + [Fact] + public async Task Hybrid_search_surfaces_semantic_match_with_zero_keyword_overlap() + { + if (Conn is null) return; + var opts = Options(); + var uid = await SeedAsync(opts); + try + { + var embeddings = new DirectionalFakeEmbeddings(); + using (var prep = new AppDbContext(opts, new FakeCurrentUser())) + { + // "Weekly notes" gets a fruit-direction embedding (semantically related to the + // query); "Invoice March" points elsewhere. Neither subject contains "banana". + var near = await prep.Emails.FirstAsync(e => e.UserId == uid && e.Subject == "Weekly notes"); + near.Embedding = new Vector(await embeddings.EmbedAsync("tropical")); + var far = await prep.Emails.FirstAsync(e => e.UserId == uid && e.Subject == "Invoice March"); + far.Embedding = new Vector(await embeddings.EmbedAsync("finance")); + await prep.SaveChangesAsync(); + } + + using var ctx = new AppDbContext(opts, new FakeCurrentUser { UserId = uid }); + var res = await new SearchService(ctx, embeddings) + .SearchAsync(uid, GmailQueryParser.Parse("banana", 1, 10)); + + // Zero lexical hits for "banana" — hybrid must still surface the semantically + // nearest email, ranked first. + res.Items.Should().NotBeEmpty("semantic recall should fire with zero keyword overlap"); + res.Items[0].Subject.Should().Be("Weekly notes"); + } + finally { await CleanupAsync(opts, uid); } + } }