feat(search): hybrid semantic search via RRF fusion (RECOMMENDATIONS #4b)
CI / backend (pull_request) Successful in 1m0s
CI / frontend (pull_request) Successful in 14s
CI / format (pull_request) Successful in 50s
CI / db-tests (pull_request) Successful in 53s
Security / secrets (pull_request) Successful in 4s
Security / dependencies (pull_request) Successful in 56s

The flagship slice (docs/discovery/05): when a free-text query runs and embeddings
are available, fuse lexical top-50 (ts_rank_cd) with vector top-50 (pgvector cosine
over the structured-filtered set) using Reciprocal Rank Fusion. Placed BEFORE the
fuzzy fallback so queries with ZERO lexical hits still get semantic recall — found
and fixed during review (the fuzzy path would otherwise swallow pure-semantic
queries). Exact lexical hits keep winning (rank in both lists); deeper pages fall
through to lexical paging; ANY failure (Ollama down, nothing embedded) silently
degrades to the lexical/fuzzy path — AI can never break search.

LiveDb test proves the recall win: a 'banana' query surfaces an email with no
keyword overlap, ranked first, against real pgvector. 5 LiveDb + full suite green;
format clean.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
cesnimda
2026-07-02 17:17:41 +02:00
parent 5191dd010f
commit 50bf6d977a
2 changed files with 139 additions and 1 deletions
@@ -123,4 +123,54 @@ public class LiveDbSearchTests
}
finally { await CleanupAsync(opts, uid); }
}
private sealed class DirectionalFakeEmbeddings : IEmbeddingProvider
{
public bool IsAvailable => true;
public Task<float[]> EmbedAsync(string text, CancellationToken ct = default)
{
// Deterministic "semantics": anything fruit-flavoured points one way, else the other.
var v = new float[768];
if (text.Contains("banana") || text.Contains("tropical")) v[0] = 1; else v[1] = 1;
return Task.FromResult(v);
}
public async Task<IReadOnlyList<float[]>> EmbedBatchAsync(IReadOnlyList<string> texts, CancellationToken ct = default)
{
var list = new List<float[]>();
foreach (var t in texts) list.Add(await EmbedAsync(t, ct));
return list;
}
}
[Fact]
public async Task Hybrid_search_surfaces_semantic_match_with_zero_keyword_overlap()
{
if (Conn is null) return;
var opts = Options();
var uid = await SeedAsync(opts);
try
{
var embeddings = new DirectionalFakeEmbeddings();
using (var prep = new AppDbContext(opts, new FakeCurrentUser()))
{
// "Weekly notes" gets a fruit-direction embedding (semantically related to the
// query); "Invoice March" points elsewhere. Neither subject contains "banana".
var near = await prep.Emails.FirstAsync(e => e.UserId == uid && e.Subject == "Weekly notes");
near.Embedding = new Vector(await embeddings.EmbedAsync("tropical"));
var far = await prep.Emails.FirstAsync(e => e.UserId == uid && e.Subject == "Invoice March");
far.Embedding = new Vector(await embeddings.EmbedAsync("finance"));
await prep.SaveChangesAsync();
}
using var ctx = new AppDbContext(opts, new FakeCurrentUser { UserId = uid });
var res = await new SearchService(ctx, embeddings)
.SearchAsync(uid, GmailQueryParser.Parse("banana", 1, 10));
// Zero lexical hits for "banana" — hybrid must still surface the semantically
// nearest email, ranked first.
res.Items.Should().NotBeEmpty("semantic recall should fire with zero keyword overlap");
res.Items[0].Subject.Should().Be("Weekly notes");
}
finally { await CleanupAsync(opts, uid); }
}
}