feat(search): hybrid semantic search via RRF fusion (RECOMMENDATIONS #4b)
CI / backend (pull_request) Successful in 1m0s
CI / frontend (pull_request) Successful in 14s
CI / format (pull_request) Successful in 50s
CI / db-tests (pull_request) Successful in 53s
Security / secrets (pull_request) Successful in 4s
Security / dependencies (pull_request) Successful in 56s
CI / backend (pull_request) Successful in 1m0s
CI / frontend (pull_request) Successful in 14s
CI / format (pull_request) Successful in 50s
CI / db-tests (pull_request) Successful in 53s
Security / secrets (pull_request) Successful in 4s
Security / dependencies (pull_request) Successful in 56s
The flagship slice (docs/discovery/05): when a free-text query runs and embeddings are available, fuse lexical top-50 (ts_rank_cd) with vector top-50 (pgvector cosine over the structured-filtered set) using Reciprocal Rank Fusion. Placed BEFORE the fuzzy fallback so queries with ZERO lexical hits still get semantic recall — found and fixed during review (the fuzzy path would otherwise swallow pure-semantic queries). Exact lexical hits keep winning (rank in both lists); deeper pages fall through to lexical paging; ANY failure (Ollama down, nothing embedded) silently degrades to the lexical/fuzzy path — AI can never break search. LiveDb test proves the recall win: a 'banana' query surfaces an email with no keyword overlap, ranked first, against real pgvector. 5 LiveDb + full suite green; format clean. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -123,4 +123,54 @@ public class LiveDbSearchTests
|
||||
}
|
||||
finally { await CleanupAsync(opts, uid); }
|
||||
}
|
||||
|
||||
private sealed class DirectionalFakeEmbeddings : IEmbeddingProvider
|
||||
{
|
||||
public bool IsAvailable => true;
|
||||
public Task<float[]> EmbedAsync(string text, CancellationToken ct = default)
|
||||
{
|
||||
// Deterministic "semantics": anything fruit-flavoured points one way, else the other.
|
||||
var v = new float[768];
|
||||
if (text.Contains("banana") || text.Contains("tropical")) v[0] = 1; else v[1] = 1;
|
||||
return Task.FromResult(v);
|
||||
}
|
||||
public async Task<IReadOnlyList<float[]>> EmbedBatchAsync(IReadOnlyList<string> texts, CancellationToken ct = default)
|
||||
{
|
||||
var list = new List<float[]>();
|
||||
foreach (var t in texts) list.Add(await EmbedAsync(t, ct));
|
||||
return list;
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Hybrid_search_surfaces_semantic_match_with_zero_keyword_overlap()
|
||||
{
|
||||
if (Conn is null) return;
|
||||
var opts = Options();
|
||||
var uid = await SeedAsync(opts);
|
||||
try
|
||||
{
|
||||
var embeddings = new DirectionalFakeEmbeddings();
|
||||
using (var prep = new AppDbContext(opts, new FakeCurrentUser()))
|
||||
{
|
||||
// "Weekly notes" gets a fruit-direction embedding (semantically related to the
|
||||
// query); "Invoice March" points elsewhere. Neither subject contains "banana".
|
||||
var near = await prep.Emails.FirstAsync(e => e.UserId == uid && e.Subject == "Weekly notes");
|
||||
near.Embedding = new Vector(await embeddings.EmbedAsync("tropical"));
|
||||
var far = await prep.Emails.FirstAsync(e => e.UserId == uid && e.Subject == "Invoice March");
|
||||
far.Embedding = new Vector(await embeddings.EmbedAsync("finance"));
|
||||
await prep.SaveChangesAsync();
|
||||
}
|
||||
|
||||
using var ctx = new AppDbContext(opts, new FakeCurrentUser { UserId = uid });
|
||||
var res = await new SearchService(ctx, embeddings)
|
||||
.SearchAsync(uid, GmailQueryParser.Parse("banana", 1, 10));
|
||||
|
||||
// Zero lexical hits for "banana" — hybrid must still surface the semantically
|
||||
// nearest email, ranked first.
|
||||
res.Items.Should().NotBeEmpty("semantic recall should fire with zero keyword overlap");
|
||||
res.Items[0].Subject.Should().Be("Weekly notes");
|
||||
}
|
||||
finally { await CleanupAsync(opts, uid); }
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user