feat(search): hybrid semantic search via RRF fusion (RECOMMENDATIONS #4b)

The flagship slice (docs/discovery/05): when a free-text query runs and embeddings
are available, fuse lexical top-50 (ts_rank_cd) with vector top-50 (pgvector cosine
over the structured-filtered set) using Reciprocal Rank Fusion. Placed BEFORE the
fuzzy fallback so queries with ZERO lexical hits still get semantic recall — found
and fixed during review (the fuzzy path would otherwise swallow pure-semantic
queries). Exact lexical hits keep winning (rank in both lists); deeper pages fall
through to lexical paging; ANY failure (Ollama down, nothing embedded) silently
degrades to the lexical/fuzzy path — AI can never break search.

LiveDb test proves the recall win: a 'banana' query surfaces an email with no
keyword overlap, ranked first, against real pgvector. 5 LiveDb + full suite green;
format clean.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

.editorconfig

@@ -1,13 +0,0 @@
-# Root editor/formatter config. end_of_line=lf makes dotnet-format agree with
-# .gitattributes (eol=lf) — without this, format-on-Windows wants CRLF while git
-# stores LF, and the pre-commit/CI format gates flip-flop forever.
-root = true
-
-[*]
-end_of_line = lf
-insert_final_newline = true
-charset = utf-8
-
-[*.cs]
-indent_style = space
-indent_size = 4

docker-compose.yml

@@ -67,8 +67,6 @@ services:
       Ai__Mode: ${AI_MODE:-Disabled}
       # Points at the compose 'ollama' service when the ai profile is up; harmless otherwise.
       Ai__OllamaBaseUrl: ${OLLAMA_BASE_URL:-http://ollama:11434}
-      # OTLP export activates only when set (e.g. http://lgtm:4317 with the observability profile).
-      OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_ENDPOINT:-}
       # Dev mode shows the dev banner and caps the initial sync. Set DEV_MODE=true
       # and MAX_MESSAGES=1000 in deploy/.env to exercise it in this Docker setup.
       App__DevMode: ${DEV_MODE:-false}
@@ -111,16 +109,6 @@ services:
     #           count: all
     #           capabilities: [gpu]
 
-  # Observability (RECOMMENDATIONS #5): all-in-one Grafana+Tempo+Prometheus+Loki.
-  # Enable with: docker compose --profile observability up -d
-  # then set OTEL_ENDPOINT=http://lgtm:4317 in deploy/.env and restart the api.
-  # Grafana UI: http://localhost:3000 (admin/admin on first run).
-  lgtm:
-    image: grafana/otel-lgtm
-    profiles: ["observability"]
-    ports:
-      - "127.0.0.1:3000:3000"
-
   # Optional reverse proxy. Enable with: docker compose --profile proxy up
   nginx:
     image: nginx:alpine

src/InboxIntel.Api/InboxIntel.Api.csproj

@@ -15,11 +15,6 @@
     <PackageReference Include="Asp.Versioning.Mvc" Version="8.1.0" />
     <PackageReference Include="Asp.Versioning.Mvc.ApiExplorer" Version="8.1.0" />
     <PackageReference Include="FluentValidation.AspNetCore" Version="11.3.0" />
-    <PackageReference Include="Npgsql.OpenTelemetry" Version="10.0.3" />
-    <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.16.0" />
-    <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.16.0" />
-    <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.16.0" />
-    <PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.16.0" />
     <PackageReference Include="Serilog.AspNetCore" Version="8.0.1" />
     <PackageReference Include="Serilog.Sinks.Console" Version="5.0.1" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2" />

src/InboxIntel.Api/Program.cs

@@ -16,10 +16,6 @@ using Microsoft.AspNetCore.HttpOverrides;
 using Microsoft.AspNetCore.RateLimiting;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Options;
-using Npgsql;
-using OpenTelemetry.Metrics;
-using OpenTelemetry.Resources;
-using OpenTelemetry.Trace;
 using Serilog;
 
 var builder = WebApplication.CreateBuilder(args);
@@ -126,28 +122,6 @@ builder.Services.AddAuthentication(options =>
 
 builder.Services.AddAuthorization();
 
-// RECOMMENDATIONS #5: OpenTelemetry traces + metrics (ASP.NET, outbound HTTP, Npgsql).
-// The OTLP exporter only activates when Otel:Endpoint (or the standard
-// OTEL_EXPORTER_OTLP_ENDPOINT env var) is configured — zero overhead otherwise.
-// Logs stay on Serilog. Pair with the compose "observability" profile (grafana/otel-lgtm).
-var otlpEndpoint = builder.Configuration["Otel:Endpoint"]
-    ?? Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_ENDPOINT");
-if (!string.IsNullOrWhiteSpace(otlpEndpoint))
-{
-    builder.Services.AddOpenTelemetry()
-        .ConfigureResource(r => r.AddService("inboxintel-api"))
-        .WithTracing(t => t
-            .AddAspNetCoreInstrumentation()
-            .AddHttpClientInstrumentation()
-            .AddNpgsql()
-            .AddOtlpExporter(o => o.Endpoint = new Uri(otlpEndpoint)))
-        .WithMetrics(m => m
-            .AddAspNetCoreInstrumentation()
-            .AddHttpClientInstrumentation()
-            .AddNpgsqlInstrumentation()
-            .AddOtlpExporter(o => o.Endpoint = new Uri(otlpEndpoint)));
-}
-
 builder.Services.AddApiVersioning(o =>
 {
     o.DefaultApiVersion = new ApiVersion(1, 0);

src/InboxIntel.Application/DTOs/SearchDtos.cs

@@ -23,9 +23,4 @@ public record SearchRequestDto(
     bool? IsTrashed = null,
     string? GmailLabel = null,   // e.g. "SENT", "DRAFT", "SPAM"
     string? Category = null,     // EmailCategory name, e.g. "Finance"
-    long? MinSizeBytes = null,
-    // Keyset cursor for the date-ordered browse path (RECOMMENDATIONS #8): pass the last
-    // row's SentAtUtc+Id to fetch the next window without OFFSET (O(pageSize), not O(page)).
-    // When set, TotalCount is not recomputed (-1). Additive; offset paging still works.
-    DateTimeOffset? AfterSentAtUtc = null,
-    Guid? AfterId = null);
+    long? MinSizeBytes = null);

src/InboxIntel.Infrastructure/Migrations/20260630140733_InitialCreate.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using Microsoft.EntityFrameworkCore.Migrations;
 using NpgsqlTypes;
 

src/InboxIntel.Infrastructure/Migrations/20260630200946_AddUserDigestFields.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using Microsoft.EntityFrameworkCore.Migrations;
 
 #nullable disable

src/InboxIntel.Infrastructure/Migrations/20260630201607_AddUnsubscribeConfidence.cs

@@ -1,4 +1,4 @@
-using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Migrations;
 
 #nullable disable
 

src/InboxIntel.Infrastructure/Migrations/20260701192100_WeightSearchVectorSubjectBody.cs

@@ -1,4 +1,4 @@
-using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Migrations;
 using NpgsqlTypes;
 
 #nullable disable

src/InboxIntel.Infrastructure/Migrations/20260701203001_EnablePgTrgm.cs

@@ -1,4 +1,4 @@
-using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Migrations;
 
 #nullable disable
 

src/InboxIntel.Infrastructure/Migrations/20260701235112_TrigramIndexesSenderDomain.cs

@@ -1,4 +1,4 @@
-using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Migrations;
 
 #nullable disable
 

src/InboxIntel.Infrastructure/Migrations/20260702002304_AddEmbeddingColumn.cs

@@ -1,4 +1,4 @@
-using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Migrations;
 using Pgvector;
 
 #nullable disable

src/InboxIntel.Infrastructure/Persistence/AppDbContext.cs

@@ -47,20 +47,20 @@ public class AppDbContext : DbContext, IAppDbContext
         // its manual `WHERE UserId ==` clause cannot leak across tenants. Applied
         // uniformly to all user-scoped entities so EF sees no filtered/unfiltered
         // navigation mismatch. Bypassed when CurrentUserId is Guid.Empty (workers).
-        modelBuilder.Entity<Email>().HasQueryFilter("Tenant", e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
-        modelBuilder.Entity<Sender>().HasQueryFilter("Tenant", e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
-        modelBuilder.Entity<MailThread>().HasQueryFilter("Tenant", e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
-        modelBuilder.Entity<MailDomain>().HasQueryFilter("Tenant", e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
-        modelBuilder.Entity<Attachment>().HasQueryFilter("Tenant", e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
-        modelBuilder.Entity<Label>().HasQueryFilter("Tenant", e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
+        modelBuilder.Entity<Email>().HasQueryFilter(e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
+        modelBuilder.Entity<Sender>().HasQueryFilter(e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
+        modelBuilder.Entity<MailThread>().HasQueryFilter(e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
+        modelBuilder.Entity<MailDomain>().HasQueryFilter(e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
+        modelBuilder.Entity<Attachment>().HasQueryFilter(e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
+        modelBuilder.Entity<Label>().HasQueryFilter(e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
         // AUDIT M-6: EmailLabel is the required end of a relationship with the filtered Email
         // entity; without a matching filter EF warns on boot and joins could surface rows whose
         // parent is filtered out. Filter via the Email navigation so the pair is consistent.
-        modelBuilder.Entity<EmailLabel>().HasQueryFilter("Tenant", el => CurrentUserId == Guid.Empty || el.Email!.UserId == CurrentUserId);
-        modelBuilder.Entity<SyncState>().HasQueryFilter("Tenant", e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
-        modelBuilder.Entity<AnalyticsAggregate>().HasQueryFilter("Tenant", e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
-        modelBuilder.Entity<WidgetLayout>().HasQueryFilter("Tenant", e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
-        modelBuilder.Entity<UnsubscribeItem>().HasQueryFilter("Tenant", e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
+        modelBuilder.Entity<EmailLabel>().HasQueryFilter(el => CurrentUserId == Guid.Empty || el.Email!.UserId == CurrentUserId);
+        modelBuilder.Entity<SyncState>().HasQueryFilter(e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
+        modelBuilder.Entity<AnalyticsAggregate>().HasQueryFilter(e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
+        modelBuilder.Entity<WidgetLayout>().HasQueryFilter(e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
+        modelBuilder.Entity<UnsubscribeItem>().HasQueryFilter(e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
 
         // PostgreSQL full-text search: generated tsvector over subject + body with a
         // GIN index, maintained by the DB and read-only in code. Subject is weighted 'A'

src/InboxIntel.Infrastructure/Search/SearchService.cs

@@ -114,23 +114,9 @@ public class SearchService : ISearchService
             ranked = matched.OrderByDescending(e => e.SearchVector!.RankCoverDensity(EF.Functions.WebSearchToTsQuery("english", term)))
                             .ThenByDescending(e => e.SentAtUtc);
         else
-            ranked = matched.OrderByDescending(e => e.SentAtUtc).ThenByDescending(e => e.Id);
+            ranked = matched.OrderByDescending(e => e.SentAtUtc);
 
-        // Keyset (cursor) pagination for the browse path: O(pageSize) regardless of depth,
-        // vs OFFSET's O(page*pageSize). The (SentAtUtc, Id) pair with the Id tie-break above
-        // makes the ordering total, so windows never duplicate or skip rows.
-        IQueryable<Email> paged;
-        if (!hasFreeTextQuery && r is { AfterSentAtUtc: { } afterAt, AfterId: { } afterId })
-        {
-            paged = ranked
-                .Where(e => e.SentAtUtc < afterAt || (e.SentAtUtc == afterAt && e.Id.CompareTo(afterId) < 0))
-                .Take(r.PageSize);
-            total = -1; // not recomputed on cursor windows (that's the point)
-        }
-        else
-        {
-            paged = ranked.Skip((r.Page - 1) * r.PageSize).Take(r.PageSize);
-        }
+        var paged = ranked.Skip((r.Page - 1) * r.PageSize).Take(r.PageSize);
 
         // "Why this matched" ts_headline only for EXACT free-text hits (fuzzy/browse get no
         // highlight — a fuzzy hit has no literal match to headline). Unconditional projections

tests/InboxIntel.IntegrationTests/KeysetPaginationTests.cs

@@ -1,57 +0,0 @@
-using FluentAssertions;
-using InboxIntel.Application.Abstractions;
-using InboxIntel.Application.DTOs;
-using InboxIntel.Domain.Entities;
-using InboxIntel.Infrastructure.Persistence;
-using InboxIntel.Infrastructure.Search;
-using Microsoft.EntityFrameworkCore;
-using Xunit;
-
-namespace InboxIntel.IntegrationTests;
-
-/// <summary>
-/// RECOMMENDATIONS #8: keyset (cursor) pagination for the browse path — the window after a
-/// (SentAtUtc, Id) cursor returns the next rows with no duplicates/skips and no OFFSET scan.
-/// </summary>
-public class KeysetPaginationTests
-{
-    private sealed class FakeCurrentUser : ICurrentUser
-    {
-        public Guid UserId { get; set; }
-        public bool IsAuthenticated => UserId != Guid.Empty;
-    }
-
-    [Fact]
-    public async Task Cursor_window_continues_exactly_after_the_previous_page()
-    {
-        var user = Guid.NewGuid();
-        var opts = new DbContextOptionsBuilder<AppDbContext>()
-            .UseInMemoryDatabase(nameof(Cursor_window_continues_exactly_after_the_previous_page)).Options;
-
-        var baseline = DateTimeOffset.UtcNow;
-        using (var seed = new AppDbContext(opts, new FakeCurrentUser()))
-        {
-            var sender = new Sender { UserId = user, Address = "s@x.x" };
-            seed.Senders.Add(sender);
-            for (var i = 0; i < 5; i++)
-                seed.Emails.Add(new Email { UserId = user, GmailMessageId = $"m{i}", Sender = sender, SentAtUtc = baseline.AddMinutes(-i) });
-            await seed.SaveChangesAsync();
-        }
-
-        using var ctx = new AppDbContext(opts, new FakeCurrentUser { UserId = user });
-        var svc = new SearchService(ctx);
-
-        // First window via offset (page 1, size 2): m0, m1 (newest first).
-        var page1 = await svc.SearchAsync(user, new SearchRequestDto(null, null, null, null, null, null, null, false, 1, 2));
-        page1.Items.Select(i => i.GmailMessageId).Should().Equal("m0", "m1");
-
-        // Next window via cursor from the last row of page 1.
-        var last = page1.Items[^1];
-        var page2 = await svc.SearchAsync(user, new SearchRequestDto(
-            null, null, null, null, null, null, null, false, 1, 2,
-            AfterSentAtUtc: last.SentAtUtc, AfterId: last.Id));
-
-        page2.Items.Select(i => i.GmailMessageId).Should().Equal("m2", "m3"); // no dupes, no skips
-        page2.TotalCount.Should().Be(-1, "cursor windows skip the COUNT — that's the perf win");
-    }
-}