Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| e000332b95 | |||
| b79f35f40e | |||
| 89c89183da |
@@ -1,13 +0,0 @@
|
|||||||
# Root editor/formatter config. end_of_line=lf makes dotnet-format agree with
|
|
||||||
# .gitattributes (eol=lf) — without this, format-on-Windows wants CRLF while git
|
|
||||||
# stores LF, and the pre-commit/CI format gates flip-flop forever.
|
|
||||||
root = true
|
|
||||||
|
|
||||||
[*]
|
|
||||||
end_of_line = lf
|
|
||||||
insert_final_newline = true
|
|
||||||
charset = utf-8
|
|
||||||
|
|
||||||
[*.cs]
|
|
||||||
indent_style = space
|
|
||||||
indent_size = 4
|
|
||||||
@@ -52,3 +52,21 @@ jobs:
|
|||||||
# (Vite/PostCSS/etc.) shouldn't block a merge.
|
# (Vite/PostCSS/etc.) shouldn't block a merge.
|
||||||
working-directory: frontend
|
working-directory: frontend
|
||||||
run: npm audit --omit=dev --audit-level=high
|
run: npm audit --omit=dev --audit-level=high
|
||||||
|
|
||||||
|
# RECOMMENDATIONS #9: SAST. Semgrep community rules for C#/JS + OWASP/secrets patterns —
|
||||||
|
# catches injection/crypto-misuse classes the other gates (gitleaks, dep-audit, tests)
|
||||||
|
# don't look for. Advisory at first (not a required check); promote once tuned.
|
||||||
|
sast:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
# The runner image lacks pip, and a semgrep job-container lacks the node that
|
||||||
|
# actions/checkout needs — so install pip via apt on the standard image.
|
||||||
|
- name: Install semgrep
|
||||||
|
run: |
|
||||||
|
sudo apt-get update -qq && sudo apt-get install -y -qq python3-pip pipx
|
||||||
|
pipx install semgrep
|
||||||
|
- name: Semgrep scan
|
||||||
|
run: |
|
||||||
|
export PATH="$HOME/.local/bin:$PATH"
|
||||||
|
semgrep scan --config p/csharp --config p/javascript --config p/security-audit --exclude 'frontend/dist' --exclude '**/bin' --exclude '**/obj' --error --quiet
|
||||||
|
|||||||
@@ -67,8 +67,6 @@ services:
|
|||||||
Ai__Mode: ${AI_MODE:-Disabled}
|
Ai__Mode: ${AI_MODE:-Disabled}
|
||||||
# Points at the compose 'ollama' service when the ai profile is up; harmless otherwise.
|
# Points at the compose 'ollama' service when the ai profile is up; harmless otherwise.
|
||||||
Ai__OllamaBaseUrl: ${OLLAMA_BASE_URL:-http://ollama:11434}
|
Ai__OllamaBaseUrl: ${OLLAMA_BASE_URL:-http://ollama:11434}
|
||||||
# OTLP export activates only when set (e.g. http://lgtm:4317 with the observability profile).
|
|
||||||
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_ENDPOINT:-}
|
|
||||||
# Dev mode shows the dev banner and caps the initial sync. Set DEV_MODE=true
|
# Dev mode shows the dev banner and caps the initial sync. Set DEV_MODE=true
|
||||||
# and MAX_MESSAGES=1000 in deploy/.env to exercise it in this Docker setup.
|
# and MAX_MESSAGES=1000 in deploy/.env to exercise it in this Docker setup.
|
||||||
App__DevMode: ${DEV_MODE:-false}
|
App__DevMode: ${DEV_MODE:-false}
|
||||||
@@ -111,16 +109,6 @@ services:
|
|||||||
# count: all
|
# count: all
|
||||||
# capabilities: [gpu]
|
# capabilities: [gpu]
|
||||||
|
|
||||||
# Observability (RECOMMENDATIONS #5): all-in-one Grafana+Tempo+Prometheus+Loki.
|
|
||||||
# Enable with: docker compose --profile observability up -d
|
|
||||||
# then set OTEL_ENDPOINT=http://lgtm:4317 in deploy/.env and restart the api.
|
|
||||||
# Grafana UI: http://localhost:3000 (admin/admin on first run).
|
|
||||||
lgtm:
|
|
||||||
image: grafana/otel-lgtm
|
|
||||||
profiles: ["observability"]
|
|
||||||
ports:
|
|
||||||
- "127.0.0.1:3000:3000"
|
|
||||||
|
|
||||||
# Optional reverse proxy. Enable with: docker compose --profile proxy up
|
# Optional reverse proxy. Enable with: docker compose --profile proxy up
|
||||||
nginx:
|
nginx:
|
||||||
image: nginx:alpine
|
image: nginx:alpine
|
||||||
|
|||||||
@@ -15,11 +15,6 @@
|
|||||||
<PackageReference Include="Asp.Versioning.Mvc" Version="8.1.0" />
|
<PackageReference Include="Asp.Versioning.Mvc" Version="8.1.0" />
|
||||||
<PackageReference Include="Asp.Versioning.Mvc.ApiExplorer" Version="8.1.0" />
|
<PackageReference Include="Asp.Versioning.Mvc.ApiExplorer" Version="8.1.0" />
|
||||||
<PackageReference Include="FluentValidation.AspNetCore" Version="11.3.0" />
|
<PackageReference Include="FluentValidation.AspNetCore" Version="11.3.0" />
|
||||||
<PackageReference Include="Npgsql.OpenTelemetry" Version="10.0.3" />
|
|
||||||
<PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.16.0" />
|
|
||||||
<PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.16.0" />
|
|
||||||
<PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.16.0" />
|
|
||||||
<PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.16.0" />
|
|
||||||
<PackageReference Include="Serilog.AspNetCore" Version="8.0.1" />
|
<PackageReference Include="Serilog.AspNetCore" Version="8.0.1" />
|
||||||
<PackageReference Include="Serilog.Sinks.Console" Version="5.0.1" />
|
<PackageReference Include="Serilog.Sinks.Console" Version="5.0.1" />
|
||||||
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2" />
|
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2" />
|
||||||
|
|||||||
@@ -16,10 +16,6 @@ using Microsoft.AspNetCore.HttpOverrides;
|
|||||||
using Microsoft.AspNetCore.RateLimiting;
|
using Microsoft.AspNetCore.RateLimiting;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using Microsoft.Extensions.Options;
|
using Microsoft.Extensions.Options;
|
||||||
using Npgsql;
|
|
||||||
using OpenTelemetry.Metrics;
|
|
||||||
using OpenTelemetry.Resources;
|
|
||||||
using OpenTelemetry.Trace;
|
|
||||||
using Serilog;
|
using Serilog;
|
||||||
|
|
||||||
var builder = WebApplication.CreateBuilder(args);
|
var builder = WebApplication.CreateBuilder(args);
|
||||||
@@ -126,28 +122,6 @@ builder.Services.AddAuthentication(options =>
|
|||||||
|
|
||||||
builder.Services.AddAuthorization();
|
builder.Services.AddAuthorization();
|
||||||
|
|
||||||
// RECOMMENDATIONS #5: OpenTelemetry traces + metrics (ASP.NET, outbound HTTP, Npgsql).
|
|
||||||
// The OTLP exporter only activates when Otel:Endpoint (or the standard
|
|
||||||
// OTEL_EXPORTER_OTLP_ENDPOINT env var) is configured — zero overhead otherwise.
|
|
||||||
// Logs stay on Serilog. Pair with the compose "observability" profile (grafana/otel-lgtm).
|
|
||||||
var otlpEndpoint = builder.Configuration["Otel:Endpoint"]
|
|
||||||
?? Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_ENDPOINT");
|
|
||||||
if (!string.IsNullOrWhiteSpace(otlpEndpoint))
|
|
||||||
{
|
|
||||||
builder.Services.AddOpenTelemetry()
|
|
||||||
.ConfigureResource(r => r.AddService("inboxintel-api"))
|
|
||||||
.WithTracing(t => t
|
|
||||||
.AddAspNetCoreInstrumentation()
|
|
||||||
.AddHttpClientInstrumentation()
|
|
||||||
.AddNpgsql()
|
|
||||||
.AddOtlpExporter(o => o.Endpoint = new Uri(otlpEndpoint)))
|
|
||||||
.WithMetrics(m => m
|
|
||||||
.AddAspNetCoreInstrumentation()
|
|
||||||
.AddHttpClientInstrumentation()
|
|
||||||
.AddNpgsqlInstrumentation()
|
|
||||||
.AddOtlpExporter(o => o.Endpoint = new Uri(otlpEndpoint)));
|
|
||||||
}
|
|
||||||
|
|
||||||
builder.Services.AddApiVersioning(o =>
|
builder.Services.AddApiVersioning(o =>
|
||||||
{
|
{
|
||||||
o.DefaultApiVersion = new ApiVersion(1, 0);
|
o.DefaultApiVersion = new ApiVersion(1, 0);
|
||||||
|
|||||||
@@ -23,9 +23,4 @@ public record SearchRequestDto(
|
|||||||
bool? IsTrashed = null,
|
bool? IsTrashed = null,
|
||||||
string? GmailLabel = null, // e.g. "SENT", "DRAFT", "SPAM"
|
string? GmailLabel = null, // e.g. "SENT", "DRAFT", "SPAM"
|
||||||
string? Category = null, // EmailCategory name, e.g. "Finance"
|
string? Category = null, // EmailCategory name, e.g. "Finance"
|
||||||
long? MinSizeBytes = null,
|
long? MinSizeBytes = null);
|
||||||
// Keyset cursor for the date-ordered browse path (RECOMMENDATIONS #8): pass the last
|
|
||||||
// row's SentAtUtc+Id to fetch the next window without OFFSET (O(pageSize), not O(page)).
|
|
||||||
// When set, TotalCount is not recomputed (-1). Additive; offset paging still works.
|
|
||||||
DateTimeOffset? AfterSentAtUtc = null,
|
|
||||||
Guid? AfterId = null);
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
using System;
|
using System;
|
||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using NpgsqlTypes;
|
using NpgsqlTypes;
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
using System;
|
using System;
|
||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -1,4 +1,4 @@
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using NpgsqlTypes;
|
using NpgsqlTypes;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -1,4 +1,4 @@
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Pgvector;
|
using Pgvector;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|||||||
@@ -47,20 +47,20 @@ public class AppDbContext : DbContext, IAppDbContext
|
|||||||
// its manual `WHERE UserId ==` clause cannot leak across tenants. Applied
|
// its manual `WHERE UserId ==` clause cannot leak across tenants. Applied
|
||||||
// uniformly to all user-scoped entities so EF sees no filtered/unfiltered
|
// uniformly to all user-scoped entities so EF sees no filtered/unfiltered
|
||||||
// navigation mismatch. Bypassed when CurrentUserId is Guid.Empty (workers).
|
// navigation mismatch. Bypassed when CurrentUserId is Guid.Empty (workers).
|
||||||
modelBuilder.Entity<Email>().HasQueryFilter("Tenant", e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
|
modelBuilder.Entity<Email>().HasQueryFilter(e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
|
||||||
modelBuilder.Entity<Sender>().HasQueryFilter("Tenant", e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
|
modelBuilder.Entity<Sender>().HasQueryFilter(e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
|
||||||
modelBuilder.Entity<MailThread>().HasQueryFilter("Tenant", e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
|
modelBuilder.Entity<MailThread>().HasQueryFilter(e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
|
||||||
modelBuilder.Entity<MailDomain>().HasQueryFilter("Tenant", e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
|
modelBuilder.Entity<MailDomain>().HasQueryFilter(e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
|
||||||
modelBuilder.Entity<Attachment>().HasQueryFilter("Tenant", e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
|
modelBuilder.Entity<Attachment>().HasQueryFilter(e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
|
||||||
modelBuilder.Entity<Label>().HasQueryFilter("Tenant", e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
|
modelBuilder.Entity<Label>().HasQueryFilter(e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
|
||||||
// AUDIT M-6: EmailLabel is the required end of a relationship with the filtered Email
|
// AUDIT M-6: EmailLabel is the required end of a relationship with the filtered Email
|
||||||
// entity; without a matching filter EF warns on boot and joins could surface rows whose
|
// entity; without a matching filter EF warns on boot and joins could surface rows whose
|
||||||
// parent is filtered out. Filter via the Email navigation so the pair is consistent.
|
// parent is filtered out. Filter via the Email navigation so the pair is consistent.
|
||||||
modelBuilder.Entity<EmailLabel>().HasQueryFilter("Tenant", el => CurrentUserId == Guid.Empty || el.Email!.UserId == CurrentUserId);
|
modelBuilder.Entity<EmailLabel>().HasQueryFilter(el => CurrentUserId == Guid.Empty || el.Email!.UserId == CurrentUserId);
|
||||||
modelBuilder.Entity<SyncState>().HasQueryFilter("Tenant", e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
|
modelBuilder.Entity<SyncState>().HasQueryFilter(e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
|
||||||
modelBuilder.Entity<AnalyticsAggregate>().HasQueryFilter("Tenant", e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
|
modelBuilder.Entity<AnalyticsAggregate>().HasQueryFilter(e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
|
||||||
modelBuilder.Entity<WidgetLayout>().HasQueryFilter("Tenant", e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
|
modelBuilder.Entity<WidgetLayout>().HasQueryFilter(e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
|
||||||
modelBuilder.Entity<UnsubscribeItem>().HasQueryFilter("Tenant", e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
|
modelBuilder.Entity<UnsubscribeItem>().HasQueryFilter(e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
|
||||||
|
|
||||||
// PostgreSQL full-text search: generated tsvector over subject + body with a
|
// PostgreSQL full-text search: generated tsvector over subject + body with a
|
||||||
// GIN index, maintained by the DB and read-only in code. Subject is weighted 'A'
|
// GIN index, maintained by the DB and read-only in code. Subject is weighted 'A'
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ using InboxIntel.Domain.Entities;
|
|||||||
using InboxIntel.Domain.Enums;
|
using InboxIntel.Domain.Enums;
|
||||||
using InboxIntel.Infrastructure.Persistence;
|
using InboxIntel.Infrastructure.Persistence;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using Pgvector.EntityFrameworkCore;
|
|
||||||
|
|
||||||
namespace InboxIntel.Infrastructure.Search;
|
namespace InboxIntel.Infrastructure.Search;
|
||||||
|
|
||||||
@@ -19,13 +18,7 @@ namespace InboxIntel.Infrastructure.Search;
|
|||||||
public class SearchService : ISearchService
|
public class SearchService : ISearchService
|
||||||
{
|
{
|
||||||
private readonly AppDbContext _db;
|
private readonly AppDbContext _db;
|
||||||
private readonly IEmbeddingProvider? _embeddings;
|
public SearchService(AppDbContext db) => _db = db;
|
||||||
|
|
||||||
public SearchService(AppDbContext db, IEmbeddingProvider? embeddings = null)
|
|
||||||
{
|
|
||||||
_db = db;
|
|
||||||
_embeddings = embeddings;
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task<PagedResult<EmailSummaryDto>> SearchAsync(Guid userId, SearchRequestDto r, CancellationToken ct = default)
|
public async Task<PagedResult<EmailSummaryDto>> SearchAsync(Guid userId, SearchRequestDto r, CancellationToken ct = default)
|
||||||
{
|
{
|
||||||
@@ -80,18 +73,6 @@ public class SearchService : ISearchService
|
|||||||
|
|
||||||
var total = await matched.CountAsync(ct);
|
var total = await matched.CountAsync(ct);
|
||||||
|
|
||||||
// Hybrid semantic fusion (docs/discovery/05): when embeddings are available, fuse
|
|
||||||
// lexical top-K with vector top-K via Reciprocal Rank Fusion. Runs BEFORE the fuzzy
|
|
||||||
// fallback so a query with ZERO lexical hits (pure semantic recall — "gym receipt"
|
|
||||||
// phrased differently) still surfaces results. Exact lexical hits keep winning (they
|
|
||||||
// rank in both lists). Deeper pages fall through to lexical paging; any failure
|
|
||||||
// (Ollama down, nothing embedded yet) silently degrades to the lexical/fuzzy path.
|
|
||||||
if (hasFreeTextQuery && _embeddings is { IsAvailable: true })
|
|
||||||
{
|
|
||||||
var hybrid = await TryHybridAsync(structured, matched, term, total, r, ct);
|
|
||||||
if (hybrid is not null) return hybrid;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fuzzy/typo fallback: ONLY when a free-text search found nothing exact. word_similarity
|
// Fuzzy/typo fallback: ONLY when a free-text search found nothing exact. word_similarity
|
||||||
// with an explicit 0.3 threshold — pg_trgm's default 0.6 misses real typos
|
// with an explicit 0.3 threshold — pg_trgm's default 0.6 misses real typos
|
||||||
// ("recieved" -> "received" scores ~0.39). Rare path, so the (non-indexed) scan over the
|
// ("recieved" -> "received" scores ~0.39). Rare path, so the (non-indexed) scan over the
|
||||||
@@ -114,23 +95,9 @@ public class SearchService : ISearchService
|
|||||||
ranked = matched.OrderByDescending(e => e.SearchVector!.RankCoverDensity(EF.Functions.WebSearchToTsQuery("english", term)))
|
ranked = matched.OrderByDescending(e => e.SearchVector!.RankCoverDensity(EF.Functions.WebSearchToTsQuery("english", term)))
|
||||||
.ThenByDescending(e => e.SentAtUtc);
|
.ThenByDescending(e => e.SentAtUtc);
|
||||||
else
|
else
|
||||||
ranked = matched.OrderByDescending(e => e.SentAtUtc).ThenByDescending(e => e.Id);
|
ranked = matched.OrderByDescending(e => e.SentAtUtc);
|
||||||
|
|
||||||
// Keyset (cursor) pagination for the browse path: O(pageSize) regardless of depth,
|
var paged = ranked.Skip((r.Page - 1) * r.PageSize).Take(r.PageSize);
|
||||||
// vs OFFSET's O(page*pageSize). The (SentAtUtc, Id) pair with the Id tie-break above
|
|
||||||
// makes the ordering total, so windows never duplicate or skip rows.
|
|
||||||
IQueryable<Email> paged;
|
|
||||||
if (!hasFreeTextQuery && r is { AfterSentAtUtc: { } afterAt, AfterId: { } afterId })
|
|
||||||
{
|
|
||||||
paged = ranked
|
|
||||||
.Where(e => e.SentAtUtc < afterAt || (e.SentAtUtc == afterAt && e.Id.CompareTo(afterId) < 0))
|
|
||||||
.Take(r.PageSize);
|
|
||||||
total = -1; // not recomputed on cursor windows (that's the point)
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
paged = ranked.Skip((r.Page - 1) * r.PageSize).Take(r.PageSize);
|
|
||||||
}
|
|
||||||
|
|
||||||
// "Why this matched" ts_headline only for EXACT free-text hits (fuzzy/browse get no
|
// "Why this matched" ts_headline only for EXACT free-text hits (fuzzy/browse get no
|
||||||
// highlight — a fuzzy hit has no literal match to headline). Unconditional projections
|
// highlight — a fuzzy hit has no literal match to headline). Unconditional projections
|
||||||
@@ -170,73 +137,4 @@ public class SearchService : ISearchService
|
|||||||
TotalCount = total
|
TotalCount = total
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
private const int HybridK = 50; // candidates taken from each layer
|
|
||||||
private const int RrfConstant = 60; // standard RRF dampening constant
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// RRF fusion of lexical and vector candidates. Returns null when the requested page
|
|
||||||
/// lies beyond the fused window or anything fails — caller falls back to lexical.
|
|
||||||
/// </summary>
|
|
||||||
private async Task<PagedResult<EmailSummaryDto>?> TryHybridAsync(
|
|
||||||
IQueryable<Email> structured, IQueryable<Email> lexical, string term, int lexicalTotal,
|
|
||||||
SearchRequestDto r, CancellationToken ct)
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
var lexIds = await lexical
|
|
||||||
.OrderByDescending(e => e.SearchVector!.RankCoverDensity(EF.Functions.WebSearchToTsQuery("english", term)))
|
|
||||||
.ThenByDescending(e => e.SentAtUtc)
|
|
||||||
.Take(HybridK).Select(e => e.Id).ToListAsync(ct);
|
|
||||||
|
|
||||||
var queryVec = await _embeddings!.EmbedAsync(term, ct);
|
|
||||||
List<Guid> vecIds = new();
|
|
||||||
if (queryVec.Length > 0)
|
|
||||||
{
|
|
||||||
var qv = new Pgvector.Vector(queryVec);
|
|
||||||
vecIds = await structured
|
|
||||||
.Where(e => e.Embedding != null)
|
|
||||||
.OrderBy(e => e.Embedding!.CosineDistance(qv))
|
|
||||||
.Take(HybridK).Select(e => e.Id).ToListAsync(ct);
|
|
||||||
}
|
|
||||||
if (vecIds.Count == 0) return null; // nothing embedded yet → lexical path
|
|
||||||
|
|
||||||
var scores = new Dictionary<Guid, double>();
|
|
||||||
for (var i = 0; i < lexIds.Count; i++)
|
|
||||||
scores[lexIds[i]] = scores.GetValueOrDefault(lexIds[i]) + 1.0 / (RrfConstant + i + 1);
|
|
||||||
for (var i = 0; i < vecIds.Count; i++)
|
|
||||||
scores[vecIds[i]] = scores.GetValueOrDefault(vecIds[i]) + 1.0 / (RrfConstant + i + 1);
|
|
||||||
|
|
||||||
var fused = scores.OrderByDescending(kv => kv.Value).Select(kv => kv.Key).ToList();
|
|
||||||
var pageIds = fused.Skip((r.Page - 1) * r.PageSize).Take(r.PageSize).ToList();
|
|
||||||
if (pageIds.Count == 0 && r.Page > 1) return null; // deep page → lexical paging
|
|
||||||
|
|
||||||
var headlineOpts =
|
|
||||||
$"StartSel={(char)0xE000},StopSel={(char)0xE001},MaxWords=16,MinWords=5,ShortWord=2,HighlightAll=false";
|
|
||||||
var rows = await _db.Emails.AsNoTracking()
|
|
||||||
.Where(e => pageIds.Contains(e.Id))
|
|
||||||
.Select(e => new EmailSummaryDto(
|
|
||||||
e.Id, e.GmailMessageId, e.Subject, e.Snippet,
|
|
||||||
e.Sender!.Address, e.Sender.DisplayName, e.SentAtUtc,
|
|
||||||
e.IsUnread, e.IsStarred, e.HasAttachments, e.SizeEstimateBytes, e.Category,
|
|
||||||
e.HasListUnsubscribe, e.SupportsOneClickUnsubscribe,
|
|
||||||
EF.Functions.WebSearchToTsQuery("english", term).GetResultHeadline("english", e.BodyText ?? "", headlineOpts)))
|
|
||||||
.ToListAsync(ct);
|
|
||||||
var byId = rows.ToDictionary(x => x.Id);
|
|
||||||
var items = pageIds.Where(byId.ContainsKey).Select(id => byId[id]).ToList();
|
|
||||||
|
|
||||||
return new PagedResult<EmailSummaryDto>
|
|
||||||
{
|
|
||||||
Items = items,
|
|
||||||
Page = r.Page,
|
|
||||||
PageSize = r.PageSize,
|
|
||||||
// Semantic recall can exceed the lexical match count.
|
|
||||||
TotalCount = Math.Max(lexicalTotal, fused.Count)
|
|
||||||
};
|
|
||||||
}
|
|
||||||
catch
|
|
||||||
{
|
|
||||||
return null; // AI must never break search — degrade to lexical
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,57 +0,0 @@
|
|||||||
using FluentAssertions;
|
|
||||||
using InboxIntel.Application.Abstractions;
|
|
||||||
using InboxIntel.Application.DTOs;
|
|
||||||
using InboxIntel.Domain.Entities;
|
|
||||||
using InboxIntel.Infrastructure.Persistence;
|
|
||||||
using InboxIntel.Infrastructure.Search;
|
|
||||||
using Microsoft.EntityFrameworkCore;
|
|
||||||
using Xunit;
|
|
||||||
|
|
||||||
namespace InboxIntel.IntegrationTests;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// RECOMMENDATIONS #8: keyset (cursor) pagination for the browse path — the window after a
|
|
||||||
/// (SentAtUtc, Id) cursor returns the next rows with no duplicates/skips and no OFFSET scan.
|
|
||||||
/// </summary>
|
|
||||||
public class KeysetPaginationTests
|
|
||||||
{
|
|
||||||
private sealed class FakeCurrentUser : ICurrentUser
|
|
||||||
{
|
|
||||||
public Guid UserId { get; set; }
|
|
||||||
public bool IsAuthenticated => UserId != Guid.Empty;
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async Task Cursor_window_continues_exactly_after_the_previous_page()
|
|
||||||
{
|
|
||||||
var user = Guid.NewGuid();
|
|
||||||
var opts = new DbContextOptionsBuilder<AppDbContext>()
|
|
||||||
.UseInMemoryDatabase(nameof(Cursor_window_continues_exactly_after_the_previous_page)).Options;
|
|
||||||
|
|
||||||
var baseline = DateTimeOffset.UtcNow;
|
|
||||||
using (var seed = new AppDbContext(opts, new FakeCurrentUser()))
|
|
||||||
{
|
|
||||||
var sender = new Sender { UserId = user, Address = "s@x.x" };
|
|
||||||
seed.Senders.Add(sender);
|
|
||||||
for (var i = 0; i < 5; i++)
|
|
||||||
seed.Emails.Add(new Email { UserId = user, GmailMessageId = $"m{i}", Sender = sender, SentAtUtc = baseline.AddMinutes(-i) });
|
|
||||||
await seed.SaveChangesAsync();
|
|
||||||
}
|
|
||||||
|
|
||||||
using var ctx = new AppDbContext(opts, new FakeCurrentUser { UserId = user });
|
|
||||||
var svc = new SearchService(ctx);
|
|
||||||
|
|
||||||
// First window via offset (page 1, size 2): m0, m1 (newest first).
|
|
||||||
var page1 = await svc.SearchAsync(user, new SearchRequestDto(null, null, null, null, null, null, null, false, 1, 2));
|
|
||||||
page1.Items.Select(i => i.GmailMessageId).Should().Equal("m0", "m1");
|
|
||||||
|
|
||||||
// Next window via cursor from the last row of page 1.
|
|
||||||
var last = page1.Items[^1];
|
|
||||||
var page2 = await svc.SearchAsync(user, new SearchRequestDto(
|
|
||||||
null, null, null, null, null, null, null, false, 1, 2,
|
|
||||||
AfterSentAtUtc: last.SentAtUtc, AfterId: last.Id));
|
|
||||||
|
|
||||||
page2.Items.Select(i => i.GmailMessageId).Should().Equal("m2", "m3"); // no dupes, no skips
|
|
||||||
page2.TotalCount.Should().Be(-1, "cursor windows skip the COUNT — that's the perf win");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -123,54 +123,4 @@ public class LiveDbSearchTests
|
|||||||
}
|
}
|
||||||
finally { await CleanupAsync(opts, uid); }
|
finally { await CleanupAsync(opts, uid); }
|
||||||
}
|
}
|
||||||
|
|
||||||
private sealed class DirectionalFakeEmbeddings : IEmbeddingProvider
|
|
||||||
{
|
|
||||||
public bool IsAvailable => true;
|
|
||||||
public Task<float[]> EmbedAsync(string text, CancellationToken ct = default)
|
|
||||||
{
|
|
||||||
// Deterministic "semantics": anything fruit-flavoured points one way, else the other.
|
|
||||||
var v = new float[768];
|
|
||||||
if (text.Contains("banana") || text.Contains("tropical")) v[0] = 1; else v[1] = 1;
|
|
||||||
return Task.FromResult(v);
|
|
||||||
}
|
|
||||||
public async Task<IReadOnlyList<float[]>> EmbedBatchAsync(IReadOnlyList<string> texts, CancellationToken ct = default)
|
|
||||||
{
|
|
||||||
var list = new List<float[]>();
|
|
||||||
foreach (var t in texts) list.Add(await EmbedAsync(t, ct));
|
|
||||||
return list;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public async Task Hybrid_search_surfaces_semantic_match_with_zero_keyword_overlap()
|
|
||||||
{
|
|
||||||
if (Conn is null) return;
|
|
||||||
var opts = Options();
|
|
||||||
var uid = await SeedAsync(opts);
|
|
||||||
try
|
|
||||||
{
|
|
||||||
var embeddings = new DirectionalFakeEmbeddings();
|
|
||||||
using (var prep = new AppDbContext(opts, new FakeCurrentUser()))
|
|
||||||
{
|
|
||||||
// "Weekly notes" gets a fruit-direction embedding (semantically related to the
|
|
||||||
// query); "Invoice March" points elsewhere. Neither subject contains "banana".
|
|
||||||
var near = await prep.Emails.FirstAsync(e => e.UserId == uid && e.Subject == "Weekly notes");
|
|
||||||
near.Embedding = new Vector(await embeddings.EmbedAsync("tropical"));
|
|
||||||
var far = await prep.Emails.FirstAsync(e => e.UserId == uid && e.Subject == "Invoice March");
|
|
||||||
far.Embedding = new Vector(await embeddings.EmbedAsync("finance"));
|
|
||||||
await prep.SaveChangesAsync();
|
|
||||||
}
|
|
||||||
|
|
||||||
using var ctx = new AppDbContext(opts, new FakeCurrentUser { UserId = uid });
|
|
||||||
var res = await new SearchService(ctx, embeddings)
|
|
||||||
.SearchAsync(uid, GmailQueryParser.Parse("banana", 1, 10));
|
|
||||||
|
|
||||||
// Zero lexical hits for "banana" — hybrid must still surface the semantically
|
|
||||||
// nearest email, ranked first.
|
|
||||||
res.Items.Should().NotBeEmpty("semantic recall should fire with zero keyword overlap");
|
|
||||||
res.Items[0].Subject.Should().Be("Weekly notes");
|
|
||||||
}
|
|
||||||
finally { await CleanupAsync(opts, uid); }
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user