Compare commits

..

9 Commits

Author SHA1 Message Date
cesnimda 01fb74eb38 perf(search): keyset pagination + fix the format-gate line-ending flip-flop
CI / backend (pull_request) Successful in 1m18s
CI / frontend (pull_request) Successful in 22s
CI / format (pull_request) Successful in 1m8s
CI / db-tests (pull_request) Successful in 1m13s
Security / secrets (pull_request) Successful in 5s
Security / dependencies (pull_request) Successful in 1m7s
RECOMMENDATIONS #8: additive AfterSentAtUtc+AfterId cursor — browse windows fetch
via a (SentAtUtc, Id) keyset filter, O(pageSize) at any depth, COUNT skipped
(TotalCount=-1). Adds the Id tie-break to browse ordering (fixes latent
duplicate/skip on equal timestamps). Verified: InMemory continuation test + a
live-Postgres throwaway proving the Guid cursor comparison translates (removed).

Also root-causes the recurring pre-commit/CI format failures: dotnet-format on
Windows defaulted to CRLF while .gitattributes stores LF — the two fought forever.
Adds .editorconfig (end_of_line=lf etc.) so every tool agrees, and normalises the
straggler files. This bug bit three separate commits before being diagnosed.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-02 17:53:18 +02:00
cesnimda b7d4b87d75 feat(search): hybrid semantic search (RRF) (#34)
CI / backend (push) Successful in 59s
CI / frontend (push) Successful in 15s
CI / format (push) Successful in 56s
CI / db-tests (push) Successful in 1m4s
Deploy Staging / deploy (push) Successful in 30s
Security / secrets (push) Successful in 4s
Security / dependencies (push) Successful in 1m4s
CI / backend (pull_request) Successful in 1m9s
CI / frontend (pull_request) Successful in 31s
CI / format (pull_request) Successful in 1m4s
CI / db-tests (pull_request) Successful in 1m5s
Security / secrets (pull_request) Successful in 5s
Security / dependencies (pull_request) Successful in 1m5s
2026-07-02 17:36:27 +02:00
cesnimda fb6b89b6dc feat(ops): OpenTelemetry + observability profile (#33)
CI / backend (push) Successful in 1m12s
CI / frontend (push) Successful in 19s
CI / format (push) Successful in 1m3s
CI / db-tests (push) Successful in 1m9s
Deploy Staging / deploy (push) Successful in 48s
CI / backend (pull_request) Successful in 1m1s
CI / frontend (pull_request) Successful in 16s
CI / format (pull_request) Successful in 55s
CI / db-tests (pull_request) Successful in 59s
Security / secrets (push) Successful in 4s
Security / dependencies (push) Successful in 1m3s
Security / secrets (pull_request) Successful in 4s
Security / dependencies (pull_request) Successful in 1m1s
2026-07-02 17:32:35 +02:00
cesnimda 402628a3a7 refactor(db): named tenant query filters (#31)
CI / backend (push) Successful in 51s
CI / frontend (push) Successful in 12s
CI / format (push) Successful in 50s
CI / db-tests (push) Successful in 52s
Deploy Staging / deploy (push) Successful in 25s
Security / secrets (push) Successful in 4s
Security / dependencies (push) Successful in 1m18s
CI / backend (pull_request) Successful in 1m9s
CI / frontend (pull_request) Successful in 23s
CI / format (pull_request) Successful in 1m2s
CI / db-tests (pull_request) Successful in 1m11s
Security / secrets (pull_request) Successful in 6s
Security / dependencies (pull_request) Successful in 1m10s
2026-07-02 17:24:21 +02:00
cesnimda 5191dd010f feat(ai): semantic-search backfill + Ollama profile (#30)
CI / backend (push) Successful in 55s
CI / frontend (push) Successful in 12s
CI / format (push) Successful in 48s
CI / db-tests (push) Successful in 55s
Deploy Staging / deploy (push) Successful in 42s
CI / backend (pull_request) Successful in 54s
CI / frontend (pull_request) Successful in 13s
CI / format (pull_request) Successful in 49s
CI / db-tests (pull_request) Successful in 56s
Security / secrets (push) Successful in 4s
Security / dependencies (push) Successful in 58s
Security / secrets (pull_request) Successful in 4s
Security / dependencies (pull_request) Successful in 55s
2026-07-02 16:59:30 +02:00
cesnimda 7ec4f1ddb8 perf(ui): route-level code splitting (#29)
CI / backend (push) Successful in 54s
CI / frontend (push) Successful in 13s
CI / format (push) Successful in 52s
CI / db-tests (push) Successful in 56s
CI / backend (pull_request) Successful in 52s
CI / frontend (pull_request) Successful in 13s
CI / format (pull_request) Successful in 49s
CI / db-tests (pull_request) Successful in 52s
Deploy Staging / deploy (push) Successful in 26s
Security / secrets (push) Successful in 5s
Security / dependencies (push) Successful in 1m0s
Security / secrets (pull_request) Successful in 5s
Security / dependencies (pull_request) Successful in 1m2s
2026-07-02 16:53:31 +02:00
cesnimda a3d8654198 feat!: migrate to .NET 10 LTS (#28)
CI / backend (push) Successful in 52s
CI / frontend (push) Successful in 12s
CI / format (push) Successful in 48s
CI / db-tests (push) Successful in 51s
Deploy Staging / deploy (push) Successful in 26s
CI / backend (pull_request) Successful in 52s
CI / frontend (pull_request) Successful in 12s
CI / format (pull_request) Successful in 46s
CI / db-tests (pull_request) Successful in 50s
Security / secrets (push) Successful in 3s
Security / dependencies (push) Successful in 59s
Security / secrets (pull_request) Successful in 4s
Security / dependencies (pull_request) Successful in 57s
2026-07-02 16:53:24 +02:00
cesnimda 86ecf03963 feat(ci): Renovate dependency automation (#27)
CI / backend (push) Successful in 50s
CI / frontend (push) Successful in 12s
CI / format (push) Successful in 47s
CI / db-tests (push) Successful in 49s
Deploy Staging / deploy (push) Successful in 18s
CI / backend (pull_request) Successful in 53s
CI / frontend (pull_request) Successful in 15s
CI / format (pull_request) Successful in 49s
CI / db-tests (pull_request) Successful in 50s
Security / secrets (push) Successful in 3s
Security / dependencies (push) Successful in 52s
Security / secrets (pull_request) Successful in 3s
Security / dependencies (pull_request) Successful in 53s
2026-07-02 16:28:02 +02:00
cesnimda dc9d5fc301 feat(ops): nightly DB backups with rotation (#26)
CI / backend (push) Successful in 1m0s
CI / frontend (push) Successful in 14s
CI / format (push) Successful in 52s
CI / db-tests (push) Successful in 1m10s
CI / backend (pull_request) Successful in 52s
CI / frontend (pull_request) Successful in 13s
CI / format (pull_request) Successful in 50s
CI / db-tests (pull_request) Successful in 51s
Deploy Staging / deploy (push) Successful in 17s
Security / secrets (pull_request) Successful in 4s
Security / dependencies (pull_request) Successful in 55s
Security / secrets (push) Successful in 4s
Security / dependencies (push) Successful in 56s
2026-07-02 16:27:57 +02:00
32 changed files with 654 additions and 80 deletions
+13
View File
@@ -0,0 +1,13 @@
# Root editor/formatter config. end_of_line=lf makes dotnet-format agree with
# .gitattributes (eol=lf) — without this, format-on-Windows wants CRLF while git
# stores LF, and the pre-commit/CI format gates flip-flop forever.
root = true
[*]
end_of_line = lf
insert_final_newline = true
charset = utf-8
[*.cs]
indent_style = space
indent_size = 4
+3 -3
View File
@@ -14,7 +14,7 @@ jobs:
- uses: actions/checkout@v4
- uses: actions/setup-dotnet@v4
with:
dotnet-version: '8.0.x'
dotnet-version: '10.0.x'
- name: Restore
run: dotnet restore InboxIntel.sln
- name: Build
@@ -45,7 +45,7 @@ jobs:
- uses: actions/checkout@v4
- uses: actions/setup-dotnet@v4
with:
dotnet-version: '8.0.x'
dotnet-version: '10.0.x'
- name: dotnet format (verify only)
run: dotnet format InboxIntel.sln --verify-no-changes
@@ -67,7 +67,7 @@ jobs:
- uses: actions/checkout@v4
- uses: actions/setup-dotnet@v4
with:
dotnet-version: '8.0.x'
dotnet-version: '10.0.x'
- name: Wait for Postgres
run: |
for i in $(seq 1 30); do
+35
View File
@@ -0,0 +1,35 @@
name: Renovate
# RECOMMENDATIONS #2: automated dependency-update PRs (NuGet, npm, Dockerfiles, Actions)
# that ride the existing required CI gates. Runs weekly + on demand.
#
# ONE-TIME SETUP (manual): create a Gitea personal access token with scopes
# repo (rw) + user (r) + issue (rw) + organization (r), and add it as the Actions
# secret RENOVATE_TOKEN (repo Settings -> Actions -> Secrets). Without the secret this
# workflow fails fast with a clear message. See https://docs.renovatebot.com/modules/platform/gitea/
on:
schedule:
- cron: '30 4 * * 1' # Mondays 04:30 UTC
workflow_dispatch: {}
jobs:
renovate:
runs-on: ubuntu-latest
steps:
- name: Require RENOVATE_TOKEN
run: |
if [ -z "${{ secrets.RENOVATE_TOKEN }}" ]; then
echo "RENOVATE_TOKEN secret is not set — see the comment at the top of this workflow." >&2
exit 1
fi
- name: Run Renovate
uses: https://github.com/renovatebot/github-action@v40.3.6
with:
token: ${{ secrets.RENOVATE_TOKEN }}
env:
RENOVATE_PLATFORM: gitea
RENOVATE_ENDPOINT: https://git.cesnimda.uk/api/v1
RENOVATE_REPOSITORIES: cesnimda/Inboxintel
RENOVATE_ONBOARDING: "false"
RENOVATE_REQUIRE_CONFIG: optional
LOG_LEVEL: info
+1 -1
View File
@@ -31,7 +31,7 @@ jobs:
- uses: actions/checkout@v4
- uses: actions/setup-dotnet@v4
with:
dotnet-version: '8.0.x'
dotnet-version: '10.0.x'
- name: Restore
run: dotnet restore InboxIntel.sln
- name: .NET vulnerable packages (fail on any)
+1 -1
View File
@@ -1,6 +1,6 @@
<Project>
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<TargetFramework>net10.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<LangVersion>latest</LangVersion>
+32
View File
@@ -65,6 +65,10 @@ services:
GoogleOAuth__ClientId: ${GOOGLE_CLIENT_ID:-}
GoogleOAuth__ClientSecret: ${GOOGLE_CLIENT_SECRET:-}
Ai__Mode: ${AI_MODE:-Disabled}
# Points at the compose 'ollama' service when the ai profile is up; harmless otherwise.
Ai__OllamaBaseUrl: ${OLLAMA_BASE_URL:-http://ollama:11434}
# OTLP export activates only when set (e.g. http://lgtm:4317 with the observability profile).
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_ENDPOINT:-}
# Dev mode shows the dev banner and caps the initial sync. Set DEV_MODE=true
# and MAX_MESSAGES=1000 in deploy/.env to exercise it in this Docker setup.
App__DevMode: ${DEV_MODE:-false}
@@ -90,6 +94,33 @@ services:
ports:
- "8081:80"
# Local AI (semantic search + assistants). Enable with:
# docker compose --profile ai up -d && set AI_MODE=LocalOllama in deploy/.env
# First run: docker compose exec ollama ollama pull nomic-embed-text
# GPU (RTX 3080): uncomment the deploy block to pass the GPU through.
ollama:
image: ollama/ollama
profiles: ["ai"]
volumes:
- ollama:/root/.ollama
# deploy:
# resources:
# reservations:
# devices:
# - driver: nvidia
# count: all
# capabilities: [gpu]
# Observability (RECOMMENDATIONS #5): all-in-one Grafana+Tempo+Prometheus+Loki.
# Enable with: docker compose --profile observability up -d
# then set OTEL_ENDPOINT=http://lgtm:4317 in deploy/.env and restart the api.
# Grafana UI: http://localhost:3000 (admin/admin on first run).
lgtm:
image: grafana/otel-lgtm
profiles: ["observability"]
ports:
- "127.0.0.1:3000:3000"
# Optional reverse proxy. Enable with: docker compose --profile proxy up
nginx:
image: nginx:alpine
@@ -105,3 +136,4 @@ services:
volumes:
pgdata:
keys:
ollama:
+34 -23
View File
@@ -1,42 +1,53 @@
import React from 'react';
import React, { Suspense, lazy } from 'react';
import ReactDOM from 'react-dom/client';
import { BrowserRouter, Routes, Route, Navigate } from 'react-router-dom';
import Landing from './pages/Landing.jsx';
import Dashboard from './pages/Dashboard.jsx';
import Senders from './pages/Senders.jsx';
import Cleanup from './pages/Cleanup.jsx';
import Unsubscribe from './pages/Unsubscribe.jsx';
import FolderView from './pages/FolderView.jsx';
import SearchResults from './pages/SearchResults.jsx';
import Layout from './components/Layout.jsx';
import DesignSystem from './pages/DesignSystem.jsx';
import { ToastProvider, TooltipProvider } from './components/ui';
import '@fontsource-variable/inter';
import './index.css';
import './styles.css';
// Route-level code splitting: each page loads its own chunk on first visit, so the
// initial bundle no longer carries Chart.js / grid-layout / every page at once.
// Landing + Layout stay eager (they're the first paint).
const Dashboard = lazy(() => import('./pages/Dashboard.jsx'));
const Senders = lazy(() => import('./pages/Senders.jsx'));
const Cleanup = lazy(() => import('./pages/Cleanup.jsx'));
const Unsubscribe = lazy(() => import('./pages/Unsubscribe.jsx'));
const FolderView = lazy(() => import('./pages/FolderView.jsx'));
const SearchResults = lazy(() => import('./pages/SearchResults.jsx'));
const DesignSystem = lazy(() => import('./pages/DesignSystem.jsx'));
// Minimal, theme-correct route fallback (skeleton-style, per the design system).
const RouteFallback = () => (
<div className="p-6 text-sm text-muted-foreground" aria-busy="true">Loading</div>
);
ReactDOM.createRoot(document.getElementById('root')).render(
<React.StrictMode>
<ToastProvider>
<TooltipProvider delayDuration={200}>
<BrowserRouter>
<Routes>
{/* Public landing page */}
<Route path="/" element={<Landing />} />
<Suspense fallback={<RouteFallback />}>
<Routes>
{/* Public landing page */}
<Route path="/" element={<Landing />} />
{/* Authenticated app */}
<Route path="/app" element={<Layout />}>
<Route index element={<Dashboard />} />
<Route path="senders" element={<Senders />} />
<Route path="cleanup" element={<Cleanup />} />
<Route path="unsubscribe" element={<Unsubscribe />} />
<Route path="folder/:slug" element={<FolderView />} />
<Route path="search" element={<SearchResults />} />
<Route path="design" element={<DesignSystem />} />
</Route>
{/* Authenticated app */}
<Route path="/app" element={<Layout />}>
<Route index element={<Dashboard />} />
<Route path="senders" element={<Senders />} />
<Route path="cleanup" element={<Cleanup />} />
<Route path="unsubscribe" element={<Unsubscribe />} />
<Route path="folder/:slug" element={<FolderView />} />
<Route path="search" element={<SearchResults />} />
<Route path="design" element={<DesignSystem />} />
</Route>
<Route path="*" element={<Navigate to="/" replace />} />
</Routes>
<Route path="*" element={<Navigate to="/" replace />} />
</Routes>
</Suspense>
</BrowserRouter>
</TooltipProvider>
</ToastProvider>
+27
View File
@@ -0,0 +1,27 @@
{
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
"extends": ["config:recommended"],
"timezone": "Europe/Berlin",
"schedule": ["before 6am on monday"],
"labels": ["dependencies"],
"prConcurrentLimit": 5,
"commitMessagePrefix": "chore(deps):",
"packageRules": [
{
"description": "Group safe minor+patch updates into one weekly PR per ecosystem",
"matchUpdateTypes": ["minor", "patch"],
"groupName": "{{manager}} minor & patch"
},
{
"description": "Major updates stay individual PRs for careful review",
"matchUpdateTypes": ["major"],
"dependencyDashboardApproval": true
}
],
"vulnerabilityAlerts": {
"enabled": true,
"labels": ["security"],
"schedule": ["at any time"]
},
"ignorePaths": ["**/node_modules/**", "**/bin/**", "**/obj/**"]
}
+2 -2
View File
@@ -1,5 +1,5 @@
# Multi-stage build for the ASP.NET Core API.
FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build
WORKDIR /src
# Copy solution + project files first for layer-cached restore.
@@ -13,7 +13,7 @@ RUN dotnet restore src/InboxIntel.Api/InboxIntel.Api.csproj
COPY src/ src/
RUN dotnet publish src/InboxIntel.Api/InboxIntel.Api.csproj -c Release -o /app/publish /p:UseAppHost=false
FROM mcr.microsoft.com/dotnet/aspnet:8.0 AS runtime
FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS runtime
WORKDIR /app
COPY --from=build /app/publish .
+8 -3
View File
@@ -5,16 +5,21 @@
<UserSecretsId>210c6d96-c7e4-4ee9-8982-8b91424979b8</UserSecretsId>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.AspNetCore.Authentication.Google" Version="8.0.7" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="8.0.7" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.Google" Version="10.0.9" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="10.0.9" />
<!-- Required on the startup project for `dotnet ef migrations` to work. -->
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.4">
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="10.0.9">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="Asp.Versioning.Mvc" Version="8.1.0" />
<PackageReference Include="Asp.Versioning.Mvc.ApiExplorer" Version="8.1.0" />
<PackageReference Include="FluentValidation.AspNetCore" Version="11.3.0" />
<PackageReference Include="Npgsql.OpenTelemetry" Version="10.0.3" />
<PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.16.0" />
<PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.16.0" />
<PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.16.0" />
<PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.16.0" />
<PackageReference Include="Serilog.AspNetCore" Version="8.0.1" />
<PackageReference Include="Serilog.Sinks.Console" Version="5.0.1" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2" />
+31 -6
View File
@@ -16,6 +16,10 @@ using Microsoft.AspNetCore.HttpOverrides;
using Microsoft.AspNetCore.RateLimiting;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options;
using Npgsql;
using OpenTelemetry.Metrics;
using OpenTelemetry.Resources;
using OpenTelemetry.Trace;
using Serilog;
var builder = WebApplication.CreateBuilder(args);
@@ -38,8 +42,8 @@ var dp = builder.Services.AddDataProtection()
var dpCertPath = builder.Configuration["DataProtection:CertificatePath"];
if (!string.IsNullOrWhiteSpace(dpCertPath))
{
dp.ProtectKeysWithCertificate(new System.Security.Cryptography.X509Certificates.X509Certificate2(
dpCertPath, builder.Configuration["DataProtection:CertificatePassword"]));
dp.ProtectKeysWithCertificate(System.Security.Cryptography.X509Certificates.X509CertificateLoader
.LoadPkcs12FromFile(dpCertPath, builder.Configuration["DataProtection:CertificatePassword"]));
}
builder.Services.AddApplication();
@@ -122,6 +126,28 @@ builder.Services.AddAuthentication(options =>
builder.Services.AddAuthorization();
// RECOMMENDATIONS #5: OpenTelemetry traces + metrics (ASP.NET, outbound HTTP, Npgsql).
// The OTLP exporter only activates when Otel:Endpoint (or the standard
// OTEL_EXPORTER_OTLP_ENDPOINT env var) is configured — zero overhead otherwise.
// Logs stay on Serilog. Pair with the compose "observability" profile (grafana/otel-lgtm).
var otlpEndpoint = builder.Configuration["Otel:Endpoint"]
?? Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_ENDPOINT");
if (!string.IsNullOrWhiteSpace(otlpEndpoint))
{
builder.Services.AddOpenTelemetry()
.ConfigureResource(r => r.AddService("inboxintel-api"))
.WithTracing(t => t
.AddAspNetCoreInstrumentation()
.AddHttpClientInstrumentation()
.AddNpgsql()
.AddOtlpExporter(o => o.Endpoint = new Uri(otlpEndpoint)))
.WithMetrics(m => m
.AddAspNetCoreInstrumentation()
.AddHttpClientInstrumentation()
.AddNpgsqlInstrumentation()
.AddOtlpExporter(o => o.Endpoint = new Uri(otlpEndpoint)));
}
builder.Services.AddApiVersioning(o =>
{
o.DefaultApiVersion = new ApiVersion(1, 0);
@@ -201,15 +227,14 @@ var forwardedOptions = new ForwardedHeadersOptions
ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto | ForwardedHeaders.XForwardedHost,
ForwardLimit = app.Configuration.GetValue<int?>("ForwardedHeaders:ForwardLimit") ?? 1
};
forwardedOptions.KnownNetworks.Clear();
forwardedOptions.KnownIPNetworks.Clear();
forwardedOptions.KnownProxies.Clear();
var trustedNetworks = app.Configuration.GetSection("ForwardedHeaders:KnownNetworks").Get<string[]>()
?? new[] { "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "127.0.0.0/8", "::1/128" };
foreach (var cidr in trustedNetworks)
{
var parts = cidr.Split('/');
if (parts.Length == 2 && System.Net.IPAddress.TryParse(parts[0], out var prefix) && int.TryParse(parts[1], out var len))
forwardedOptions.KnownNetworks.Add(new Microsoft.AspNetCore.HttpOverrides.IPNetwork(prefix, len));
if (System.Net.IPNetwork.TryParse(cidr, out var network))
forwardedOptions.KnownIPNetworks.Add(network);
}
app.UseForwardedHeaders(forwardedOptions);
@@ -23,4 +23,9 @@ public record SearchRequestDto(
bool? IsTrashed = null,
string? GmailLabel = null, // e.g. "SENT", "DRAFT", "SPAM"
string? Category = null, // EmailCategory name, e.g. "Finance"
long? MinSizeBytes = null);
long? MinSizeBytes = null,
// Keyset cursor for the date-ordered browse path (RECOMMENDATIONS #8): pass the last
// row's SentAtUtc+Id to fetch the next window without OFFSET (O(pageSize), not O(page)).
// When set, TotalCount is not recomputed (-1). Additive; offset paging still works.
DateTimeOffset? AfterSentAtUtc = null,
Guid? AfterId = null);
@@ -6,13 +6,13 @@
<ItemGroup>
<PackageReference Include="FluentValidation" Version="11.9.2" />
<PackageReference Include="FluentValidation.DependencyInjectionExtensions" Version="11.9.2" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.2" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.9" />
<!-- DbSet<> is exposed on IAppDbContext so the Application layer can query.
Pinned to 8.0.4 to match the Npgsql provider's Relational dependency. -->
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.4" />
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="10.0.9" />
<!-- Transitive security pins: patch known .NET 8.0.0 advisories pulled in by EF Core. -->
<PackageReference Include="System.Text.Json" Version="8.0.6" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" />
<PackageReference Include="System.Text.Json" Version="10.0.9" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="10.0.9" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\InboxIntel.Domain\InboxIntel.Domain.csproj" />
@@ -6,7 +6,7 @@
<ItemGroup>
<!-- NpgsqlTypes.NpgsqlTsVector (FTS) and Pgvector.Vector (semantic search) are used as
column types on the Email entity — same pragmatic precedent for both. -->
<PackageReference Include="Npgsql" Version="8.0.3" />
<PackageReference Include="Pgvector" Version="0.2.0" />
<PackageReference Include="Npgsql" Version="10.0.2" />
<PackageReference Include="Pgvector" Version="0.3.0" />
</ItemGroup>
</Project>
@@ -0,0 +1,104 @@
using InboxIntel.Application.Abstractions;
using InboxIntel.Infrastructure.Persistence;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace InboxIntel.Infrastructure.Ai;
/// <summary>
/// Fills <c>Email.Embedding</c> (pgvector) for semantic search, in small background batches
/// so interactive requests are never starved (per docs/discovery/06: embeddings are the
/// small always-on model; the batch pause keeps VRAM/CPU pressure low). Exits immediately
/// when the embedding provider is unavailable (AI disabled / Ollama down) — semantic search
/// simply stays dormant and lexical search is unaffected.
/// </summary>
public class EmbeddingBackfillWorker : BackgroundService
{
private const int BatchSize = 32;
private static readonly TimeSpan BatchPause = TimeSpan.FromSeconds(2);
private static readonly TimeSpan IdleRescan = TimeSpan.FromMinutes(15);
private readonly IServiceScopeFactory _scopeFactory;
private readonly ILogger<EmbeddingBackfillWorker> _logger;
public EmbeddingBackfillWorker(IServiceScopeFactory scopeFactory, ILogger<EmbeddingBackfillWorker> logger)
{
_scopeFactory = scopeFactory;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
// Provider availability is fixed by configuration for the process lifetime.
using (var probe = _scopeFactory.CreateScope())
{
if (!probe.ServiceProvider.GetRequiredService<IEmbeddingProvider>().IsAvailable)
{
_logger.LogDebug("EmbeddingBackfillWorker idle: no embedding provider (AI disabled).");
return;
}
}
_logger.LogInformation("EmbeddingBackfillWorker started (batch {Batch}, pause {Pause}s)",
BatchSize, BatchPause.TotalSeconds);
while (!stoppingToken.IsCancellationRequested)
{
int processed;
try
{
processed = await ProcessBatchAsync(stoppingToken);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; }
catch (Exception ex)
{
// Ollama hiccups must never crash the host; back off and retry.
_logger.LogWarning(ex, "Embedding batch failed; retrying after idle pause.");
processed = 0;
}
await Task.Delay(processed > 0 ? BatchPause : IdleRescan, stoppingToken);
}
}
/// <summary>Embeds one batch. Public-ish (internal) for direct testing.</summary>
internal async Task<int> ProcessBatchAsync(CancellationToken ct)
{
using var scope = _scopeFactory.CreateScope();
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
var embeddings = scope.ServiceProvider.GetRequiredService<IEmbeddingProvider>();
var batch = await db.Emails
.Where(e => e.Embedding == null)
.OrderByDescending(e => e.SentAtUtc) // newest mail becomes searchable first
.Take(BatchSize)
.ToListAsync(ct);
if (batch.Count == 0) return 0;
// Subject + snippet is the semantic core; bodies are noisy (signatures, quoting)
// and slow to embed. Truncate defensively to keep well inside the model context.
var texts = batch
.Select(e => Truncate($"{e.Subject}\n{e.Snippet ?? e.BodyText}", 2000))
.ToList();
var vectors = await embeddings.EmbedBatchAsync(texts, ct);
if (vectors.Count != batch.Count)
{
_logger.LogWarning("Embedding batch returned {Got} vectors for {Want} emails; skipping batch.",
vectors.Count, batch.Count);
return 0;
}
for (var i = 0; i < batch.Count; i++)
{
if (vectors[i].Length == 0) continue; // provider soft-failure for one item
batch[i].Embedding = new Pgvector.Vector(vectors[i]);
}
await db.SaveChangesAsync(ct);
_logger.LogDebug("Embedded {Count} emails", batch.Count);
return batch.Count;
}
private static string Truncate(string s, int max) => s.Length <= max ? s : s[..max];
}
@@ -92,6 +92,9 @@ public static class DependencyInjection
break;
}
services.AddScoped<IAiService, AiService>();
// Semantic search: fills Email.Embedding in the background; no-ops when the
// embedding provider is unavailable (AI disabled), so lexical search is unaffected.
services.AddHostedService<EmbeddingBackfillWorker>();
// Background worker (daily incremental sync + aggregate refresh)
services.AddHostedService<GmailSyncWorker>();
@@ -4,18 +4,18 @@
<AssemblyName>InboxIntel.Infrastructure</AssemblyName>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.4" />
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.4" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.4">
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="10.0.9" />
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="10.0.2" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="10.0.9">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="Google.Apis.Gmail.v1" Version="1.68.0.3427" />
<PackageReference Include="Google.Apis.Auth" Version="1.68.0" />
<PackageReference Include="Microsoft.AspNetCore.DataProtection" Version="8.0.7" />
<PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="8.0.0" />
<PackageReference Include="Pgvector.EntityFrameworkCore" Version="0.2.0" />
<PackageReference Include="Microsoft.AspNetCore.DataProtection" Version="10.0.9" />
<PackageReference Include="Microsoft.Extensions.Http" Version="10.0.9" />
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="10.0.9" />
<PackageReference Include="Pgvector.EntityFrameworkCore" Version="0.3.0" />
<PackageReference Include="Polly" Version="8.4.1" />
<PackageReference Include="QuestPDF" Version="2024.7.0" />
<PackageReference Include="CsvHelper" Version="33.0.1" />
@@ -24,12 +24,15 @@
<!-- Transitive security pins: patch known .NET 8.0.0 advisories pulled in by
EF Core / ASP.NET / DataProtection. Remove once the parent packages ship
these versions transitively. -->
<PackageReference Include="System.Text.Json" Version="8.0.6" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" />
<PackageReference Include="System.Security.Cryptography.Xml" Version="8.0.3" />
<PackageReference Include="System.Text.Json" Version="10.0.9" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="10.0.9" />
<PackageReference Include="System.Security.Cryptography.Xml" Version="10.0.9" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\InboxIntel.Application\InboxIntel.Application.csproj" />
<ProjectReference Include="..\InboxIntel.Domain\InboxIntel.Domain.csproj" />
</ItemGroup>
<ItemGroup>
<InternalsVisibleTo Include="InboxIntel.IntegrationTests" />
</ItemGroup>
</Project>
@@ -1,4 +1,4 @@
using System;
using System;
using Microsoft.EntityFrameworkCore.Migrations;
using NpgsqlTypes;
@@ -1,4 +1,4 @@
using System;
using System;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
@@ -1,4 +1,4 @@
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
@@ -1,4 +1,4 @@
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Migrations;
using NpgsqlTypes;
#nullable disable
@@ -1,4 +1,4 @@
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
@@ -1,4 +1,4 @@
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
@@ -1,4 +1,4 @@
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Migrations;
using Pgvector;
#nullable disable
@@ -47,20 +47,20 @@ public class AppDbContext : DbContext, IAppDbContext
// its manual `WHERE UserId ==` clause cannot leak across tenants. Applied
// uniformly to all user-scoped entities so EF sees no filtered/unfiltered
// navigation mismatch. Bypassed when CurrentUserId is Guid.Empty (workers).
modelBuilder.Entity<Email>().HasQueryFilter(e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
modelBuilder.Entity<Sender>().HasQueryFilter(e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
modelBuilder.Entity<MailThread>().HasQueryFilter(e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
modelBuilder.Entity<MailDomain>().HasQueryFilter(e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
modelBuilder.Entity<Attachment>().HasQueryFilter(e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
modelBuilder.Entity<Label>().HasQueryFilter(e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
modelBuilder.Entity<Email>().HasQueryFilter("Tenant", e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
modelBuilder.Entity<Sender>().HasQueryFilter("Tenant", e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
modelBuilder.Entity<MailThread>().HasQueryFilter("Tenant", e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
modelBuilder.Entity<MailDomain>().HasQueryFilter("Tenant", e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
modelBuilder.Entity<Attachment>().HasQueryFilter("Tenant", e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
modelBuilder.Entity<Label>().HasQueryFilter("Tenant", e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
// AUDIT M-6: EmailLabel is the required end of a relationship with the filtered Email
// entity; without a matching filter EF warns on boot and joins could surface rows whose
// parent is filtered out. Filter via the Email navigation so the pair is consistent.
modelBuilder.Entity<EmailLabel>().HasQueryFilter(el => CurrentUserId == Guid.Empty || el.Email!.UserId == CurrentUserId);
modelBuilder.Entity<SyncState>().HasQueryFilter(e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
modelBuilder.Entity<AnalyticsAggregate>().HasQueryFilter(e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
modelBuilder.Entity<WidgetLayout>().HasQueryFilter(e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
modelBuilder.Entity<UnsubscribeItem>().HasQueryFilter(e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
modelBuilder.Entity<EmailLabel>().HasQueryFilter("Tenant", el => CurrentUserId == Guid.Empty || el.Email!.UserId == CurrentUserId);
modelBuilder.Entity<SyncState>().HasQueryFilter("Tenant", e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
modelBuilder.Entity<AnalyticsAggregate>().HasQueryFilter("Tenant", e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
modelBuilder.Entity<WidgetLayout>().HasQueryFilter("Tenant", e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
modelBuilder.Entity<UnsubscribeItem>().HasQueryFilter("Tenant", e => CurrentUserId == Guid.Empty || e.UserId == CurrentUserId);
// PostgreSQL full-text search: generated tsvector over subject + body with a
// GIN index, maintained by the DB and read-only in code. Subject is weighted 'A'
@@ -5,6 +5,7 @@ using InboxIntel.Domain.Entities;
using InboxIntel.Domain.Enums;
using InboxIntel.Infrastructure.Persistence;
using Microsoft.EntityFrameworkCore;
using Pgvector.EntityFrameworkCore;
namespace InboxIntel.Infrastructure.Search;
@@ -18,7 +19,13 @@ namespace InboxIntel.Infrastructure.Search;
public class SearchService : ISearchService
{
private readonly AppDbContext _db;
public SearchService(AppDbContext db) => _db = db;
private readonly IEmbeddingProvider? _embeddings;
public SearchService(AppDbContext db, IEmbeddingProvider? embeddings = null)
{
_db = db;
_embeddings = embeddings;
}
public async Task<PagedResult<EmailSummaryDto>> SearchAsync(Guid userId, SearchRequestDto r, CancellationToken ct = default)
{
@@ -73,6 +80,18 @@ public class SearchService : ISearchService
var total = await matched.CountAsync(ct);
// Hybrid semantic fusion (docs/discovery/05): when embeddings are available, fuse
// lexical top-K with vector top-K via Reciprocal Rank Fusion. Runs BEFORE the fuzzy
// fallback so a query with ZERO lexical hits (pure semantic recall — "gym receipt"
// phrased differently) still surfaces results. Exact lexical hits keep winning (they
// rank in both lists). Deeper pages fall through to lexical paging; any failure
// (Ollama down, nothing embedded yet) silently degrades to the lexical/fuzzy path.
if (hasFreeTextQuery && _embeddings is { IsAvailable: true })
{
var hybrid = await TryHybridAsync(structured, matched, term, total, r, ct);
if (hybrid is not null) return hybrid;
}
// Fuzzy/typo fallback: ONLY when a free-text search found nothing exact. word_similarity
// with an explicit 0.3 threshold — pg_trgm's default 0.6 misses real typos
// ("recieved" -> "received" scores ~0.39). Rare path, so the (non-indexed) scan over the
@@ -95,9 +114,23 @@ public class SearchService : ISearchService
ranked = matched.OrderByDescending(e => e.SearchVector!.RankCoverDensity(EF.Functions.WebSearchToTsQuery("english", term)))
.ThenByDescending(e => e.SentAtUtc);
else
ranked = matched.OrderByDescending(e => e.SentAtUtc);
ranked = matched.OrderByDescending(e => e.SentAtUtc).ThenByDescending(e => e.Id);
var paged = ranked.Skip((r.Page - 1) * r.PageSize).Take(r.PageSize);
// Keyset (cursor) pagination for the browse path: O(pageSize) regardless of depth,
// vs OFFSET's O(page*pageSize). The (SentAtUtc, Id) pair with the Id tie-break above
// makes the ordering total, so windows never duplicate or skip rows.
IQueryable<Email> paged;
if (!hasFreeTextQuery && r is { AfterSentAtUtc: { } afterAt, AfterId: { } afterId })
{
paged = ranked
.Where(e => e.SentAtUtc < afterAt || (e.SentAtUtc == afterAt && e.Id.CompareTo(afterId) < 0))
.Take(r.PageSize);
total = -1; // not recomputed on cursor windows (that's the point)
}
else
{
paged = ranked.Skip((r.Page - 1) * r.PageSize).Take(r.PageSize);
}
// "Why this matched" ts_headline only for EXACT free-text hits (fuzzy/browse get no
// highlight — a fuzzy hit has no literal match to headline). Unconditional projections
@@ -137,4 +170,73 @@ public class SearchService : ISearchService
TotalCount = total
};
}
private const int HybridK = 50; // candidates taken from each layer
private const int RrfConstant = 60; // standard RRF dampening constant
/// <summary>
/// RRF fusion of lexical and vector candidates. Returns null when the requested page
/// lies beyond the fused window or anything fails — caller falls back to lexical.
/// </summary>
private async Task<PagedResult<EmailSummaryDto>?> TryHybridAsync(
IQueryable<Email> structured, IQueryable<Email> lexical, string term, int lexicalTotal,
SearchRequestDto r, CancellationToken ct)
{
try
{
var lexIds = await lexical
.OrderByDescending(e => e.SearchVector!.RankCoverDensity(EF.Functions.WebSearchToTsQuery("english", term)))
.ThenByDescending(e => e.SentAtUtc)
.Take(HybridK).Select(e => e.Id).ToListAsync(ct);
var queryVec = await _embeddings!.EmbedAsync(term, ct);
List<Guid> vecIds = new();
if (queryVec.Length > 0)
{
var qv = new Pgvector.Vector(queryVec);
vecIds = await structured
.Where(e => e.Embedding != null)
.OrderBy(e => e.Embedding!.CosineDistance(qv))
.Take(HybridK).Select(e => e.Id).ToListAsync(ct);
}
if (vecIds.Count == 0) return null; // nothing embedded yet → lexical path
var scores = new Dictionary<Guid, double>();
for (var i = 0; i < lexIds.Count; i++)
scores[lexIds[i]] = scores.GetValueOrDefault(lexIds[i]) + 1.0 / (RrfConstant + i + 1);
for (var i = 0; i < vecIds.Count; i++)
scores[vecIds[i]] = scores.GetValueOrDefault(vecIds[i]) + 1.0 / (RrfConstant + i + 1);
var fused = scores.OrderByDescending(kv => kv.Value).Select(kv => kv.Key).ToList();
var pageIds = fused.Skip((r.Page - 1) * r.PageSize).Take(r.PageSize).ToList();
if (pageIds.Count == 0 && r.Page > 1) return null; // deep page → lexical paging
var headlineOpts =
$"StartSel={(char)0xE000},StopSel={(char)0xE001},MaxWords=16,MinWords=5,ShortWord=2,HighlightAll=false";
var rows = await _db.Emails.AsNoTracking()
.Where(e => pageIds.Contains(e.Id))
.Select(e => new EmailSummaryDto(
e.Id, e.GmailMessageId, e.Subject, e.Snippet,
e.Sender!.Address, e.Sender.DisplayName, e.SentAtUtc,
e.IsUnread, e.IsStarred, e.HasAttachments, e.SizeEstimateBytes, e.Category,
e.HasListUnsubscribe, e.SupportsOneClickUnsubscribe,
EF.Functions.WebSearchToTsQuery("english", term).GetResultHeadline("english", e.BodyText ?? "", headlineOpts)))
.ToListAsync(ct);
var byId = rows.ToDictionary(x => x.Id);
var items = pageIds.Where(byId.ContainsKey).Select(id => byId[id]).ToList();
return new PagedResult<EmailSummaryDto>
{
Items = items,
Page = r.Page,
PageSize = r.PageSize,
// Semantic recall can exceed the lexical match count.
TotalCount = Math.Max(lexicalTotal, fused.Count)
};
}
catch
{
return null; // AI must never break search — degrade to lexical
}
}
}
@@ -25,7 +25,7 @@ namespace InboxIntel.IntegrationTests;
/// endpoints (model validation, per-user rate limits) without a real Google login.</summary>
public class TestAuthHandler : AuthenticationHandler<AuthenticationSchemeOptions>
{
public const string Scheme = "Test";
public new const string Scheme = "Test";
// Stable across requests so per-user rate-limit partitions accumulate correctly.
public static readonly string Uid = Guid.NewGuid().ToString();
@@ -52,6 +52,9 @@ public class AuditTestAppFactory : WebApplicationFactory<Program>
builder.ConfigureHostConfiguration(cfg => cfg.AddInMemoryCollection(new Dictionary<string, string?>
{
["Database:AutoMigrate"] = "false",
// Npgsql 10 eagerly validates the connection string when the DbContext is
// resolved (8.x was lazy); these tests never connect, but the string must parse.
["ConnectionStrings:Postgres"] = "Host=localhost;Database=test;Username=test;Password=test",
["GoogleOAuth:ClientId"] = "test-client-id",
["GoogleOAuth:ClientSecret"] = "test-client-secret",
// H-2: make the auth policy trip on the 3rd request within the window.
@@ -19,6 +19,9 @@ public class TestAppFactory : WebApplicationFactory<Program>
builder.ConfigureHostConfiguration(cfg => cfg.AddInMemoryCollection(new Dictionary<string, string?>
{
["Database:AutoMigrate"] = "false",
// Npgsql 10 eagerly validates the connection string when the DbContext is
// resolved (8.x was lazy); these tests never connect, but the string must parse.
["ConnectionStrings:Postgres"] = "Host=localhost;Database=test;Username=test;Password=test",
// Dummy OAuth creds so the Google challenge produces a real 302 redirect
// (an empty ClientId can make the handler throw instead of redirecting).
["GoogleOAuth:ClientId"] = "test-client-id",
@@ -0,0 +1,95 @@
using FluentAssertions;
using InboxIntel.Application.Abstractions;
using InboxIntel.Domain.Entities;
using InboxIntel.Infrastructure.Ai;
using InboxIntel.Infrastructure.Persistence;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Pgvector.EntityFrameworkCore;
using Xunit;
namespace InboxIntel.IntegrationTests;
/// <summary>
/// Semantic-search backfill: the worker embeds emails lacking an Embedding and persists the
/// vectors. Runs against live Postgres (pgvector) since the Embedding column is ignored under
/// the InMemory provider; the CI db-tests job provides the database. Uses a deterministic fake
/// embedding provider — real-Ollama integration is verified separately (endpoint contract:
/// /api/embeddings, 768 dims).
/// </summary>
[Trait("Category", "LiveDb")]
[Collection("LiveDb")] // serialise LiveDb classes: concurrent MigrateAsync on a fresh DB races
public class EmbeddingBackfillTests
{
private static string? Conn => Environment.GetEnvironmentVariable("LIVEDB_CONNECTION");
private sealed class FakeCurrentUser : ICurrentUser
{
public Guid UserId => Guid.Empty; // worker scope sees all rows
public bool IsAuthenticated => false;
}
private sealed class FakeEmbeddings : IEmbeddingProvider
{
public bool IsAvailable => true;
public Task<float[]> EmbedAsync(string text, CancellationToken ct = default)
=> Task.FromResult(Vec(text));
public Task<IReadOnlyList<float[]>> EmbedBatchAsync(IReadOnlyList<string> texts, CancellationToken ct = default)
=> Task.FromResult<IReadOnlyList<float[]>>(texts.Select(Vec).ToList());
private static float[] Vec(string text)
{
var v = new float[768];
v[0] = text.Length; // deterministic, content-dependent
return v;
}
}
[Fact]
public async Task Worker_embeds_pending_emails_and_persists_vectors()
{
if (Conn is null) return; // soft-skip outside the live-db CI job
var uid = Guid.NewGuid();
var opts = new DbContextOptionsBuilder<AppDbContext>().UseNpgsql(Conn!, o => o.UseVector()).Options;
var services = new ServiceCollection();
services.AddScoped<ICurrentUser, FakeCurrentUser>();
services.AddScoped(_ => new AppDbContext(opts, new FakeCurrentUser()));
services.AddScoped<IEmbeddingProvider, FakeEmbeddings>();
using var sp = services.BuildServiceProvider();
using (var seed = new AppDbContext(opts, new FakeCurrentUser()))
{
await seed.Database.MigrateAsync();
seed.Users.Add(new User { Id = uid, GoogleSubjectId = "g" + uid, Email = uid + "@t.t" });
var dom = new MailDomain { UserId = uid, Name = "t.t" };
var snd = new Sender { UserId = uid, Address = "a@t.t", Domain = dom };
var thr = new MailThread { UserId = uid, GmailThreadId = "th" + uid };
seed.AddRange(dom, snd, thr);
seed.Emails.Add(new Email { UserId = uid, GmailMessageId = "e1" + uid, Subject = "hello world", Sender = snd, Thread = thr, SentAtUtc = DateTimeOffset.UtcNow });
seed.Emails.Add(new Email { UserId = uid, GmailMessageId = "e2" + uid, Subject = "quarterly invoice", Sender = snd, Thread = thr, SentAtUtc = DateTimeOffset.UtcNow });
await seed.SaveChangesAsync();
}
try
{
var worker = new EmbeddingBackfillWorker(
sp.GetRequiredService<IServiceScopeFactory>(), NullLogger<EmbeddingBackfillWorker>.Instance);
var processed = await worker.ProcessBatchAsync(CancellationToken.None);
processed.Should().BeGreaterThanOrEqualTo(2);
using var check = new AppDbContext(opts, new FakeCurrentUser());
var mine = await check.Emails.Where(e => e.UserId == uid).ToListAsync();
mine.Should().OnlyContain(e => e.Embedding != null);
mine.First().Embedding!.ToArray().Length.Should().Be(768);
}
finally
{
using var c = new AppDbContext(opts, new FakeCurrentUser());
await c.Emails.Where(e => e.UserId == uid).ExecuteDeleteAsync();
await c.Threads.Where(t => t.UserId == uid).ExecuteDeleteAsync();
await c.Senders.Where(s => s.UserId == uid).ExecuteDeleteAsync();
await c.Domains.Where(d => d.UserId == uid).ExecuteDeleteAsync();
await c.Users.Where(u => u.Id == uid).ExecuteDeleteAsync();
}
}
}
@@ -7,8 +7,8 @@
<PackageReference Include="xunit" Version="2.9.0" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" />
<PackageReference Include="FluentAssertions" Version="6.12.0" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="8.0.7" />
<PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" Version="8.0.4" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.9" />
<PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" Version="10.0.9" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\src\InboxIntel.Api\InboxIntel.Api.csproj" />
@@ -0,0 +1,57 @@
using FluentAssertions;
using InboxIntel.Application.Abstractions;
using InboxIntel.Application.DTOs;
using InboxIntel.Domain.Entities;
using InboxIntel.Infrastructure.Persistence;
using InboxIntel.Infrastructure.Search;
using Microsoft.EntityFrameworkCore;
using Xunit;
namespace InboxIntel.IntegrationTests;
/// <summary>
/// RECOMMENDATIONS #8: keyset (cursor) pagination for the browse path — the window after a
/// (SentAtUtc, Id) cursor returns the next rows with no duplicates/skips and no OFFSET scan.
/// </summary>
public class KeysetPaginationTests
{
private sealed class FakeCurrentUser : ICurrentUser
{
public Guid UserId { get; set; }
public bool IsAuthenticated => UserId != Guid.Empty;
}
[Fact]
public async Task Cursor_window_continues_exactly_after_the_previous_page()
{
var user = Guid.NewGuid();
var opts = new DbContextOptionsBuilder<AppDbContext>()
.UseInMemoryDatabase(nameof(Cursor_window_continues_exactly_after_the_previous_page)).Options;
var baseline = DateTimeOffset.UtcNow;
using (var seed = new AppDbContext(opts, new FakeCurrentUser()))
{
var sender = new Sender { UserId = user, Address = "s@x.x" };
seed.Senders.Add(sender);
for (var i = 0; i < 5; i++)
seed.Emails.Add(new Email { UserId = user, GmailMessageId = $"m{i}", Sender = sender, SentAtUtc = baseline.AddMinutes(-i) });
await seed.SaveChangesAsync();
}
using var ctx = new AppDbContext(opts, new FakeCurrentUser { UserId = user });
var svc = new SearchService(ctx);
// First window via offset (page 1, size 2): m0, m1 (newest first).
var page1 = await svc.SearchAsync(user, new SearchRequestDto(null, null, null, null, null, null, null, false, 1, 2));
page1.Items.Select(i => i.GmailMessageId).Should().Equal("m0", "m1");
// Next window via cursor from the last row of page 1.
var last = page1.Items[^1];
var page2 = await svc.SearchAsync(user, new SearchRequestDto(
null, null, null, null, null, null, null, false, 1, 2,
AfterSentAtUtc: last.SentAtUtc, AfterId: last.Id));
page2.Items.Select(i => i.GmailMessageId).Should().Equal("m2", "m3"); // no dupes, no skips
page2.TotalCount.Should().Be(-1, "cursor windows skip the COUNT — that's the perf win");
}
}
@@ -21,6 +21,7 @@ namespace InboxIntel.IntegrationTests;
/// InMemory test run is unaffected.
/// </summary>
[Trait("Category", "LiveDb")]
[Collection("LiveDb")] // serialise LiveDb classes: concurrent MigrateAsync on a fresh DB races
public class LiveDbSearchTests
{
private static string? Conn => Environment.GetEnvironmentVariable("LIVEDB_CONNECTION");
@@ -122,4 +123,54 @@ public class LiveDbSearchTests
}
finally { await CleanupAsync(opts, uid); }
}
private sealed class DirectionalFakeEmbeddings : IEmbeddingProvider
{
public bool IsAvailable => true;
public Task<float[]> EmbedAsync(string text, CancellationToken ct = default)
{
// Deterministic "semantics": anything fruit-flavoured points one way, else the other.
var v = new float[768];
if (text.Contains("banana") || text.Contains("tropical")) v[0] = 1; else v[1] = 1;
return Task.FromResult(v);
}
public async Task<IReadOnlyList<float[]>> EmbedBatchAsync(IReadOnlyList<string> texts, CancellationToken ct = default)
{
var list = new List<float[]>();
foreach (var t in texts) list.Add(await EmbedAsync(t, ct));
return list;
}
}
[Fact]
public async Task Hybrid_search_surfaces_semantic_match_with_zero_keyword_overlap()
{
if (Conn is null) return;
var opts = Options();
var uid = await SeedAsync(opts);
try
{
var embeddings = new DirectionalFakeEmbeddings();
using (var prep = new AppDbContext(opts, new FakeCurrentUser()))
{
// "Weekly notes" gets a fruit-direction embedding (semantically related to the
// query); "Invoice March" points elsewhere. Neither subject contains "banana".
var near = await prep.Emails.FirstAsync(e => e.UserId == uid && e.Subject == "Weekly notes");
near.Embedding = new Vector(await embeddings.EmbedAsync("tropical"));
var far = await prep.Emails.FirstAsync(e => e.UserId == uid && e.Subject == "Invoice March");
far.Embedding = new Vector(await embeddings.EmbedAsync("finance"));
await prep.SaveChangesAsync();
}
using var ctx = new AppDbContext(opts, new FakeCurrentUser { UserId = uid });
var res = await new SearchService(ctx, embeddings)
.SearchAsync(uid, GmailQueryParser.Parse("banana", 1, 10));
// Zero lexical hits for "banana" — hybrid must still surface the semantically
// nearest email, ranked first.
res.Items.Should().NotBeEmpty("semantic recall should fire with zero keyword overlap");
res.Items[0].Subject.Should().Be("Weekly notes");
}
finally { await CleanupAsync(opts, uid); }
}
}