Compare commits

..

6 Commits

Author SHA1 Message Date
cesnimda 50bf6d977a feat(search): hybrid semantic search via RRF fusion (RECOMMENDATIONS #4b)
CI / backend (pull_request) Successful in 1m0s
CI / frontend (pull_request) Successful in 14s
CI / format (pull_request) Successful in 50s
CI / db-tests (pull_request) Successful in 53s
Security / secrets (pull_request) Successful in 4s
Security / dependencies (pull_request) Successful in 56s
The flagship slice (docs/discovery/05): when a free-text query runs and embeddings
are available, fuse lexical top-50 (ts_rank_cd) with vector top-50 (pgvector cosine
over the structured-filtered set) using Reciprocal Rank Fusion. Placed BEFORE the
fuzzy fallback so queries with ZERO lexical hits still get semantic recall — found
and fixed during review (the fuzzy path would otherwise swallow pure-semantic
queries). Exact lexical hits keep winning (rank in both lists); deeper pages fall
through to lexical paging; ANY failure (Ollama down, nothing embedded) silently
degrades to the lexical/fuzzy path — AI can never break search.

LiveDb test proves the recall win: a 'banana' query surfaces an email with no
keyword overlap, ranked first, against real pgvector. 5 LiveDb + full suite green;
format clean.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-02 17:17:41 +02:00
cesnimda 5191dd010f feat(ai): semantic-search backfill + Ollama profile (#30)
CI / backend (push) Successful in 55s
CI / frontend (push) Successful in 12s
CI / format (push) Successful in 48s
CI / db-tests (push) Successful in 55s
Deploy Staging / deploy (push) Successful in 42s
CI / backend (pull_request) Successful in 54s
CI / frontend (pull_request) Successful in 13s
CI / format (pull_request) Successful in 49s
CI / db-tests (pull_request) Successful in 56s
Security / secrets (push) Successful in 4s
Security / dependencies (push) Successful in 58s
Security / secrets (pull_request) Successful in 4s
Security / dependencies (pull_request) Successful in 55s
2026-07-02 16:59:30 +02:00
cesnimda 7ec4f1ddb8 perf(ui): route-level code splitting (#29)
CI / backend (push) Successful in 54s
CI / frontend (push) Successful in 13s
CI / format (push) Successful in 52s
CI / db-tests (push) Successful in 56s
CI / backend (pull_request) Successful in 52s
CI / frontend (pull_request) Successful in 13s
CI / format (pull_request) Successful in 49s
CI / db-tests (pull_request) Successful in 52s
Deploy Staging / deploy (push) Successful in 26s
Security / secrets (push) Successful in 5s
Security / dependencies (push) Successful in 1m0s
Security / secrets (pull_request) Successful in 5s
Security / dependencies (pull_request) Successful in 1m2s
2026-07-02 16:53:31 +02:00
cesnimda a3d8654198 feat!: migrate to .NET 10 LTS (#28)
CI / backend (push) Successful in 52s
CI / frontend (push) Successful in 12s
CI / format (push) Successful in 48s
CI / db-tests (push) Successful in 51s
Deploy Staging / deploy (push) Successful in 26s
CI / backend (pull_request) Successful in 52s
CI / frontend (pull_request) Successful in 12s
CI / format (pull_request) Successful in 46s
CI / db-tests (pull_request) Successful in 50s
Security / secrets (push) Successful in 3s
Security / dependencies (push) Successful in 59s
Security / secrets (pull_request) Successful in 4s
Security / dependencies (pull_request) Successful in 57s
2026-07-02 16:53:24 +02:00
cesnimda 86ecf03963 feat(ci): Renovate dependency automation (#27)
CI / backend (push) Successful in 50s
CI / frontend (push) Successful in 12s
CI / format (push) Successful in 47s
CI / db-tests (push) Successful in 49s
Deploy Staging / deploy (push) Successful in 18s
CI / backend (pull_request) Successful in 53s
CI / frontend (pull_request) Successful in 15s
CI / format (pull_request) Successful in 49s
CI / db-tests (pull_request) Successful in 50s
Security / secrets (push) Successful in 3s
Security / dependencies (push) Successful in 52s
Security / secrets (pull_request) Successful in 3s
Security / dependencies (pull_request) Successful in 53s
2026-07-02 16:28:02 +02:00
cesnimda dc9d5fc301 feat(ops): nightly DB backups with rotation (#26)
CI / backend (push) Successful in 1m0s
CI / frontend (push) Successful in 14s
CI / format (push) Successful in 52s
CI / db-tests (push) Successful in 1m10s
CI / backend (pull_request) Successful in 52s
CI / frontend (pull_request) Successful in 13s
CI / format (pull_request) Successful in 50s
CI / db-tests (pull_request) Successful in 51s
Deploy Staging / deploy (push) Successful in 17s
Security / secrets (pull_request) Successful in 4s
Security / dependencies (pull_request) Successful in 55s
Security / secrets (push) Successful in 4s
Security / dependencies (push) Successful in 56s
2026-07-02 16:27:57 +02:00
21 changed files with 315 additions and 60 deletions
+3
View File
@@ -15,3 +15,6 @@ FRONTEND_ORIGIN=http://localhost:8081
# Set DEV_MODE=true and MAX_MESSAGES=1000 to test against a large mailbox. # Set DEV_MODE=true and MAX_MESSAGES=1000 to test against a large mailbox.
DEV_MODE=false DEV_MODE=false
MAX_MESSAGES=0 MAX_MESSAGES=0
# Nightly DB backup rotation (days of dumps to keep in ./backups)
BACKUP_KEEP_DAYS=7
+3 -3
View File
@@ -14,7 +14,7 @@ jobs:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
- uses: actions/setup-dotnet@v4 - uses: actions/setup-dotnet@v4
with: with:
dotnet-version: '8.0.x' dotnet-version: '10.0.x'
- name: Restore - name: Restore
run: dotnet restore InboxIntel.sln run: dotnet restore InboxIntel.sln
- name: Build - name: Build
@@ -45,7 +45,7 @@ jobs:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
- uses: actions/setup-dotnet@v4 - uses: actions/setup-dotnet@v4
with: with:
dotnet-version: '8.0.x' dotnet-version: '10.0.x'
- name: dotnet format (verify only) - name: dotnet format (verify only)
run: dotnet format InboxIntel.sln --verify-no-changes run: dotnet format InboxIntel.sln --verify-no-changes
@@ -67,7 +67,7 @@ jobs:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
- uses: actions/setup-dotnet@v4 - uses: actions/setup-dotnet@v4
with: with:
dotnet-version: '8.0.x' dotnet-version: '10.0.x'
- name: Wait for Postgres - name: Wait for Postgres
run: | run: |
for i in $(seq 1 30); do for i in $(seq 1 30); do
+35
View File
@@ -0,0 +1,35 @@
name: Renovate
# RECOMMENDATIONS #2: automated dependency-update PRs (NuGet, npm, Dockerfiles, Actions)
# that ride the existing required CI gates. Runs weekly + on demand.
#
# ONE-TIME SETUP (manual): create a Gitea personal access token with scopes
# repo (rw) + user (r) + issue (rw) + organization (r), and add it as the Actions
# secret RENOVATE_TOKEN (repo Settings -> Actions -> Secrets). Without the secret this
# workflow fails fast with a clear message. See https://docs.renovatebot.com/modules/platform/gitea/
on:
schedule:
- cron: '30 4 * * 1' # Mondays 04:30 UTC
workflow_dispatch: {}
jobs:
renovate:
runs-on: ubuntu-latest
steps:
- name: Require RENOVATE_TOKEN
run: |
if [ -z "${{ secrets.RENOVATE_TOKEN }}" ]; then
echo "RENOVATE_TOKEN secret is not set — see the comment at the top of this workflow." >&2
exit 1
fi
- name: Run Renovate
uses: https://github.com/renovatebot/github-action@v40.3.6
with:
token: ${{ secrets.RENOVATE_TOKEN }}
env:
RENOVATE_PLATFORM: gitea
RENOVATE_ENDPOINT: https://git.cesnimda.uk/api/v1
RENOVATE_REPOSITORIES: cesnimda/Inboxintel
RENOVATE_ONBOARDING: "false"
RENOVATE_REQUIRE_CONFIG: optional
LOG_LEVEL: info
+1 -1
View File
@@ -31,7 +31,7 @@ jobs:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
- uses: actions/setup-dotnet@v4 - uses: actions/setup-dotnet@v4
with: with:
dotnet-version: '8.0.x' dotnet-version: '10.0.x'
- name: Restore - name: Restore
run: dotnet restore InboxIntel.sln run: dotnet restore InboxIntel.sln
- name: .NET vulnerable packages (fail on any) - name: .NET vulnerable packages (fail on any)
+3
View File
@@ -21,6 +21,9 @@ frontend/.vite/
appsettings.*.local.json appsettings.*.local.json
secrets.json secrets.json
## DB backups (never commit dumps)
backups/
## Logs ## Logs
logs/ logs/
*.log *.log
+1 -1
View File
@@ -1,6 +1,6 @@
<Project> <Project>
<PropertyGroup> <PropertyGroup>
<TargetFramework>net8.0</TargetFramework> <TargetFramework>net10.0</TargetFramework>
<Nullable>enable</Nullable> <Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings> <ImplicitUsings>enable</ImplicitUsings>
<LangVersion>latest</LangVersion> <LangVersion>latest</LangVersion>
+3 -1
View File
@@ -29,7 +29,9 @@ reverse proxy.
mitigate (a third party reading the DB files) reduces to "someone with access to your mitigate (a third party reading the DB files) reduces to "someone with access to your
machine" — mitigate it at the layer that actually works: machine" — mitigate it at the layer that actually works:
- **Use full-disk or volume encryption** on the host (BitLocker/LUKS) — strongly recommended. - **Use full-disk or volume encryption** on the host (BitLocker/LUKS) — strongly recommended.
- **Encrypt backups** of the `pgdata` volume the same way. - **Encrypt backups**: nightly `pg_dump` rotation runs via the compose `backup` service
into `./backups/` (git-ignored) — keep that directory on an encrypted disk and copy it
off-machine. Restore: `docker compose exec -T postgres psql -U inboxintel -d inboxintel < backups/<file>.sql`.
- Before any **multi-user** deployment, revisit per the multi-provider security design - Before any **multi-user** deployment, revisit per the multi-provider security design
(host admins must not be able to read members' mail — plaintext bodies break that promise). (host admins must not be able to read members' mail — plaintext bodies break that promise).
2. **DB connection is not TLS** — Postgres is only reachable on the compose-internal network / 2. **DB connection is not TLS** — Postgres is only reachable on the compose-internal network /
+31
View File
@@ -22,6 +22,37 @@ services:
timeout: 5s timeout: 5s
retries: 10 retries: 10
# Nightly logical backups (RECOMMENDATIONS #3 — previously there were NONE). Dumps
# rotate after BACKUP_KEEP_DAYS. The ./backups host directory should live on an
# encrypted disk and be included in your off-machine backup regime (see SECURITY.md).
# Restore: docker compose exec -T postgres psql -U inboxintel -d inboxintel < backups/<file>.sql
backup:
image: pgvector/pgvector:pg16
entrypoint: /bin/sh
command:
- -c
- |
while true; do
ts=$$(date -u +%Y%m%d-%H%M%S)
if pg_dump -h postgres -U inboxintel -d inboxintel > /backups/inboxintel-$$ts.sql.tmp; then
mv /backups/inboxintel-$$ts.sql.tmp /backups/inboxintel-$$ts.sql
echo "backup OK: inboxintel-$$ts.sql"
else
rm -f /backups/inboxintel-$$ts.sql.tmp
echo "backup FAILED at $$ts" >&2
fi
find /backups -name 'inboxintel-*.sql' -mtime +$${BACKUP_KEEP_DAYS:-7} -delete
sleep 86400
done
environment:
PGPASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in deploy/.env}
BACKUP_KEEP_DAYS: ${BACKUP_KEEP_DAYS:-7}
volumes:
- ./backups:/backups
depends_on:
postgres:
condition: service_healthy
api: api:
build: build:
context: . context: .
+34 -23
View File
@@ -1,42 +1,53 @@
import React from 'react'; import React, { Suspense, lazy } from 'react';
import ReactDOM from 'react-dom/client'; import ReactDOM from 'react-dom/client';
import { BrowserRouter, Routes, Route, Navigate } from 'react-router-dom'; import { BrowserRouter, Routes, Route, Navigate } from 'react-router-dom';
import Landing from './pages/Landing.jsx'; import Landing from './pages/Landing.jsx';
import Dashboard from './pages/Dashboard.jsx';
import Senders from './pages/Senders.jsx';
import Cleanup from './pages/Cleanup.jsx';
import Unsubscribe from './pages/Unsubscribe.jsx';
import FolderView from './pages/FolderView.jsx';
import SearchResults from './pages/SearchResults.jsx';
import Layout from './components/Layout.jsx'; import Layout from './components/Layout.jsx';
import DesignSystem from './pages/DesignSystem.jsx';
import { ToastProvider, TooltipProvider } from './components/ui'; import { ToastProvider, TooltipProvider } from './components/ui';
import '@fontsource-variable/inter'; import '@fontsource-variable/inter';
import './index.css'; import './index.css';
import './styles.css'; import './styles.css';
// Route-level code splitting: each page loads its own chunk on first visit, so the
// initial bundle no longer carries Chart.js / grid-layout / every page at once.
// Landing + Layout stay eager (they're the first paint).
const Dashboard = lazy(() => import('./pages/Dashboard.jsx'));
const Senders = lazy(() => import('./pages/Senders.jsx'));
const Cleanup = lazy(() => import('./pages/Cleanup.jsx'));
const Unsubscribe = lazy(() => import('./pages/Unsubscribe.jsx'));
const FolderView = lazy(() => import('./pages/FolderView.jsx'));
const SearchResults = lazy(() => import('./pages/SearchResults.jsx'));
const DesignSystem = lazy(() => import('./pages/DesignSystem.jsx'));
// Minimal, theme-correct route fallback (skeleton-style, per the design system).
const RouteFallback = () => (
<div className="p-6 text-sm text-muted-foreground" aria-busy="true">Loading</div>
);
ReactDOM.createRoot(document.getElementById('root')).render( ReactDOM.createRoot(document.getElementById('root')).render(
<React.StrictMode> <React.StrictMode>
<ToastProvider> <ToastProvider>
<TooltipProvider delayDuration={200}> <TooltipProvider delayDuration={200}>
<BrowserRouter> <BrowserRouter>
<Routes> <Suspense fallback={<RouteFallback />}>
{/* Public landing page */} <Routes>
<Route path="/" element={<Landing />} /> {/* Public landing page */}
<Route path="/" element={<Landing />} />
{/* Authenticated app */} {/* Authenticated app */}
<Route path="/app" element={<Layout />}> <Route path="/app" element={<Layout />}>
<Route index element={<Dashboard />} /> <Route index element={<Dashboard />} />
<Route path="senders" element={<Senders />} /> <Route path="senders" element={<Senders />} />
<Route path="cleanup" element={<Cleanup />} /> <Route path="cleanup" element={<Cleanup />} />
<Route path="unsubscribe" element={<Unsubscribe />} /> <Route path="unsubscribe" element={<Unsubscribe />} />
<Route path="folder/:slug" element={<FolderView />} /> <Route path="folder/:slug" element={<FolderView />} />
<Route path="search" element={<SearchResults />} /> <Route path="search" element={<SearchResults />} />
<Route path="design" element={<DesignSystem />} /> <Route path="design" element={<DesignSystem />} />
</Route> </Route>
<Route path="*" element={<Navigate to="/" replace />} /> <Route path="*" element={<Navigate to="/" replace />} />
</Routes> </Routes>
</Suspense>
</BrowserRouter> </BrowserRouter>
</TooltipProvider> </TooltipProvider>
</ToastProvider> </ToastProvider>
+27
View File
@@ -0,0 +1,27 @@
{
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
"extends": ["config:recommended"],
"timezone": "Europe/Berlin",
"schedule": ["before 6am on monday"],
"labels": ["dependencies"],
"prConcurrentLimit": 5,
"commitMessagePrefix": "chore(deps):",
"packageRules": [
{
"description": "Group safe minor+patch updates into one weekly PR per ecosystem",
"matchUpdateTypes": ["minor", "patch"],
"groupName": "{{manager}} minor & patch"
},
{
"description": "Major updates stay individual PRs for careful review",
"matchUpdateTypes": ["major"],
"dependencyDashboardApproval": true
}
],
"vulnerabilityAlerts": {
"enabled": true,
"labels": ["security"],
"schedule": ["at any time"]
},
"ignorePaths": ["**/node_modules/**", "**/bin/**", "**/obj/**"]
}
+2 -2
View File
@@ -1,5 +1,5 @@
# Multi-stage build for the ASP.NET Core API. # Multi-stage build for the ASP.NET Core API.
FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build
WORKDIR /src WORKDIR /src
# Copy solution + project files first for layer-cached restore. # Copy solution + project files first for layer-cached restore.
@@ -13,7 +13,7 @@ RUN dotnet restore src/InboxIntel.Api/InboxIntel.Api.csproj
COPY src/ src/ COPY src/ src/
RUN dotnet publish src/InboxIntel.Api/InboxIntel.Api.csproj -c Release -o /app/publish /p:UseAppHost=false RUN dotnet publish src/InboxIntel.Api/InboxIntel.Api.csproj -c Release -o /app/publish /p:UseAppHost=false
FROM mcr.microsoft.com/dotnet/aspnet:8.0 AS runtime FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS runtime
WORKDIR /app WORKDIR /app
COPY --from=build /app/publish . COPY --from=build /app/publish .
+3 -3
View File
@@ -5,10 +5,10 @@
<UserSecretsId>210c6d96-c7e4-4ee9-8982-8b91424979b8</UserSecretsId> <UserSecretsId>210c6d96-c7e4-4ee9-8982-8b91424979b8</UserSecretsId>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="Microsoft.AspNetCore.Authentication.Google" Version="8.0.7" /> <PackageReference Include="Microsoft.AspNetCore.Authentication.Google" Version="10.0.9" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="8.0.7" /> <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="10.0.9" />
<!-- Required on the startup project for `dotnet ef migrations` to work. --> <!-- Required on the startup project for `dotnet ef migrations` to work. -->
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.4"> <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="10.0.9">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets> <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets> <PrivateAssets>all</PrivateAssets>
</PackageReference> </PackageReference>
+5 -6
View File
@@ -38,8 +38,8 @@ var dp = builder.Services.AddDataProtection()
var dpCertPath = builder.Configuration["DataProtection:CertificatePath"]; var dpCertPath = builder.Configuration["DataProtection:CertificatePath"];
if (!string.IsNullOrWhiteSpace(dpCertPath)) if (!string.IsNullOrWhiteSpace(dpCertPath))
{ {
dp.ProtectKeysWithCertificate(new System.Security.Cryptography.X509Certificates.X509Certificate2( dp.ProtectKeysWithCertificate(System.Security.Cryptography.X509Certificates.X509CertificateLoader
dpCertPath, builder.Configuration["DataProtection:CertificatePassword"])); .LoadPkcs12FromFile(dpCertPath, builder.Configuration["DataProtection:CertificatePassword"]));
} }
builder.Services.AddApplication(); builder.Services.AddApplication();
@@ -201,15 +201,14 @@ var forwardedOptions = new ForwardedHeadersOptions
ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto | ForwardedHeaders.XForwardedHost, ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto | ForwardedHeaders.XForwardedHost,
ForwardLimit = app.Configuration.GetValue<int?>("ForwardedHeaders:ForwardLimit") ?? 1 ForwardLimit = app.Configuration.GetValue<int?>("ForwardedHeaders:ForwardLimit") ?? 1
}; };
forwardedOptions.KnownNetworks.Clear(); forwardedOptions.KnownIPNetworks.Clear();
forwardedOptions.KnownProxies.Clear(); forwardedOptions.KnownProxies.Clear();
var trustedNetworks = app.Configuration.GetSection("ForwardedHeaders:KnownNetworks").Get<string[]>() var trustedNetworks = app.Configuration.GetSection("ForwardedHeaders:KnownNetworks").Get<string[]>()
?? new[] { "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "127.0.0.0/8", "::1/128" }; ?? new[] { "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "127.0.0.0/8", "::1/128" };
foreach (var cidr in trustedNetworks) foreach (var cidr in trustedNetworks)
{ {
var parts = cidr.Split('/'); if (System.Net.IPNetwork.TryParse(cidr, out var network))
if (parts.Length == 2 && System.Net.IPAddress.TryParse(parts[0], out var prefix) && int.TryParse(parts[1], out var len)) forwardedOptions.KnownIPNetworks.Add(network);
forwardedOptions.KnownNetworks.Add(new Microsoft.AspNetCore.HttpOverrides.IPNetwork(prefix, len));
} }
app.UseForwardedHeaders(forwardedOptions); app.UseForwardedHeaders(forwardedOptions);
@@ -6,13 +6,13 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="FluentValidation" Version="11.9.2" /> <PackageReference Include="FluentValidation" Version="11.9.2" />
<PackageReference Include="FluentValidation.DependencyInjectionExtensions" Version="11.9.2" /> <PackageReference Include="FluentValidation.DependencyInjectionExtensions" Version="11.9.2" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.2" /> <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.9" />
<!-- DbSet<> is exposed on IAppDbContext so the Application layer can query. <!-- DbSet<> is exposed on IAppDbContext so the Application layer can query.
Pinned to 8.0.4 to match the Npgsql provider's Relational dependency. --> Pinned to 8.0.4 to match the Npgsql provider's Relational dependency. -->
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.4" /> <PackageReference Include="Microsoft.EntityFrameworkCore" Version="10.0.9" />
<!-- Transitive security pins: patch known .NET 8.0.0 advisories pulled in by EF Core. --> <!-- Transitive security pins: patch known .NET 8.0.0 advisories pulled in by EF Core. -->
<PackageReference Include="System.Text.Json" Version="8.0.6" /> <PackageReference Include="System.Text.Json" Version="10.0.9" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" /> <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="10.0.9" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\InboxIntel.Domain\InboxIntel.Domain.csproj" /> <ProjectReference Include="..\InboxIntel.Domain\InboxIntel.Domain.csproj" />
@@ -6,7 +6,7 @@
<ItemGroup> <ItemGroup>
<!-- NpgsqlTypes.NpgsqlTsVector (FTS) and Pgvector.Vector (semantic search) are used as <!-- NpgsqlTypes.NpgsqlTsVector (FTS) and Pgvector.Vector (semantic search) are used as
column types on the Email entity — same pragmatic precedent for both. --> column types on the Email entity — same pragmatic precedent for both. -->
<PackageReference Include="Npgsql" Version="8.0.3" /> <PackageReference Include="Npgsql" Version="10.0.2" />
<PackageReference Include="Pgvector" Version="0.2.0" /> <PackageReference Include="Pgvector" Version="0.3.0" />
</ItemGroup> </ItemGroup>
</Project> </Project>
@@ -4,18 +4,18 @@
<AssemblyName>InboxIntel.Infrastructure</AssemblyName> <AssemblyName>InboxIntel.Infrastructure</AssemblyName>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.4" /> <PackageReference Include="Microsoft.EntityFrameworkCore" Version="10.0.9" />
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.4" /> <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="10.0.2" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.4"> <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="10.0.9">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets> <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets> <PrivateAssets>all</PrivateAssets>
</PackageReference> </PackageReference>
<PackageReference Include="Google.Apis.Gmail.v1" Version="1.68.0.3427" /> <PackageReference Include="Google.Apis.Gmail.v1" Version="1.68.0.3427" />
<PackageReference Include="Google.Apis.Auth" Version="1.68.0" /> <PackageReference Include="Google.Apis.Auth" Version="1.68.0" />
<PackageReference Include="Microsoft.AspNetCore.DataProtection" Version="8.0.7" /> <PackageReference Include="Microsoft.AspNetCore.DataProtection" Version="10.0.9" />
<PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" /> <PackageReference Include="Microsoft.Extensions.Http" Version="10.0.9" />
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="8.0.0" /> <PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="10.0.9" />
<PackageReference Include="Pgvector.EntityFrameworkCore" Version="0.2.0" /> <PackageReference Include="Pgvector.EntityFrameworkCore" Version="0.3.0" />
<PackageReference Include="Polly" Version="8.4.1" /> <PackageReference Include="Polly" Version="8.4.1" />
<PackageReference Include="QuestPDF" Version="2024.7.0" /> <PackageReference Include="QuestPDF" Version="2024.7.0" />
<PackageReference Include="CsvHelper" Version="33.0.1" /> <PackageReference Include="CsvHelper" Version="33.0.1" />
@@ -24,9 +24,9 @@
<!-- Transitive security pins: patch known .NET 8.0.0 advisories pulled in by <!-- Transitive security pins: patch known .NET 8.0.0 advisories pulled in by
EF Core / ASP.NET / DataProtection. Remove once the parent packages ship EF Core / ASP.NET / DataProtection. Remove once the parent packages ship
these versions transitively. --> these versions transitively. -->
<PackageReference Include="System.Text.Json" Version="8.0.6" /> <PackageReference Include="System.Text.Json" Version="10.0.9" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" /> <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="10.0.9" />
<PackageReference Include="System.Security.Cryptography.Xml" Version="8.0.3" /> <PackageReference Include="System.Security.Cryptography.Xml" Version="10.0.9" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\InboxIntel.Application\InboxIntel.Application.csproj" /> <ProjectReference Include="..\InboxIntel.Application\InboxIntel.Application.csproj" />
@@ -5,6 +5,7 @@ using InboxIntel.Domain.Entities;
using InboxIntel.Domain.Enums; using InboxIntel.Domain.Enums;
using InboxIntel.Infrastructure.Persistence; using InboxIntel.Infrastructure.Persistence;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using Pgvector.EntityFrameworkCore;
namespace InboxIntel.Infrastructure.Search; namespace InboxIntel.Infrastructure.Search;
@@ -18,7 +19,13 @@ namespace InboxIntel.Infrastructure.Search;
public class SearchService : ISearchService public class SearchService : ISearchService
{ {
private readonly AppDbContext _db; private readonly AppDbContext _db;
public SearchService(AppDbContext db) => _db = db; private readonly IEmbeddingProvider? _embeddings;
public SearchService(AppDbContext db, IEmbeddingProvider? embeddings = null)
{
_db = db;
_embeddings = embeddings;
}
public async Task<PagedResult<EmailSummaryDto>> SearchAsync(Guid userId, SearchRequestDto r, CancellationToken ct = default) public async Task<PagedResult<EmailSummaryDto>> SearchAsync(Guid userId, SearchRequestDto r, CancellationToken ct = default)
{ {
@@ -73,6 +80,18 @@ public class SearchService : ISearchService
var total = await matched.CountAsync(ct); var total = await matched.CountAsync(ct);
// Hybrid semantic fusion (docs/discovery/05): when embeddings are available, fuse
// lexical top-K with vector top-K via Reciprocal Rank Fusion. Runs BEFORE the fuzzy
// fallback so a query with ZERO lexical hits (pure semantic recall — "gym receipt"
// phrased differently) still surfaces results. Exact lexical hits keep winning (they
// rank in both lists). Deeper pages fall through to lexical paging; any failure
// (Ollama down, nothing embedded yet) silently degrades to the lexical/fuzzy path.
if (hasFreeTextQuery && _embeddings is { IsAvailable: true })
{
var hybrid = await TryHybridAsync(structured, matched, term, total, r, ct);
if (hybrid is not null) return hybrid;
}
// Fuzzy/typo fallback: ONLY when a free-text search found nothing exact. word_similarity // Fuzzy/typo fallback: ONLY when a free-text search found nothing exact. word_similarity
// with an explicit 0.3 threshold — pg_trgm's default 0.6 misses real typos // with an explicit 0.3 threshold — pg_trgm's default 0.6 misses real typos
// ("recieved" -> "received" scores ~0.39). Rare path, so the (non-indexed) scan over the // ("recieved" -> "received" scores ~0.39). Rare path, so the (non-indexed) scan over the
@@ -137,4 +156,73 @@ public class SearchService : ISearchService
TotalCount = total TotalCount = total
}; };
} }
private const int HybridK = 50; // candidates taken from each layer
private const int RrfConstant = 60; // standard RRF dampening constant
/// <summary>
/// RRF fusion of lexical and vector candidates. Returns null when the requested page
/// lies beyond the fused window or anything fails — caller falls back to lexical.
/// </summary>
private async Task<PagedResult<EmailSummaryDto>?> TryHybridAsync(
IQueryable<Email> structured, IQueryable<Email> lexical, string term, int lexicalTotal,
SearchRequestDto r, CancellationToken ct)
{
try
{
var lexIds = await lexical
.OrderByDescending(e => e.SearchVector!.RankCoverDensity(EF.Functions.WebSearchToTsQuery("english", term)))
.ThenByDescending(e => e.SentAtUtc)
.Take(HybridK).Select(e => e.Id).ToListAsync(ct);
var queryVec = await _embeddings!.EmbedAsync(term, ct);
List<Guid> vecIds = new();
if (queryVec.Length > 0)
{
var qv = new Pgvector.Vector(queryVec);
vecIds = await structured
.Where(e => e.Embedding != null)
.OrderBy(e => e.Embedding!.CosineDistance(qv))
.Take(HybridK).Select(e => e.Id).ToListAsync(ct);
}
if (vecIds.Count == 0) return null; // nothing embedded yet → lexical path
var scores = new Dictionary<Guid, double>();
for (var i = 0; i < lexIds.Count; i++)
scores[lexIds[i]] = scores.GetValueOrDefault(lexIds[i]) + 1.0 / (RrfConstant + i + 1);
for (var i = 0; i < vecIds.Count; i++)
scores[vecIds[i]] = scores.GetValueOrDefault(vecIds[i]) + 1.0 / (RrfConstant + i + 1);
var fused = scores.OrderByDescending(kv => kv.Value).Select(kv => kv.Key).ToList();
var pageIds = fused.Skip((r.Page - 1) * r.PageSize).Take(r.PageSize).ToList();
if (pageIds.Count == 0 && r.Page > 1) return null; // deep page → lexical paging
var headlineOpts =
$"StartSel={(char)0xE000},StopSel={(char)0xE001},MaxWords=16,MinWords=5,ShortWord=2,HighlightAll=false";
var rows = await _db.Emails.AsNoTracking()
.Where(e => pageIds.Contains(e.Id))
.Select(e => new EmailSummaryDto(
e.Id, e.GmailMessageId, e.Subject, e.Snippet,
e.Sender!.Address, e.Sender.DisplayName, e.SentAtUtc,
e.IsUnread, e.IsStarred, e.HasAttachments, e.SizeEstimateBytes, e.Category,
e.HasListUnsubscribe, e.SupportsOneClickUnsubscribe,
EF.Functions.WebSearchToTsQuery("english", term).GetResultHeadline("english", e.BodyText ?? "", headlineOpts)))
.ToListAsync(ct);
var byId = rows.ToDictionary(x => x.Id);
var items = pageIds.Where(byId.ContainsKey).Select(id => byId[id]).ToList();
return new PagedResult<EmailSummaryDto>
{
Items = items,
Page = r.Page,
PageSize = r.PageSize,
// Semantic recall can exceed the lexical match count.
TotalCount = Math.Max(lexicalTotal, fused.Count)
};
}
catch
{
return null; // AI must never break search — degrade to lexical
}
}
} }
@@ -25,7 +25,7 @@ namespace InboxIntel.IntegrationTests;
/// endpoints (model validation, per-user rate limits) without a real Google login.</summary> /// endpoints (model validation, per-user rate limits) without a real Google login.</summary>
public class TestAuthHandler : AuthenticationHandler<AuthenticationSchemeOptions> public class TestAuthHandler : AuthenticationHandler<AuthenticationSchemeOptions>
{ {
public const string Scheme = "Test"; public new const string Scheme = "Test";
// Stable across requests so per-user rate-limit partitions accumulate correctly. // Stable across requests so per-user rate-limit partitions accumulate correctly.
public static readonly string Uid = Guid.NewGuid().ToString(); public static readonly string Uid = Guid.NewGuid().ToString();
@@ -52,6 +52,9 @@ public class AuditTestAppFactory : WebApplicationFactory<Program>
builder.ConfigureHostConfiguration(cfg => cfg.AddInMemoryCollection(new Dictionary<string, string?> builder.ConfigureHostConfiguration(cfg => cfg.AddInMemoryCollection(new Dictionary<string, string?>
{ {
["Database:AutoMigrate"] = "false", ["Database:AutoMigrate"] = "false",
// Npgsql 10 eagerly validates the connection string when the DbContext is
// resolved (8.x was lazy); these tests never connect, but the string must parse.
["ConnectionStrings:Postgres"] = "Host=localhost;Database=test;Username=test;Password=test",
["GoogleOAuth:ClientId"] = "test-client-id", ["GoogleOAuth:ClientId"] = "test-client-id",
["GoogleOAuth:ClientSecret"] = "test-client-secret", ["GoogleOAuth:ClientSecret"] = "test-client-secret",
// H-2: make the auth policy trip on the 3rd request within the window. // H-2: make the auth policy trip on the 3rd request within the window.
@@ -19,6 +19,9 @@ public class TestAppFactory : WebApplicationFactory<Program>
builder.ConfigureHostConfiguration(cfg => cfg.AddInMemoryCollection(new Dictionary<string, string?> builder.ConfigureHostConfiguration(cfg => cfg.AddInMemoryCollection(new Dictionary<string, string?>
{ {
["Database:AutoMigrate"] = "false", ["Database:AutoMigrate"] = "false",
// Npgsql 10 eagerly validates the connection string when the DbContext is
// resolved (8.x was lazy); these tests never connect, but the string must parse.
["ConnectionStrings:Postgres"] = "Host=localhost;Database=test;Username=test;Password=test",
// Dummy OAuth creds so the Google challenge produces a real 302 redirect // Dummy OAuth creds so the Google challenge produces a real 302 redirect
// (an empty ClientId can make the handler throw instead of redirecting). // (an empty ClientId can make the handler throw instead of redirecting).
["GoogleOAuth:ClientId"] = "test-client-id", ["GoogleOAuth:ClientId"] = "test-client-id",
@@ -7,8 +7,8 @@
<PackageReference Include="xunit" Version="2.9.0" /> <PackageReference Include="xunit" Version="2.9.0" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" /> <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" />
<PackageReference Include="FluentAssertions" Version="6.12.0" /> <PackageReference Include="FluentAssertions" Version="6.12.0" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="8.0.7" /> <PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.9" />
<PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" Version="8.0.4" /> <PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" Version="10.0.9" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\..\src\InboxIntel.Api\InboxIntel.Api.csproj" /> <ProjectReference Include="..\..\src\InboxIntel.Api\InboxIntel.Api.csproj" />
@@ -123,4 +123,54 @@ public class LiveDbSearchTests
} }
finally { await CleanupAsync(opts, uid); } finally { await CleanupAsync(opts, uid); }
} }
private sealed class DirectionalFakeEmbeddings : IEmbeddingProvider
{
public bool IsAvailable => true;
public Task<float[]> EmbedAsync(string text, CancellationToken ct = default)
{
// Deterministic "semantics": anything fruit-flavoured points one way, else the other.
var v = new float[768];
if (text.Contains("banana") || text.Contains("tropical")) v[0] = 1; else v[1] = 1;
return Task.FromResult(v);
}
public async Task<IReadOnlyList<float[]>> EmbedBatchAsync(IReadOnlyList<string> texts, CancellationToken ct = default)
{
var list = new List<float[]>();
foreach (var t in texts) list.Add(await EmbedAsync(t, ct));
return list;
}
}
[Fact]
public async Task Hybrid_search_surfaces_semantic_match_with_zero_keyword_overlap()
{
if (Conn is null) return;
var opts = Options();
var uid = await SeedAsync(opts);
try
{
var embeddings = new DirectionalFakeEmbeddings();
using (var prep = new AppDbContext(opts, new FakeCurrentUser()))
{
// "Weekly notes" gets a fruit-direction embedding (semantically related to the
// query); "Invoice March" points elsewhere. Neither subject contains "banana".
var near = await prep.Emails.FirstAsync(e => e.UserId == uid && e.Subject == "Weekly notes");
near.Embedding = new Vector(await embeddings.EmbedAsync("tropical"));
var far = await prep.Emails.FirstAsync(e => e.UserId == uid && e.Subject == "Invoice March");
far.Embedding = new Vector(await embeddings.EmbedAsync("finance"));
await prep.SaveChangesAsync();
}
using var ctx = new AppDbContext(opts, new FakeCurrentUser { UserId = uid });
var res = await new SearchService(ctx, embeddings)
.SearchAsync(uid, GmailQueryParser.Parse("banana", 1, 10));
// Zero lexical hits for "banana" — hybrid must still surface the semantically
// nearest email, ranked first.
res.Items.Should().NotBeEmpty("semantic recall should fire with zero keyword overlap");
res.Items[0].Subject.Should().Be("Weekly notes");
}
finally { await CleanupAsync(opts, uid); }
}
} }