Compare commits

..

1 Commits

Author SHA1 Message Date
cesnimda b31758e23d feat(ci): Renovate automated dependency updates (RECOMMENDATIONS #2)
CI / backend (pull_request) Successful in 54s
CI / frontend (pull_request) Successful in 16s
CI / format (pull_request) Successful in 56s
CI / db-tests (pull_request) Successful in 52s
Security / secrets (pull_request) Successful in 4s
Security / dependencies (pull_request) Successful in 58s
Weekly Renovate run against this Gitea instance: grouped minor/patch PRs per
ecosystem, individually-reviewed majors (dashboard approval), immediate
security-alert PRs. All update PRs ride the existing required CI gates
(build/tests/gitleaks/vuln/live-db/format), so merging one is as safe as any
other PR.

ONE MANUAL STEP remains: create a Gitea PAT (repo rw, user r, issue rw, org r)
and add it as the RENOVATE_TOKEN Actions secret — the workflow fails fast with
instructions until then.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-02 16:20:50 +02:00
21 changed files with 59 additions and 383 deletions
-3
View File
@@ -15,6 +15,3 @@ FRONTEND_ORIGIN=http://localhost:8081
# Set DEV_MODE=true and MAX_MESSAGES=1000 to test against a large mailbox. # Set DEV_MODE=true and MAX_MESSAGES=1000 to test against a large mailbox.
DEV_MODE=false DEV_MODE=false
MAX_MESSAGES=0 MAX_MESSAGES=0
# Nightly DB backup rotation (days of dumps to keep in ./backups)
BACKUP_KEEP_DAYS=7
+3 -3
View File
@@ -14,7 +14,7 @@ jobs:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
- uses: actions/setup-dotnet@v4 - uses: actions/setup-dotnet@v4
with: with:
dotnet-version: '10.0.x' dotnet-version: '8.0.x'
- name: Restore - name: Restore
run: dotnet restore InboxIntel.sln run: dotnet restore InboxIntel.sln
- name: Build - name: Build
@@ -45,7 +45,7 @@ jobs:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
- uses: actions/setup-dotnet@v4 - uses: actions/setup-dotnet@v4
with: with:
dotnet-version: '10.0.x' dotnet-version: '8.0.x'
- name: dotnet format (verify only) - name: dotnet format (verify only)
run: dotnet format InboxIntel.sln --verify-no-changes run: dotnet format InboxIntel.sln --verify-no-changes
@@ -67,7 +67,7 @@ jobs:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
- uses: actions/setup-dotnet@v4 - uses: actions/setup-dotnet@v4
with: with:
dotnet-version: '10.0.x' dotnet-version: '8.0.x'
- name: Wait for Postgres - name: Wait for Postgres
run: | run: |
for i in $(seq 1 30); do for i in $(seq 1 30); do
+1 -1
View File
@@ -31,7 +31,7 @@ jobs:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
- uses: actions/setup-dotnet@v4 - uses: actions/setup-dotnet@v4
with: with:
dotnet-version: '10.0.x' dotnet-version: '8.0.x'
- name: Restore - name: Restore
run: dotnet restore InboxIntel.sln run: dotnet restore InboxIntel.sln
- name: .NET vulnerable packages (fail on any) - name: .NET vulnerable packages (fail on any)
-3
View File
@@ -21,9 +21,6 @@ frontend/.vite/
appsettings.*.local.json appsettings.*.local.json
secrets.json secrets.json
## DB backups (never commit dumps)
backups/
## Logs ## Logs
logs/ logs/
*.log *.log
+1 -1
View File
@@ -1,6 +1,6 @@
<Project> <Project>
<PropertyGroup> <PropertyGroup>
<TargetFramework>net10.0</TargetFramework> <TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable> <Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings> <ImplicitUsings>enable</ImplicitUsings>
<LangVersion>latest</LangVersion> <LangVersion>latest</LangVersion>
+1 -3
View File
@@ -29,9 +29,7 @@ reverse proxy.
mitigate (a third party reading the DB files) reduces to "someone with access to your mitigate (a third party reading the DB files) reduces to "someone with access to your
machine" — mitigate it at the layer that actually works: machine" — mitigate it at the layer that actually works:
- **Use full-disk or volume encryption** on the host (BitLocker/LUKS) — strongly recommended. - **Use full-disk or volume encryption** on the host (BitLocker/LUKS) — strongly recommended.
- **Encrypt backups**: nightly `pg_dump` rotation runs via the compose `backup` service - **Encrypt backups** of the `pgdata` volume the same way.
into `./backups/` (git-ignored) — keep that directory on an encrypted disk and copy it
off-machine. Restore: `docker compose exec -T postgres psql -U inboxintel -d inboxintel < backups/<file>.sql`.
- Before any **multi-user** deployment, revisit per the multi-provider security design - Before any **multi-user** deployment, revisit per the multi-provider security design
(host admins must not be able to read members' mail — plaintext bodies break that promise). (host admins must not be able to read members' mail — plaintext bodies break that promise).
2. **DB connection is not TLS** — Postgres is only reachable on the compose-internal network / 2. **DB connection is not TLS** — Postgres is only reachable on the compose-internal network /
-63
View File
@@ -22,37 +22,6 @@ services:
timeout: 5s timeout: 5s
retries: 10 retries: 10
# Nightly logical backups (RECOMMENDATIONS #3 — previously there were NONE). Dumps
# rotate after BACKUP_KEEP_DAYS. The ./backups host directory should live on an
# encrypted disk and be included in your off-machine backup regime (see SECURITY.md).
# Restore: docker compose exec -T postgres psql -U inboxintel -d inboxintel < backups/<file>.sql
backup:
image: pgvector/pgvector:pg16
entrypoint: /bin/sh
command:
- -c
- |
while true; do
ts=$$(date -u +%Y%m%d-%H%M%S)
if pg_dump -h postgres -U inboxintel -d inboxintel > /backups/inboxintel-$$ts.sql.tmp; then
mv /backups/inboxintel-$$ts.sql.tmp /backups/inboxintel-$$ts.sql
echo "backup OK: inboxintel-$$ts.sql"
else
rm -f /backups/inboxintel-$$ts.sql.tmp
echo "backup FAILED at $$ts" >&2
fi
find /backups -name 'inboxintel-*.sql' -mtime +$${BACKUP_KEEP_DAYS:-7} -delete
sleep 86400
done
environment:
PGPASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in deploy/.env}
BACKUP_KEEP_DAYS: ${BACKUP_KEEP_DAYS:-7}
volumes:
- ./backups:/backups
depends_on:
postgres:
condition: service_healthy
api: api:
build: build:
context: . context: .
@@ -65,10 +34,6 @@ services:
GoogleOAuth__ClientId: ${GOOGLE_CLIENT_ID:-} GoogleOAuth__ClientId: ${GOOGLE_CLIENT_ID:-}
GoogleOAuth__ClientSecret: ${GOOGLE_CLIENT_SECRET:-} GoogleOAuth__ClientSecret: ${GOOGLE_CLIENT_SECRET:-}
Ai__Mode: ${AI_MODE:-Disabled} Ai__Mode: ${AI_MODE:-Disabled}
# Points at the compose 'ollama' service when the ai profile is up; harmless otherwise.
Ai__OllamaBaseUrl: ${OLLAMA_BASE_URL:-http://ollama:11434}
# OTLP export activates only when set (e.g. http://lgtm:4317 with the observability profile).
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_ENDPOINT:-}
# Dev mode shows the dev banner and caps the initial sync. Set DEV_MODE=true # Dev mode shows the dev banner and caps the initial sync. Set DEV_MODE=true
# and MAX_MESSAGES=1000 in deploy/.env to exercise it in this Docker setup. # and MAX_MESSAGES=1000 in deploy/.env to exercise it in this Docker setup.
App__DevMode: ${DEV_MODE:-false} App__DevMode: ${DEV_MODE:-false}
@@ -94,33 +59,6 @@ services:
ports: ports:
- "8081:80" - "8081:80"
# Local AI (semantic search + assistants). Enable with:
# docker compose --profile ai up -d && set AI_MODE=LocalOllama in deploy/.env
# First run: docker compose exec ollama ollama pull nomic-embed-text
# GPU (RTX 3080): uncomment the deploy block to pass the GPU through.
ollama:
image: ollama/ollama
profiles: ["ai"]
volumes:
- ollama:/root/.ollama
# deploy:
# resources:
# reservations:
# devices:
# - driver: nvidia
# count: all
# capabilities: [gpu]
# Observability (RECOMMENDATIONS #5): all-in-one Grafana+Tempo+Prometheus+Loki.
# Enable with: docker compose --profile observability up -d
# then set OTEL_ENDPOINT=http://lgtm:4317 in deploy/.env and restart the api.
# Grafana UI: http://localhost:3000 (admin/admin on first run).
lgtm:
image: grafana/otel-lgtm
profiles: ["observability"]
ports:
- "127.0.0.1:3000:3000"
# Optional reverse proxy. Enable with: docker compose --profile proxy up # Optional reverse proxy. Enable with: docker compose --profile proxy up
nginx: nginx:
image: nginx:alpine image: nginx:alpine
@@ -136,4 +74,3 @@ services:
volumes: volumes:
pgdata: pgdata:
keys: keys:
ollama:
+23 -34
View File
@@ -1,53 +1,42 @@
import React, { Suspense, lazy } from 'react'; import React from 'react';
import ReactDOM from 'react-dom/client'; import ReactDOM from 'react-dom/client';
import { BrowserRouter, Routes, Route, Navigate } from 'react-router-dom'; import { BrowserRouter, Routes, Route, Navigate } from 'react-router-dom';
import Landing from './pages/Landing.jsx'; import Landing from './pages/Landing.jsx';
import Dashboard from './pages/Dashboard.jsx';
import Senders from './pages/Senders.jsx';
import Cleanup from './pages/Cleanup.jsx';
import Unsubscribe from './pages/Unsubscribe.jsx';
import FolderView from './pages/FolderView.jsx';
import SearchResults from './pages/SearchResults.jsx';
import Layout from './components/Layout.jsx'; import Layout from './components/Layout.jsx';
import DesignSystem from './pages/DesignSystem.jsx';
import { ToastProvider, TooltipProvider } from './components/ui'; import { ToastProvider, TooltipProvider } from './components/ui';
import '@fontsource-variable/inter'; import '@fontsource-variable/inter';
import './index.css'; import './index.css';
import './styles.css'; import './styles.css';
// Route-level code splitting: each page loads its own chunk on first visit, so the
// initial bundle no longer carries Chart.js / grid-layout / every page at once.
// Landing + Layout stay eager (they're the first paint).
const Dashboard = lazy(() => import('./pages/Dashboard.jsx'));
const Senders = lazy(() => import('./pages/Senders.jsx'));
const Cleanup = lazy(() => import('./pages/Cleanup.jsx'));
const Unsubscribe = lazy(() => import('./pages/Unsubscribe.jsx'));
const FolderView = lazy(() => import('./pages/FolderView.jsx'));
const SearchResults = lazy(() => import('./pages/SearchResults.jsx'));
const DesignSystem = lazy(() => import('./pages/DesignSystem.jsx'));
// Minimal, theme-correct route fallback (skeleton-style, per the design system).
const RouteFallback = () => (
<div className="p-6 text-sm text-muted-foreground" aria-busy="true">Loading</div>
);
ReactDOM.createRoot(document.getElementById('root')).render( ReactDOM.createRoot(document.getElementById('root')).render(
<React.StrictMode> <React.StrictMode>
<ToastProvider> <ToastProvider>
<TooltipProvider delayDuration={200}> <TooltipProvider delayDuration={200}>
<BrowserRouter> <BrowserRouter>
<Suspense fallback={<RouteFallback />}> <Routes>
<Routes> {/* Public landing page */}
{/* Public landing page */} <Route path="/" element={<Landing />} />
<Route path="/" element={<Landing />} />
{/* Authenticated app */} {/* Authenticated app */}
<Route path="/app" element={<Layout />}> <Route path="/app" element={<Layout />}>
<Route index element={<Dashboard />} /> <Route index element={<Dashboard />} />
<Route path="senders" element={<Senders />} /> <Route path="senders" element={<Senders />} />
<Route path="cleanup" element={<Cleanup />} /> <Route path="cleanup" element={<Cleanup />} />
<Route path="unsubscribe" element={<Unsubscribe />} /> <Route path="unsubscribe" element={<Unsubscribe />} />
<Route path="folder/:slug" element={<FolderView />} /> <Route path="folder/:slug" element={<FolderView />} />
<Route path="search" element={<SearchResults />} /> <Route path="search" element={<SearchResults />} />
<Route path="design" element={<DesignSystem />} /> <Route path="design" element={<DesignSystem />} />
</Route> </Route>
<Route path="*" element={<Navigate to="/" replace />} /> <Route path="*" element={<Navigate to="/" replace />} />
</Routes> </Routes>
</Suspense>
</BrowserRouter> </BrowserRouter>
</TooltipProvider> </TooltipProvider>
</ToastProvider> </ToastProvider>
+2 -2
View File
@@ -1,5 +1,5 @@
# Multi-stage build for the ASP.NET Core API. # Multi-stage build for the ASP.NET Core API.
FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
WORKDIR /src WORKDIR /src
# Copy solution + project files first for layer-cached restore. # Copy solution + project files first for layer-cached restore.
@@ -13,7 +13,7 @@ RUN dotnet restore src/InboxIntel.Api/InboxIntel.Api.csproj
COPY src/ src/ COPY src/ src/
RUN dotnet publish src/InboxIntel.Api/InboxIntel.Api.csproj -c Release -o /app/publish /p:UseAppHost=false RUN dotnet publish src/InboxIntel.Api/InboxIntel.Api.csproj -c Release -o /app/publish /p:UseAppHost=false
FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS runtime FROM mcr.microsoft.com/dotnet/aspnet:8.0 AS runtime
WORKDIR /app WORKDIR /app
COPY --from=build /app/publish . COPY --from=build /app/publish .
+3 -8
View File
@@ -5,21 +5,16 @@
<UserSecretsId>210c6d96-c7e4-4ee9-8982-8b91424979b8</UserSecretsId> <UserSecretsId>210c6d96-c7e4-4ee9-8982-8b91424979b8</UserSecretsId>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="Microsoft.AspNetCore.Authentication.Google" Version="10.0.9" /> <PackageReference Include="Microsoft.AspNetCore.Authentication.Google" Version="8.0.7" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="10.0.9" /> <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="8.0.7" />
<!-- Required on the startup project for `dotnet ef migrations` to work. --> <!-- Required on the startup project for `dotnet ef migrations` to work. -->
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="10.0.9"> <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.4">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets> <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets> <PrivateAssets>all</PrivateAssets>
</PackageReference> </PackageReference>
<PackageReference Include="Asp.Versioning.Mvc" Version="8.1.0" /> <PackageReference Include="Asp.Versioning.Mvc" Version="8.1.0" />
<PackageReference Include="Asp.Versioning.Mvc.ApiExplorer" Version="8.1.0" /> <PackageReference Include="Asp.Versioning.Mvc.ApiExplorer" Version="8.1.0" />
<PackageReference Include="FluentValidation.AspNetCore" Version="11.3.0" /> <PackageReference Include="FluentValidation.AspNetCore" Version="11.3.0" />
<PackageReference Include="Npgsql.OpenTelemetry" Version="10.0.3" />
<PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.16.0" />
<PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.16.0" />
<PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.16.0" />
<PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.16.0" />
<PackageReference Include="Serilog.AspNetCore" Version="8.0.1" /> <PackageReference Include="Serilog.AspNetCore" Version="8.0.1" />
<PackageReference Include="Serilog.Sinks.Console" Version="5.0.1" /> <PackageReference Include="Serilog.Sinks.Console" Version="5.0.1" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2" /> <PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2" />
+6 -31
View File
@@ -16,10 +16,6 @@ using Microsoft.AspNetCore.HttpOverrides;
using Microsoft.AspNetCore.RateLimiting; using Microsoft.AspNetCore.RateLimiting;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using Npgsql;
using OpenTelemetry.Metrics;
using OpenTelemetry.Resources;
using OpenTelemetry.Trace;
using Serilog; using Serilog;
var builder = WebApplication.CreateBuilder(args); var builder = WebApplication.CreateBuilder(args);
@@ -42,8 +38,8 @@ var dp = builder.Services.AddDataProtection()
var dpCertPath = builder.Configuration["DataProtection:CertificatePath"]; var dpCertPath = builder.Configuration["DataProtection:CertificatePath"];
if (!string.IsNullOrWhiteSpace(dpCertPath)) if (!string.IsNullOrWhiteSpace(dpCertPath))
{ {
dp.ProtectKeysWithCertificate(System.Security.Cryptography.X509Certificates.X509CertificateLoader dp.ProtectKeysWithCertificate(new System.Security.Cryptography.X509Certificates.X509Certificate2(
.LoadPkcs12FromFile(dpCertPath, builder.Configuration["DataProtection:CertificatePassword"])); dpCertPath, builder.Configuration["DataProtection:CertificatePassword"]));
} }
builder.Services.AddApplication(); builder.Services.AddApplication();
@@ -126,28 +122,6 @@ builder.Services.AddAuthentication(options =>
builder.Services.AddAuthorization(); builder.Services.AddAuthorization();
// RECOMMENDATIONS #5: OpenTelemetry traces + metrics (ASP.NET, outbound HTTP, Npgsql).
// The OTLP exporter only activates when Otel:Endpoint (or the standard
// OTEL_EXPORTER_OTLP_ENDPOINT env var) is configured — zero overhead otherwise.
// Logs stay on Serilog. Pair with the compose "observability" profile (grafana/otel-lgtm).
var otlpEndpoint = builder.Configuration["Otel:Endpoint"]
?? Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_ENDPOINT");
if (!string.IsNullOrWhiteSpace(otlpEndpoint))
{
builder.Services.AddOpenTelemetry()
.ConfigureResource(r => r.AddService("inboxintel-api"))
.WithTracing(t => t
.AddAspNetCoreInstrumentation()
.AddHttpClientInstrumentation()
.AddNpgsql()
.AddOtlpExporter(o => o.Endpoint = new Uri(otlpEndpoint)))
.WithMetrics(m => m
.AddAspNetCoreInstrumentation()
.AddHttpClientInstrumentation()
.AddNpgsqlInstrumentation()
.AddOtlpExporter(o => o.Endpoint = new Uri(otlpEndpoint)));
}
builder.Services.AddApiVersioning(o => builder.Services.AddApiVersioning(o =>
{ {
o.DefaultApiVersion = new ApiVersion(1, 0); o.DefaultApiVersion = new ApiVersion(1, 0);
@@ -227,14 +201,15 @@ var forwardedOptions = new ForwardedHeadersOptions
ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto | ForwardedHeaders.XForwardedHost, ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto | ForwardedHeaders.XForwardedHost,
ForwardLimit = app.Configuration.GetValue<int?>("ForwardedHeaders:ForwardLimit") ?? 1 ForwardLimit = app.Configuration.GetValue<int?>("ForwardedHeaders:ForwardLimit") ?? 1
}; };
forwardedOptions.KnownIPNetworks.Clear(); forwardedOptions.KnownNetworks.Clear();
forwardedOptions.KnownProxies.Clear(); forwardedOptions.KnownProxies.Clear();
var trustedNetworks = app.Configuration.GetSection("ForwardedHeaders:KnownNetworks").Get<string[]>() var trustedNetworks = app.Configuration.GetSection("ForwardedHeaders:KnownNetworks").Get<string[]>()
?? new[] { "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "127.0.0.0/8", "::1/128" }; ?? new[] { "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "127.0.0.0/8", "::1/128" };
foreach (var cidr in trustedNetworks) foreach (var cidr in trustedNetworks)
{ {
if (System.Net.IPNetwork.TryParse(cidr, out var network)) var parts = cidr.Split('/');
forwardedOptions.KnownIPNetworks.Add(network); if (parts.Length == 2 && System.Net.IPAddress.TryParse(parts[0], out var prefix) && int.TryParse(parts[1], out var len))
forwardedOptions.KnownNetworks.Add(new Microsoft.AspNetCore.HttpOverrides.IPNetwork(prefix, len));
} }
app.UseForwardedHeaders(forwardedOptions); app.UseForwardedHeaders(forwardedOptions);
@@ -6,13 +6,13 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="FluentValidation" Version="11.9.2" /> <PackageReference Include="FluentValidation" Version="11.9.2" />
<PackageReference Include="FluentValidation.DependencyInjectionExtensions" Version="11.9.2" /> <PackageReference Include="FluentValidation.DependencyInjectionExtensions" Version="11.9.2" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.9" /> <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.2" />
<!-- DbSet<> is exposed on IAppDbContext so the Application layer can query. <!-- DbSet<> is exposed on IAppDbContext so the Application layer can query.
Pinned to 8.0.4 to match the Npgsql provider's Relational dependency. --> Pinned to 8.0.4 to match the Npgsql provider's Relational dependency. -->
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="10.0.9" /> <PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.4" />
<!-- Transitive security pins: patch known .NET 8.0.0 advisories pulled in by EF Core. --> <!-- Transitive security pins: patch known .NET 8.0.0 advisories pulled in by EF Core. -->
<PackageReference Include="System.Text.Json" Version="10.0.9" /> <PackageReference Include="System.Text.Json" Version="8.0.6" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="10.0.9" /> <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\InboxIntel.Domain\InboxIntel.Domain.csproj" /> <ProjectReference Include="..\InboxIntel.Domain\InboxIntel.Domain.csproj" />
@@ -6,7 +6,7 @@
<ItemGroup> <ItemGroup>
<!-- NpgsqlTypes.NpgsqlTsVector (FTS) and Pgvector.Vector (semantic search) are used as <!-- NpgsqlTypes.NpgsqlTsVector (FTS) and Pgvector.Vector (semantic search) are used as
column types on the Email entity — same pragmatic precedent for both. --> column types on the Email entity — same pragmatic precedent for both. -->
<PackageReference Include="Npgsql" Version="10.0.2" /> <PackageReference Include="Npgsql" Version="8.0.3" />
<PackageReference Include="Pgvector" Version="0.3.0" /> <PackageReference Include="Pgvector" Version="0.2.0" />
</ItemGroup> </ItemGroup>
</Project> </Project>
@@ -1,104 +0,0 @@
using InboxIntel.Application.Abstractions;
using InboxIntel.Infrastructure.Persistence;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace InboxIntel.Infrastructure.Ai;
/// <summary>
/// Fills <c>Email.Embedding</c> (pgvector) for semantic search, in small background batches
/// so interactive requests are never starved (per docs/discovery/06: embeddings are the
/// small always-on model; the batch pause keeps VRAM/CPU pressure low). Exits immediately
/// when the embedding provider is unavailable (AI disabled / Ollama down) — semantic search
/// simply stays dormant and lexical search is unaffected.
/// </summary>
public class EmbeddingBackfillWorker : BackgroundService
{
private const int BatchSize = 32;
private static readonly TimeSpan BatchPause = TimeSpan.FromSeconds(2);
private static readonly TimeSpan IdleRescan = TimeSpan.FromMinutes(15);
private readonly IServiceScopeFactory _scopeFactory;
private readonly ILogger<EmbeddingBackfillWorker> _logger;
public EmbeddingBackfillWorker(IServiceScopeFactory scopeFactory, ILogger<EmbeddingBackfillWorker> logger)
{
_scopeFactory = scopeFactory;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
// Provider availability is fixed by configuration for the process lifetime.
using (var probe = _scopeFactory.CreateScope())
{
if (!probe.ServiceProvider.GetRequiredService<IEmbeddingProvider>().IsAvailable)
{
_logger.LogDebug("EmbeddingBackfillWorker idle: no embedding provider (AI disabled).");
return;
}
}
_logger.LogInformation("EmbeddingBackfillWorker started (batch {Batch}, pause {Pause}s)",
BatchSize, BatchPause.TotalSeconds);
while (!stoppingToken.IsCancellationRequested)
{
int processed;
try
{
processed = await ProcessBatchAsync(stoppingToken);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; }
catch (Exception ex)
{
// Ollama hiccups must never crash the host; back off and retry.
_logger.LogWarning(ex, "Embedding batch failed; retrying after idle pause.");
processed = 0;
}
await Task.Delay(processed > 0 ? BatchPause : IdleRescan, stoppingToken);
}
}
/// <summary>Embeds one batch. Public-ish (internal) for direct testing.</summary>
internal async Task<int> ProcessBatchAsync(CancellationToken ct)
{
using var scope = _scopeFactory.CreateScope();
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
var embeddings = scope.ServiceProvider.GetRequiredService<IEmbeddingProvider>();
var batch = await db.Emails
.Where(e => e.Embedding == null)
.OrderByDescending(e => e.SentAtUtc) // newest mail becomes searchable first
.Take(BatchSize)
.ToListAsync(ct);
if (batch.Count == 0) return 0;
// Subject + snippet is the semantic core; bodies are noisy (signatures, quoting)
// and slow to embed. Truncate defensively to keep well inside the model context.
var texts = batch
.Select(e => Truncate($"{e.Subject}\n{e.Snippet ?? e.BodyText}", 2000))
.ToList();
var vectors = await embeddings.EmbedBatchAsync(texts, ct);
if (vectors.Count != batch.Count)
{
_logger.LogWarning("Embedding batch returned {Got} vectors for {Want} emails; skipping batch.",
vectors.Count, batch.Count);
return 0;
}
for (var i = 0; i < batch.Count; i++)
{
if (vectors[i].Length == 0) continue; // provider soft-failure for one item
batch[i].Embedding = new Pgvector.Vector(vectors[i]);
}
await db.SaveChangesAsync(ct);
_logger.LogDebug("Embedded {Count} emails", batch.Count);
return batch.Count;
}
private static string Truncate(string s, int max) => s.Length <= max ? s : s[..max];
}
@@ -92,9 +92,6 @@ public static class DependencyInjection
break; break;
} }
services.AddScoped<IAiService, AiService>(); services.AddScoped<IAiService, AiService>();
// Semantic search: fills Email.Embedding in the background; no-ops when the
// embedding provider is unavailable (AI disabled), so lexical search is unaffected.
services.AddHostedService<EmbeddingBackfillWorker>();
// Background worker (daily incremental sync + aggregate refresh) // Background worker (daily incremental sync + aggregate refresh)
services.AddHostedService<GmailSyncWorker>(); services.AddHostedService<GmailSyncWorker>();
@@ -4,18 +4,18 @@
<AssemblyName>InboxIntel.Infrastructure</AssemblyName> <AssemblyName>InboxIntel.Infrastructure</AssemblyName>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="10.0.9" /> <PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.4" />
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="10.0.2" /> <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.4" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="10.0.9"> <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.4">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets> <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets> <PrivateAssets>all</PrivateAssets>
</PackageReference> </PackageReference>
<PackageReference Include="Google.Apis.Gmail.v1" Version="1.68.0.3427" /> <PackageReference Include="Google.Apis.Gmail.v1" Version="1.68.0.3427" />
<PackageReference Include="Google.Apis.Auth" Version="1.68.0" /> <PackageReference Include="Google.Apis.Auth" Version="1.68.0" />
<PackageReference Include="Microsoft.AspNetCore.DataProtection" Version="10.0.9" /> <PackageReference Include="Microsoft.AspNetCore.DataProtection" Version="8.0.7" />
<PackageReference Include="Microsoft.Extensions.Http" Version="10.0.9" /> <PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="10.0.9" /> <PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="8.0.0" />
<PackageReference Include="Pgvector.EntityFrameworkCore" Version="0.3.0" /> <PackageReference Include="Pgvector.EntityFrameworkCore" Version="0.2.0" />
<PackageReference Include="Polly" Version="8.4.1" /> <PackageReference Include="Polly" Version="8.4.1" />
<PackageReference Include="QuestPDF" Version="2024.7.0" /> <PackageReference Include="QuestPDF" Version="2024.7.0" />
<PackageReference Include="CsvHelper" Version="33.0.1" /> <PackageReference Include="CsvHelper" Version="33.0.1" />
@@ -24,15 +24,12 @@
<!-- Transitive security pins: patch known .NET 8.0.0 advisories pulled in by <!-- Transitive security pins: patch known .NET 8.0.0 advisories pulled in by
EF Core / ASP.NET / DataProtection. Remove once the parent packages ship EF Core / ASP.NET / DataProtection. Remove once the parent packages ship
these versions transitively. --> these versions transitively. -->
<PackageReference Include="System.Text.Json" Version="10.0.9" /> <PackageReference Include="System.Text.Json" Version="8.0.6" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="10.0.9" /> <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" />
<PackageReference Include="System.Security.Cryptography.Xml" Version="10.0.9" /> <PackageReference Include="System.Security.Cryptography.Xml" Version="8.0.3" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\InboxIntel.Application\InboxIntel.Application.csproj" /> <ProjectReference Include="..\InboxIntel.Application\InboxIntel.Application.csproj" />
<ProjectReference Include="..\InboxIntel.Domain\InboxIntel.Domain.csproj" /> <ProjectReference Include="..\InboxIntel.Domain\InboxIntel.Domain.csproj" />
</ItemGroup> </ItemGroup>
<ItemGroup>
<InternalsVisibleTo Include="InboxIntel.IntegrationTests" />
</ItemGroup>
</Project> </Project>
@@ -25,7 +25,7 @@ namespace InboxIntel.IntegrationTests;
/// endpoints (model validation, per-user rate limits) without a real Google login.</summary> /// endpoints (model validation, per-user rate limits) without a real Google login.</summary>
public class TestAuthHandler : AuthenticationHandler<AuthenticationSchemeOptions> public class TestAuthHandler : AuthenticationHandler<AuthenticationSchemeOptions>
{ {
public new const string Scheme = "Test"; public const string Scheme = "Test";
// Stable across requests so per-user rate-limit partitions accumulate correctly. // Stable across requests so per-user rate-limit partitions accumulate correctly.
public static readonly string Uid = Guid.NewGuid().ToString(); public static readonly string Uid = Guid.NewGuid().ToString();
@@ -52,9 +52,6 @@ public class AuditTestAppFactory : WebApplicationFactory<Program>
builder.ConfigureHostConfiguration(cfg => cfg.AddInMemoryCollection(new Dictionary<string, string?> builder.ConfigureHostConfiguration(cfg => cfg.AddInMemoryCollection(new Dictionary<string, string?>
{ {
["Database:AutoMigrate"] = "false", ["Database:AutoMigrate"] = "false",
// Npgsql 10 eagerly validates the connection string when the DbContext is
// resolved (8.x was lazy); these tests never connect, but the string must parse.
["ConnectionStrings:Postgres"] = "Host=localhost;Database=test;Username=test;Password=test",
["GoogleOAuth:ClientId"] = "test-client-id", ["GoogleOAuth:ClientId"] = "test-client-id",
["GoogleOAuth:ClientSecret"] = "test-client-secret", ["GoogleOAuth:ClientSecret"] = "test-client-secret",
// H-2: make the auth policy trip on the 3rd request within the window. // H-2: make the auth policy trip on the 3rd request within the window.
@@ -19,9 +19,6 @@ public class TestAppFactory : WebApplicationFactory<Program>
builder.ConfigureHostConfiguration(cfg => cfg.AddInMemoryCollection(new Dictionary<string, string?> builder.ConfigureHostConfiguration(cfg => cfg.AddInMemoryCollection(new Dictionary<string, string?>
{ {
["Database:AutoMigrate"] = "false", ["Database:AutoMigrate"] = "false",
// Npgsql 10 eagerly validates the connection string when the DbContext is
// resolved (8.x was lazy); these tests never connect, but the string must parse.
["ConnectionStrings:Postgres"] = "Host=localhost;Database=test;Username=test;Password=test",
// Dummy OAuth creds so the Google challenge produces a real 302 redirect // Dummy OAuth creds so the Google challenge produces a real 302 redirect
// (an empty ClientId can make the handler throw instead of redirecting). // (an empty ClientId can make the handler throw instead of redirecting).
["GoogleOAuth:ClientId"] = "test-client-id", ["GoogleOAuth:ClientId"] = "test-client-id",
@@ -1,95 +0,0 @@
using FluentAssertions;
using InboxIntel.Application.Abstractions;
using InboxIntel.Domain.Entities;
using InboxIntel.Infrastructure.Ai;
using InboxIntel.Infrastructure.Persistence;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Pgvector.EntityFrameworkCore;
using Xunit;
namespace InboxIntel.IntegrationTests;
/// <summary>
/// Semantic-search backfill: the worker embeds emails lacking an Embedding and persists the
/// vectors. Runs against live Postgres (pgvector) since the Embedding column is ignored under
/// the InMemory provider; the CI db-tests job provides the database. Uses a deterministic fake
/// embedding provider — real-Ollama integration is verified separately (endpoint contract:
/// /api/embeddings, 768 dims).
/// </summary>
[Trait("Category", "LiveDb")]
[Collection("LiveDb")] // serialise LiveDb classes: concurrent MigrateAsync on a fresh DB races
public class EmbeddingBackfillTests
{
private static string? Conn => Environment.GetEnvironmentVariable("LIVEDB_CONNECTION");
private sealed class FakeCurrentUser : ICurrentUser
{
public Guid UserId => Guid.Empty; // worker scope sees all rows
public bool IsAuthenticated => false;
}
private sealed class FakeEmbeddings : IEmbeddingProvider
{
public bool IsAvailable => true;
public Task<float[]> EmbedAsync(string text, CancellationToken ct = default)
=> Task.FromResult(Vec(text));
public Task<IReadOnlyList<float[]>> EmbedBatchAsync(IReadOnlyList<string> texts, CancellationToken ct = default)
=> Task.FromResult<IReadOnlyList<float[]>>(texts.Select(Vec).ToList());
private static float[] Vec(string text)
{
var v = new float[768];
v[0] = text.Length; // deterministic, content-dependent
return v;
}
}
[Fact]
public async Task Worker_embeds_pending_emails_and_persists_vectors()
{
if (Conn is null) return; // soft-skip outside the live-db CI job
var uid = Guid.NewGuid();
var opts = new DbContextOptionsBuilder<AppDbContext>().UseNpgsql(Conn!, o => o.UseVector()).Options;
var services = new ServiceCollection();
services.AddScoped<ICurrentUser, FakeCurrentUser>();
services.AddScoped(_ => new AppDbContext(opts, new FakeCurrentUser()));
services.AddScoped<IEmbeddingProvider, FakeEmbeddings>();
using var sp = services.BuildServiceProvider();
using (var seed = new AppDbContext(opts, new FakeCurrentUser()))
{
await seed.Database.MigrateAsync();
seed.Users.Add(new User { Id = uid, GoogleSubjectId = "g" + uid, Email = uid + "@t.t" });
var dom = new MailDomain { UserId = uid, Name = "t.t" };
var snd = new Sender { UserId = uid, Address = "a@t.t", Domain = dom };
var thr = new MailThread { UserId = uid, GmailThreadId = "th" + uid };
seed.AddRange(dom, snd, thr);
seed.Emails.Add(new Email { UserId = uid, GmailMessageId = "e1" + uid, Subject = "hello world", Sender = snd, Thread = thr, SentAtUtc = DateTimeOffset.UtcNow });
seed.Emails.Add(new Email { UserId = uid, GmailMessageId = "e2" + uid, Subject = "quarterly invoice", Sender = snd, Thread = thr, SentAtUtc = DateTimeOffset.UtcNow });
await seed.SaveChangesAsync();
}
try
{
var worker = new EmbeddingBackfillWorker(
sp.GetRequiredService<IServiceScopeFactory>(), NullLogger<EmbeddingBackfillWorker>.Instance);
var processed = await worker.ProcessBatchAsync(CancellationToken.None);
processed.Should().BeGreaterThanOrEqualTo(2);
using var check = new AppDbContext(opts, new FakeCurrentUser());
var mine = await check.Emails.Where(e => e.UserId == uid).ToListAsync();
mine.Should().OnlyContain(e => e.Embedding != null);
mine.First().Embedding!.ToArray().Length.Should().Be(768);
}
finally
{
using var c = new AppDbContext(opts, new FakeCurrentUser());
await c.Emails.Where(e => e.UserId == uid).ExecuteDeleteAsync();
await c.Threads.Where(t => t.UserId == uid).ExecuteDeleteAsync();
await c.Senders.Where(s => s.UserId == uid).ExecuteDeleteAsync();
await c.Domains.Where(d => d.UserId == uid).ExecuteDeleteAsync();
await c.Users.Where(u => u.Id == uid).ExecuteDeleteAsync();
}
}
}
@@ -7,8 +7,8 @@
<PackageReference Include="xunit" Version="2.9.0" /> <PackageReference Include="xunit" Version="2.9.0" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" /> <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" />
<PackageReference Include="FluentAssertions" Version="6.12.0" /> <PackageReference Include="FluentAssertions" Version="6.12.0" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.9" /> <PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="8.0.7" />
<PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" Version="10.0.9" /> <PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" Version="8.0.4" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\..\src\InboxIntel.Api\InboxIntel.Api.csproj" /> <ProjectReference Include="..\..\src\InboxIntel.Api\InboxIntel.Api.csproj" />
@@ -21,7 +21,6 @@ namespace InboxIntel.IntegrationTests;
/// InMemory test run is unaffected. /// InMemory test run is unaffected.
/// </summary> /// </summary>
[Trait("Category", "LiveDb")] [Trait("Category", "LiveDb")]
[Collection("LiveDb")] // serialise LiveDb classes: concurrent MigrateAsync on a fresh DB races
public class LiveDbSearchTests public class LiveDbSearchTests
{ {
private static string? Conn => Environment.GetEnvironmentVariable("LIVEDB_CONNECTION"); private static string? Conn => Environment.GetEnvironmentVariable("LIVEDB_CONNECTION");