Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 3b93893618 |
@@ -15,6 +15,3 @@ FRONTEND_ORIGIN=http://localhost:8081
|
||||
# Set DEV_MODE=true and MAX_MESSAGES=1000 to test against a large mailbox.
|
||||
DEV_MODE=false
|
||||
MAX_MESSAGES=0
|
||||
|
||||
# Nightly DB backup rotation (days of dumps to keep in ./backups)
|
||||
BACKUP_KEEP_DAYS=7
|
||||
|
||||
@@ -14,7 +14,7 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-dotnet@v4
|
||||
with:
|
||||
dotnet-version: '10.0.x'
|
||||
dotnet-version: '8.0.x'
|
||||
- name: Restore
|
||||
run: dotnet restore InboxIntel.sln
|
||||
- name: Build
|
||||
@@ -45,7 +45,7 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-dotnet@v4
|
||||
with:
|
||||
dotnet-version: '10.0.x'
|
||||
dotnet-version: '8.0.x'
|
||||
- name: dotnet format (verify only)
|
||||
run: dotnet format InboxIntel.sln --verify-no-changes
|
||||
|
||||
@@ -67,7 +67,7 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-dotnet@v4
|
||||
with:
|
||||
dotnet-version: '10.0.x'
|
||||
dotnet-version: '8.0.x'
|
||||
- name: Wait for Postgres
|
||||
run: |
|
||||
for i in $(seq 1 30); do
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
name: Renovate
|
||||
|
||||
# RECOMMENDATIONS #2: automated dependency-update PRs (NuGet, npm, Dockerfiles, Actions)
|
||||
# that ride the existing required CI gates. Runs weekly + on demand.
|
||||
#
|
||||
# ONE-TIME SETUP (manual): create a Gitea personal access token with scopes
|
||||
# repo (rw) + user (r) + issue (rw) + organization (r), and add it as the Actions
|
||||
# secret RENOVATE_TOKEN (repo Settings -> Actions -> Secrets). Without the secret this
|
||||
# workflow fails fast with a clear message. See https://docs.renovatebot.com/modules/platform/gitea/
|
||||
on:
|
||||
schedule:
|
||||
- cron: '30 4 * * 1' # Mondays 04:30 UTC
|
||||
workflow_dispatch: {}
|
||||
|
||||
jobs:
|
||||
renovate:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Require RENOVATE_TOKEN
|
||||
run: |
|
||||
if [ -z "${{ secrets.RENOVATE_TOKEN }}" ]; then
|
||||
echo "RENOVATE_TOKEN secret is not set — see the comment at the top of this workflow." >&2
|
||||
exit 1
|
||||
fi
|
||||
- name: Run Renovate
|
||||
uses: https://github.com/renovatebot/github-action@v40.3.6
|
||||
with:
|
||||
token: ${{ secrets.RENOVATE_TOKEN }}
|
||||
env:
|
||||
RENOVATE_PLATFORM: gitea
|
||||
RENOVATE_ENDPOINT: https://git.cesnimda.uk/api/v1
|
||||
RENOVATE_REPOSITORIES: cesnimda/Inboxintel
|
||||
RENOVATE_ONBOARDING: "false"
|
||||
RENOVATE_REQUIRE_CONFIG: optional
|
||||
LOG_LEVEL: info
|
||||
@@ -31,7 +31,7 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-dotnet@v4
|
||||
with:
|
||||
dotnet-version: '10.0.x'
|
||||
dotnet-version: '8.0.x'
|
||||
- name: Restore
|
||||
run: dotnet restore InboxIntel.sln
|
||||
- name: .NET vulnerable packages (fail on any)
|
||||
|
||||
@@ -21,9 +21,6 @@ frontend/.vite/
|
||||
appsettings.*.local.json
|
||||
secrets.json
|
||||
|
||||
## DB backups (never commit dumps)
|
||||
backups/
|
||||
|
||||
## Logs
|
||||
logs/
|
||||
*.log
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
<Project>
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<LangVersion>latest</LangVersion>
|
||||
|
||||
+1
-3
@@ -29,9 +29,7 @@ reverse proxy.
|
||||
mitigate (a third party reading the DB files) reduces to "someone with access to your
|
||||
machine" — mitigate it at the layer that actually works:
|
||||
- **Use full-disk or volume encryption** on the host (BitLocker/LUKS) — strongly recommended.
|
||||
- **Encrypt backups**: nightly `pg_dump` rotation runs via the compose `backup` service
|
||||
into `./backups/` (git-ignored) — keep that directory on an encrypted disk and copy it
|
||||
off-machine. Restore: `docker compose exec -T postgres psql -U inboxintel -d inboxintel < backups/<file>.sql`.
|
||||
- **Encrypt backups** of the `pgdata` volume the same way.
|
||||
- Before any **multi-user** deployment, revisit per the multi-provider security design
|
||||
(host admins must not be able to read members' mail — plaintext bodies break that promise).
|
||||
2. **DB connection is not TLS** — Postgres is only reachable on the compose-internal network /
|
||||
|
||||
@@ -22,37 +22,6 @@ services:
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
|
||||
# Nightly logical backups (RECOMMENDATIONS #3 — previously there were NONE). Dumps
|
||||
# rotate after BACKUP_KEEP_DAYS. The ./backups host directory should live on an
|
||||
# encrypted disk and be included in your off-machine backup regime (see SECURITY.md).
|
||||
# Restore: docker compose exec -T postgres psql -U inboxintel -d inboxintel < backups/<file>.sql
|
||||
backup:
|
||||
image: pgvector/pgvector:pg16
|
||||
entrypoint: /bin/sh
|
||||
command:
|
||||
- -c
|
||||
- |
|
||||
while true; do
|
||||
ts=$$(date -u +%Y%m%d-%H%M%S)
|
||||
if pg_dump -h postgres -U inboxintel -d inboxintel > /backups/inboxintel-$$ts.sql.tmp; then
|
||||
mv /backups/inboxintel-$$ts.sql.tmp /backups/inboxintel-$$ts.sql
|
||||
echo "backup OK: inboxintel-$$ts.sql"
|
||||
else
|
||||
rm -f /backups/inboxintel-$$ts.sql.tmp
|
||||
echo "backup FAILED at $$ts" >&2
|
||||
fi
|
||||
find /backups -name 'inboxintel-*.sql' -mtime +$${BACKUP_KEEP_DAYS:-7} -delete
|
||||
sleep 86400
|
||||
done
|
||||
environment:
|
||||
PGPASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in deploy/.env}
|
||||
BACKUP_KEEP_DAYS: ${BACKUP_KEEP_DAYS:-7}
|
||||
volumes:
|
||||
- ./backups:/backups
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
|
||||
api:
|
||||
build:
|
||||
context: .
|
||||
|
||||
+8
-19
@@ -1,35 +1,25 @@
|
||||
import React, { Suspense, lazy } from 'react';
|
||||
import React from 'react';
|
||||
import ReactDOM from 'react-dom/client';
|
||||
import { BrowserRouter, Routes, Route, Navigate } from 'react-router-dom';
|
||||
import Landing from './pages/Landing.jsx';
|
||||
import Dashboard from './pages/Dashboard.jsx';
|
||||
import Senders from './pages/Senders.jsx';
|
||||
import Cleanup from './pages/Cleanup.jsx';
|
||||
import Unsubscribe from './pages/Unsubscribe.jsx';
|
||||
import FolderView from './pages/FolderView.jsx';
|
||||
import SearchResults from './pages/SearchResults.jsx';
|
||||
import Layout from './components/Layout.jsx';
|
||||
import DesignSystem from './pages/DesignSystem.jsx';
|
||||
import { ToastProvider, TooltipProvider } from './components/ui';
|
||||
import '@fontsource-variable/inter';
|
||||
import './index.css';
|
||||
import './styles.css';
|
||||
|
||||
// Route-level code splitting: each page loads its own chunk on first visit, so the
|
||||
// initial bundle no longer carries Chart.js / grid-layout / every page at once.
|
||||
// Landing + Layout stay eager (they're the first paint).
|
||||
const Dashboard = lazy(() => import('./pages/Dashboard.jsx'));
|
||||
const Senders = lazy(() => import('./pages/Senders.jsx'));
|
||||
const Cleanup = lazy(() => import('./pages/Cleanup.jsx'));
|
||||
const Unsubscribe = lazy(() => import('./pages/Unsubscribe.jsx'));
|
||||
const FolderView = lazy(() => import('./pages/FolderView.jsx'));
|
||||
const SearchResults = lazy(() => import('./pages/SearchResults.jsx'));
|
||||
const DesignSystem = lazy(() => import('./pages/DesignSystem.jsx'));
|
||||
|
||||
// Minimal, theme-correct route fallback (skeleton-style, per the design system).
|
||||
const RouteFallback = () => (
|
||||
<div className="p-6 text-sm text-muted-foreground" aria-busy="true">Loading…</div>
|
||||
);
|
||||
|
||||
ReactDOM.createRoot(document.getElementById('root')).render(
|
||||
<React.StrictMode>
|
||||
<ToastProvider>
|
||||
<TooltipProvider delayDuration={200}>
|
||||
<BrowserRouter>
|
||||
<Suspense fallback={<RouteFallback />}>
|
||||
<Routes>
|
||||
{/* Public landing page */}
|
||||
<Route path="/" element={<Landing />} />
|
||||
@@ -47,7 +37,6 @@ ReactDOM.createRoot(document.getElementById('root')).render(
|
||||
|
||||
<Route path="*" element={<Navigate to="/" replace />} />
|
||||
</Routes>
|
||||
</Suspense>
|
||||
</BrowserRouter>
|
||||
</TooltipProvider>
|
||||
</ToastProvider>
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
{
|
||||
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
|
||||
"extends": ["config:recommended"],
|
||||
"timezone": "Europe/Berlin",
|
||||
"schedule": ["before 6am on monday"],
|
||||
"labels": ["dependencies"],
|
||||
"prConcurrentLimit": 5,
|
||||
"commitMessagePrefix": "chore(deps):",
|
||||
"packageRules": [
|
||||
{
|
||||
"description": "Group safe minor+patch updates into one weekly PR per ecosystem",
|
||||
"matchUpdateTypes": ["minor", "patch"],
|
||||
"groupName": "{{manager}} minor & patch"
|
||||
},
|
||||
{
|
||||
"description": "Major updates stay individual PRs for careful review",
|
||||
"matchUpdateTypes": ["major"],
|
||||
"dependencyDashboardApproval": true
|
||||
}
|
||||
],
|
||||
"vulnerabilityAlerts": {
|
||||
"enabled": true,
|
||||
"labels": ["security"],
|
||||
"schedule": ["at any time"]
|
||||
},
|
||||
"ignorePaths": ["**/node_modules/**", "**/bin/**", "**/obj/**"]
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
# Multi-stage build for the ASP.NET Core API.
|
||||
FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build
|
||||
FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
|
||||
WORKDIR /src
|
||||
|
||||
# Copy solution + project files first for layer-cached restore.
|
||||
@@ -13,7 +13,7 @@ RUN dotnet restore src/InboxIntel.Api/InboxIntel.Api.csproj
|
||||
COPY src/ src/
|
||||
RUN dotnet publish src/InboxIntel.Api/InboxIntel.Api.csproj -c Release -o /app/publish /p:UseAppHost=false
|
||||
|
||||
FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS runtime
|
||||
FROM mcr.microsoft.com/dotnet/aspnet:8.0 AS runtime
|
||||
WORKDIR /app
|
||||
COPY --from=build /app/publish .
|
||||
|
||||
|
||||
@@ -5,10 +5,10 @@
|
||||
<UserSecretsId>210c6d96-c7e4-4ee9-8982-8b91424979b8</UserSecretsId>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.AspNetCore.Authentication.Google" Version="10.0.9" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="10.0.9" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.Authentication.Google" Version="8.0.7" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="8.0.7" />
|
||||
<!-- Required on the startup project for `dotnet ef migrations` to work. -->
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="10.0.9">
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.4">
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
</PackageReference>
|
||||
|
||||
@@ -38,8 +38,8 @@ var dp = builder.Services.AddDataProtection()
|
||||
var dpCertPath = builder.Configuration["DataProtection:CertificatePath"];
|
||||
if (!string.IsNullOrWhiteSpace(dpCertPath))
|
||||
{
|
||||
dp.ProtectKeysWithCertificate(System.Security.Cryptography.X509Certificates.X509CertificateLoader
|
||||
.LoadPkcs12FromFile(dpCertPath, builder.Configuration["DataProtection:CertificatePassword"]));
|
||||
dp.ProtectKeysWithCertificate(new System.Security.Cryptography.X509Certificates.X509Certificate2(
|
||||
dpCertPath, builder.Configuration["DataProtection:CertificatePassword"]));
|
||||
}
|
||||
|
||||
builder.Services.AddApplication();
|
||||
@@ -201,14 +201,15 @@ var forwardedOptions = new ForwardedHeadersOptions
|
||||
ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto | ForwardedHeaders.XForwardedHost,
|
||||
ForwardLimit = app.Configuration.GetValue<int?>("ForwardedHeaders:ForwardLimit") ?? 1
|
||||
};
|
||||
forwardedOptions.KnownIPNetworks.Clear();
|
||||
forwardedOptions.KnownNetworks.Clear();
|
||||
forwardedOptions.KnownProxies.Clear();
|
||||
var trustedNetworks = app.Configuration.GetSection("ForwardedHeaders:KnownNetworks").Get<string[]>()
|
||||
?? new[] { "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "127.0.0.0/8", "::1/128" };
|
||||
foreach (var cidr in trustedNetworks)
|
||||
{
|
||||
if (System.Net.IPNetwork.TryParse(cidr, out var network))
|
||||
forwardedOptions.KnownIPNetworks.Add(network);
|
||||
var parts = cidr.Split('/');
|
||||
if (parts.Length == 2 && System.Net.IPAddress.TryParse(parts[0], out var prefix) && int.TryParse(parts[1], out var len))
|
||||
forwardedOptions.KnownNetworks.Add(new Microsoft.AspNetCore.HttpOverrides.IPNetwork(prefix, len));
|
||||
}
|
||||
app.UseForwardedHeaders(forwardedOptions);
|
||||
|
||||
|
||||
@@ -6,13 +6,13 @@
|
||||
<ItemGroup>
|
||||
<PackageReference Include="FluentValidation" Version="11.9.2" />
|
||||
<PackageReference Include="FluentValidation.DependencyInjectionExtensions" Version="11.9.2" />
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.9" />
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.2" />
|
||||
<!-- DbSet<> is exposed on IAppDbContext so the Application layer can query.
|
||||
Pinned to 8.0.4 to match the Npgsql provider's Relational dependency. -->
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="10.0.9" />
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.4" />
|
||||
<!-- Transitive security pins: patch known .NET 8.0.0 advisories pulled in by EF Core. -->
|
||||
<PackageReference Include="System.Text.Json" Version="10.0.9" />
|
||||
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="10.0.9" />
|
||||
<PackageReference Include="System.Text.Json" Version="8.0.6" />
|
||||
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\InboxIntel.Domain\InboxIntel.Domain.csproj" />
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
<ItemGroup>
|
||||
<!-- NpgsqlTypes.NpgsqlTsVector (FTS) and Pgvector.Vector (semantic search) are used as
|
||||
column types on the Email entity — same pragmatic precedent for both. -->
|
||||
<PackageReference Include="Npgsql" Version="10.0.2" />
|
||||
<PackageReference Include="Pgvector" Version="0.3.0" />
|
||||
<PackageReference Include="Npgsql" Version="8.0.3" />
|
||||
<PackageReference Include="Pgvector" Version="0.2.0" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
@@ -4,18 +4,18 @@
|
||||
<AssemblyName>InboxIntel.Infrastructure</AssemblyName>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="10.0.9" />
|
||||
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="10.0.2" />
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="10.0.9">
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.4" />
|
||||
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.4" />
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.4">
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
</PackageReference>
|
||||
<PackageReference Include="Google.Apis.Gmail.v1" Version="1.68.0.3427" />
|
||||
<PackageReference Include="Google.Apis.Auth" Version="1.68.0" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.DataProtection" Version="10.0.9" />
|
||||
<PackageReference Include="Microsoft.Extensions.Http" Version="10.0.9" />
|
||||
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="10.0.9" />
|
||||
<PackageReference Include="Pgvector.EntityFrameworkCore" Version="0.3.0" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.DataProtection" Version="8.0.7" />
|
||||
<PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="8.0.0" />
|
||||
<PackageReference Include="Pgvector.EntityFrameworkCore" Version="0.2.0" />
|
||||
<PackageReference Include="Polly" Version="8.4.1" />
|
||||
<PackageReference Include="QuestPDF" Version="2024.7.0" />
|
||||
<PackageReference Include="CsvHelper" Version="33.0.1" />
|
||||
@@ -24,9 +24,9 @@
|
||||
<!-- Transitive security pins: patch known .NET 8.0.0 advisories pulled in by
|
||||
EF Core / ASP.NET / DataProtection. Remove once the parent packages ship
|
||||
these versions transitively. -->
|
||||
<PackageReference Include="System.Text.Json" Version="10.0.9" />
|
||||
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="10.0.9" />
|
||||
<PackageReference Include="System.Security.Cryptography.Xml" Version="10.0.9" />
|
||||
<PackageReference Include="System.Text.Json" Version="8.0.6" />
|
||||
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" />
|
||||
<PackageReference Include="System.Security.Cryptography.Xml" Version="8.0.3" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\InboxIntel.Application\InboxIntel.Application.csproj" />
|
||||
|
||||
@@ -5,7 +5,6 @@ using InboxIntel.Domain.Entities;
|
||||
using InboxIntel.Domain.Enums;
|
||||
using InboxIntel.Infrastructure.Persistence;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Pgvector.EntityFrameworkCore;
|
||||
|
||||
namespace InboxIntel.Infrastructure.Search;
|
||||
|
||||
@@ -19,13 +18,7 @@ namespace InboxIntel.Infrastructure.Search;
|
||||
public class SearchService : ISearchService
|
||||
{
|
||||
private readonly AppDbContext _db;
|
||||
private readonly IEmbeddingProvider? _embeddings;
|
||||
|
||||
public SearchService(AppDbContext db, IEmbeddingProvider? embeddings = null)
|
||||
{
|
||||
_db = db;
|
||||
_embeddings = embeddings;
|
||||
}
|
||||
public SearchService(AppDbContext db) => _db = db;
|
||||
|
||||
public async Task<PagedResult<EmailSummaryDto>> SearchAsync(Guid userId, SearchRequestDto r, CancellationToken ct = default)
|
||||
{
|
||||
@@ -80,18 +73,6 @@ public class SearchService : ISearchService
|
||||
|
||||
var total = await matched.CountAsync(ct);
|
||||
|
||||
// Hybrid semantic fusion (docs/discovery/05): when embeddings are available, fuse
|
||||
// lexical top-K with vector top-K via Reciprocal Rank Fusion. Runs BEFORE the fuzzy
|
||||
// fallback so a query with ZERO lexical hits (pure semantic recall — "gym receipt"
|
||||
// phrased differently) still surfaces results. Exact lexical hits keep winning (they
|
||||
// rank in both lists). Deeper pages fall through to lexical paging; any failure
|
||||
// (Ollama down, nothing embedded yet) silently degrades to the lexical/fuzzy path.
|
||||
if (hasFreeTextQuery && _embeddings is { IsAvailable: true })
|
||||
{
|
||||
var hybrid = await TryHybridAsync(structured, matched, term, total, r, ct);
|
||||
if (hybrid is not null) return hybrid;
|
||||
}
|
||||
|
||||
// Fuzzy/typo fallback: ONLY when a free-text search found nothing exact. word_similarity
|
||||
// with an explicit 0.3 threshold — pg_trgm's default 0.6 misses real typos
|
||||
// ("recieved" -> "received" scores ~0.39). Rare path, so the (non-indexed) scan over the
|
||||
@@ -156,73 +137,4 @@ public class SearchService : ISearchService
|
||||
TotalCount = total
|
||||
};
|
||||
}
|
||||
|
||||
private const int HybridK = 50; // candidates taken from each layer
|
||||
private const int RrfConstant = 60; // standard RRF dampening constant
|
||||
|
||||
/// <summary>
|
||||
/// RRF fusion of lexical and vector candidates. Returns null when the requested page
|
||||
/// lies beyond the fused window or anything fails — caller falls back to lexical.
|
||||
/// </summary>
|
||||
private async Task<PagedResult<EmailSummaryDto>?> TryHybridAsync(
|
||||
IQueryable<Email> structured, IQueryable<Email> lexical, string term, int lexicalTotal,
|
||||
SearchRequestDto r, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
var lexIds = await lexical
|
||||
.OrderByDescending(e => e.SearchVector!.RankCoverDensity(EF.Functions.WebSearchToTsQuery("english", term)))
|
||||
.ThenByDescending(e => e.SentAtUtc)
|
||||
.Take(HybridK).Select(e => e.Id).ToListAsync(ct);
|
||||
|
||||
var queryVec = await _embeddings!.EmbedAsync(term, ct);
|
||||
List<Guid> vecIds = new();
|
||||
if (queryVec.Length > 0)
|
||||
{
|
||||
var qv = new Pgvector.Vector(queryVec);
|
||||
vecIds = await structured
|
||||
.Where(e => e.Embedding != null)
|
||||
.OrderBy(e => e.Embedding!.CosineDistance(qv))
|
||||
.Take(HybridK).Select(e => e.Id).ToListAsync(ct);
|
||||
}
|
||||
if (vecIds.Count == 0) return null; // nothing embedded yet → lexical path
|
||||
|
||||
var scores = new Dictionary<Guid, double>();
|
||||
for (var i = 0; i < lexIds.Count; i++)
|
||||
scores[lexIds[i]] = scores.GetValueOrDefault(lexIds[i]) + 1.0 / (RrfConstant + i + 1);
|
||||
for (var i = 0; i < vecIds.Count; i++)
|
||||
scores[vecIds[i]] = scores.GetValueOrDefault(vecIds[i]) + 1.0 / (RrfConstant + i + 1);
|
||||
|
||||
var fused = scores.OrderByDescending(kv => kv.Value).Select(kv => kv.Key).ToList();
|
||||
var pageIds = fused.Skip((r.Page - 1) * r.PageSize).Take(r.PageSize).ToList();
|
||||
if (pageIds.Count == 0 && r.Page > 1) return null; // deep page → lexical paging
|
||||
|
||||
var headlineOpts =
|
||||
$"StartSel={(char)0xE000},StopSel={(char)0xE001},MaxWords=16,MinWords=5,ShortWord=2,HighlightAll=false";
|
||||
var rows = await _db.Emails.AsNoTracking()
|
||||
.Where(e => pageIds.Contains(e.Id))
|
||||
.Select(e => new EmailSummaryDto(
|
||||
e.Id, e.GmailMessageId, e.Subject, e.Snippet,
|
||||
e.Sender!.Address, e.Sender.DisplayName, e.SentAtUtc,
|
||||
e.IsUnread, e.IsStarred, e.HasAttachments, e.SizeEstimateBytes, e.Category,
|
||||
e.HasListUnsubscribe, e.SupportsOneClickUnsubscribe,
|
||||
EF.Functions.WebSearchToTsQuery("english", term).GetResultHeadline("english", e.BodyText ?? "", headlineOpts)))
|
||||
.ToListAsync(ct);
|
||||
var byId = rows.ToDictionary(x => x.Id);
|
||||
var items = pageIds.Where(byId.ContainsKey).Select(id => byId[id]).ToList();
|
||||
|
||||
return new PagedResult<EmailSummaryDto>
|
||||
{
|
||||
Items = items,
|
||||
Page = r.Page,
|
||||
PageSize = r.PageSize,
|
||||
// Semantic recall can exceed the lexical match count.
|
||||
TotalCount = Math.Max(lexicalTotal, fused.Count)
|
||||
};
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null; // AI must never break search — degrade to lexical
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,7 +25,7 @@ namespace InboxIntel.IntegrationTests;
|
||||
/// endpoints (model validation, per-user rate limits) without a real Google login.</summary>
|
||||
public class TestAuthHandler : AuthenticationHandler<AuthenticationSchemeOptions>
|
||||
{
|
||||
public new const string Scheme = "Test";
|
||||
public const string Scheme = "Test";
|
||||
// Stable across requests so per-user rate-limit partitions accumulate correctly.
|
||||
public static readonly string Uid = Guid.NewGuid().ToString();
|
||||
|
||||
@@ -52,9 +52,6 @@ public class AuditTestAppFactory : WebApplicationFactory<Program>
|
||||
builder.ConfigureHostConfiguration(cfg => cfg.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["Database:AutoMigrate"] = "false",
|
||||
// Npgsql 10 eagerly validates the connection string when the DbContext is
|
||||
// resolved (8.x was lazy); these tests never connect, but the string must parse.
|
||||
["ConnectionStrings:Postgres"] = "Host=localhost;Database=test;Username=test;Password=test",
|
||||
["GoogleOAuth:ClientId"] = "test-client-id",
|
||||
["GoogleOAuth:ClientSecret"] = "test-client-secret",
|
||||
// H-2: make the auth policy trip on the 3rd request within the window.
|
||||
|
||||
@@ -19,9 +19,6 @@ public class TestAppFactory : WebApplicationFactory<Program>
|
||||
builder.ConfigureHostConfiguration(cfg => cfg.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["Database:AutoMigrate"] = "false",
|
||||
// Npgsql 10 eagerly validates the connection string when the DbContext is
|
||||
// resolved (8.x was lazy); these tests never connect, but the string must parse.
|
||||
["ConnectionStrings:Postgres"] = "Host=localhost;Database=test;Username=test;Password=test",
|
||||
// Dummy OAuth creds so the Google challenge produces a real 302 redirect
|
||||
// (an empty ClientId can make the handler throw instead of redirecting).
|
||||
["GoogleOAuth:ClientId"] = "test-client-id",
|
||||
|
||||
@@ -7,8 +7,8 @@
|
||||
<PackageReference Include="xunit" Version="2.9.0" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" />
|
||||
<PackageReference Include="FluentAssertions" Version="6.12.0" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.9" />
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" Version="10.0.9" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="8.0.7" />
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" Version="8.0.4" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\src\InboxIntel.Api\InboxIntel.Api.csproj" />
|
||||
|
||||
@@ -123,54 +123,4 @@ public class LiveDbSearchTests
|
||||
}
|
||||
finally { await CleanupAsync(opts, uid); }
|
||||
}
|
||||
|
||||
private sealed class DirectionalFakeEmbeddings : IEmbeddingProvider
|
||||
{
|
||||
public bool IsAvailable => true;
|
||||
public Task<float[]> EmbedAsync(string text, CancellationToken ct = default)
|
||||
{
|
||||
// Deterministic "semantics": anything fruit-flavoured points one way, else the other.
|
||||
var v = new float[768];
|
||||
if (text.Contains("banana") || text.Contains("tropical")) v[0] = 1; else v[1] = 1;
|
||||
return Task.FromResult(v);
|
||||
}
|
||||
public async Task<IReadOnlyList<float[]>> EmbedBatchAsync(IReadOnlyList<string> texts, CancellationToken ct = default)
|
||||
{
|
||||
var list = new List<float[]>();
|
||||
foreach (var t in texts) list.Add(await EmbedAsync(t, ct));
|
||||
return list;
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Hybrid_search_surfaces_semantic_match_with_zero_keyword_overlap()
|
||||
{
|
||||
if (Conn is null) return;
|
||||
var opts = Options();
|
||||
var uid = await SeedAsync(opts);
|
||||
try
|
||||
{
|
||||
var embeddings = new DirectionalFakeEmbeddings();
|
||||
using (var prep = new AppDbContext(opts, new FakeCurrentUser()))
|
||||
{
|
||||
// "Weekly notes" gets a fruit-direction embedding (semantically related to the
|
||||
// query); "Invoice March" points elsewhere. Neither subject contains "banana".
|
||||
var near = await prep.Emails.FirstAsync(e => e.UserId == uid && e.Subject == "Weekly notes");
|
||||
near.Embedding = new Vector(await embeddings.EmbedAsync("tropical"));
|
||||
var far = await prep.Emails.FirstAsync(e => e.UserId == uid && e.Subject == "Invoice March");
|
||||
far.Embedding = new Vector(await embeddings.EmbedAsync("finance"));
|
||||
await prep.SaveChangesAsync();
|
||||
}
|
||||
|
||||
using var ctx = new AppDbContext(opts, new FakeCurrentUser { UserId = uid });
|
||||
var res = await new SearchService(ctx, embeddings)
|
||||
.SearchAsync(uid, GmailQueryParser.Parse("banana", 1, 10));
|
||||
|
||||
// Zero lexical hits for "banana" — hybrid must still surface the semantically
|
||||
// nearest email, ranked first.
|
||||
res.Items.Should().NotBeEmpty("semantic recall should fire with zero keyword overlap");
|
||||
res.Items[0].Subject.Should().Be("Weekly notes");
|
||||
}
|
||||
finally { await CleanupAsync(opts, uid); }
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user