perf(search): keyset pagination (browse path) (#36)
CI / backend (push) Successful in 54s
CI / frontend (push) Successful in 12s
CI / format (push) Failing after 51s
CI / db-tests (push) Successful in 57s
Deploy Staging / deploy (push) Successful in 25s
Security / secrets (push) Successful in 4s
Security / dependencies (push) Successful in 59s
Security / sast (push) Successful in 40s
CI / backend (pull_request) Successful in 55s
CI / frontend (pull_request) Successful in 15s
CI / format (pull_request) Failing after 54s
CI / db-tests (pull_request) Successful in 52s
Security / secrets (pull_request) Successful in 4s
Security / dependencies (pull_request) Successful in 56s
Security / sast (pull_request) Successful in 37s
CI / backend (push) Successful in 54s
CI / frontend (push) Successful in 12s
CI / format (push) Failing after 51s
CI / db-tests (push) Successful in 57s
Deploy Staging / deploy (push) Successful in 25s
Security / secrets (push) Successful in 4s
Security / dependencies (push) Successful in 59s
Security / sast (push) Successful in 40s
CI / backend (pull_request) Successful in 55s
CI / frontend (pull_request) Successful in 15s
CI / format (pull_request) Failing after 54s
CI / db-tests (pull_request) Successful in 52s
Security / secrets (pull_request) Successful in 4s
Security / dependencies (pull_request) Successful in 56s
Security / sast (pull_request) Successful in 37s
This commit was merged in pull request #36.
This commit is contained in:
@@ -0,0 +1,13 @@
|
|||||||
|
# Root editor/formatter config. end_of_line=lf makes dotnet-format agree with
|
||||||
|
# .gitattributes (eol=lf) — without this, format-on-Windows wants CRLF while git
|
||||||
|
# stores LF, and the pre-commit/CI format gates flip-flop forever.
|
||||||
|
root = true
|
||||||
|
|
||||||
|
[*]
|
||||||
|
end_of_line = lf
|
||||||
|
insert_final_newline = true
|
||||||
|
charset = utf-8
|
||||||
|
|
||||||
|
[*.cs]
|
||||||
|
indent_style = space
|
||||||
|
indent_size = 4
|
||||||
@@ -23,4 +23,9 @@ public record SearchRequestDto(
|
|||||||
bool? IsTrashed = null,
|
bool? IsTrashed = null,
|
||||||
string? GmailLabel = null, // e.g. "SENT", "DRAFT", "SPAM"
|
string? GmailLabel = null, // e.g. "SENT", "DRAFT", "SPAM"
|
||||||
string? Category = null, // EmailCategory name, e.g. "Finance"
|
string? Category = null, // EmailCategory name, e.g. "Finance"
|
||||||
long? MinSizeBytes = null);
|
long? MinSizeBytes = null,
|
||||||
|
// Keyset cursor for the date-ordered browse path (RECOMMENDATIONS #8): pass the last
|
||||||
|
// row's SentAtUtc+Id to fetch the next window without OFFSET (O(pageSize), not O(page)).
|
||||||
|
// When set, TotalCount is not recomputed (-1). Additive; offset paging still works.
|
||||||
|
DateTimeOffset? AfterSentAtUtc = null,
|
||||||
|
Guid? AfterId = null);
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
using System;
|
using System;
|
||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using NpgsqlTypes;
|
using NpgsqlTypes;
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
using System;
|
using System;
|
||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -1,4 +1,4 @@
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using NpgsqlTypes;
|
using NpgsqlTypes;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -1,4 +1,4 @@
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Pgvector;
|
using Pgvector;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|||||||
@@ -114,9 +114,23 @@ public class SearchService : ISearchService
|
|||||||
ranked = matched.OrderByDescending(e => e.SearchVector!.RankCoverDensity(EF.Functions.WebSearchToTsQuery("english", term)))
|
ranked = matched.OrderByDescending(e => e.SearchVector!.RankCoverDensity(EF.Functions.WebSearchToTsQuery("english", term)))
|
||||||
.ThenByDescending(e => e.SentAtUtc);
|
.ThenByDescending(e => e.SentAtUtc);
|
||||||
else
|
else
|
||||||
ranked = matched.OrderByDescending(e => e.SentAtUtc);
|
ranked = matched.OrderByDescending(e => e.SentAtUtc).ThenByDescending(e => e.Id);
|
||||||
|
|
||||||
var paged = ranked.Skip((r.Page - 1) * r.PageSize).Take(r.PageSize);
|
// Keyset (cursor) pagination for the browse path: O(pageSize) regardless of depth,
|
||||||
|
// vs OFFSET's O(page*pageSize). The (SentAtUtc, Id) pair with the Id tie-break above
|
||||||
|
// makes the ordering total, so windows never duplicate or skip rows.
|
||||||
|
IQueryable<Email> paged;
|
||||||
|
if (!hasFreeTextQuery && r is { AfterSentAtUtc: { } afterAt, AfterId: { } afterId })
|
||||||
|
{
|
||||||
|
paged = ranked
|
||||||
|
.Where(e => e.SentAtUtc < afterAt || (e.SentAtUtc == afterAt && e.Id.CompareTo(afterId) < 0))
|
||||||
|
.Take(r.PageSize);
|
||||||
|
total = -1; // not recomputed on cursor windows (that's the point)
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
paged = ranked.Skip((r.Page - 1) * r.PageSize).Take(r.PageSize);
|
||||||
|
}
|
||||||
|
|
||||||
// "Why this matched" ts_headline only for EXACT free-text hits (fuzzy/browse get no
|
// "Why this matched" ts_headline only for EXACT free-text hits (fuzzy/browse get no
|
||||||
// highlight — a fuzzy hit has no literal match to headline). Unconditional projections
|
// highlight — a fuzzy hit has no literal match to headline). Unconditional projections
|
||||||
|
|||||||
@@ -0,0 +1,57 @@
|
|||||||
|
using FluentAssertions;
|
||||||
|
using InboxIntel.Application.Abstractions;
|
||||||
|
using InboxIntel.Application.DTOs;
|
||||||
|
using InboxIntel.Domain.Entities;
|
||||||
|
using InboxIntel.Infrastructure.Persistence;
|
||||||
|
using InboxIntel.Infrastructure.Search;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Xunit;
|
||||||
|
|
||||||
|
namespace InboxIntel.IntegrationTests;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// RECOMMENDATIONS #8: keyset (cursor) pagination for the browse path — the window after a
|
||||||
|
/// (SentAtUtc, Id) cursor returns the next rows with no duplicates/skips and no OFFSET scan.
|
||||||
|
/// </summary>
|
||||||
|
public class KeysetPaginationTests
|
||||||
|
{
|
||||||
|
private sealed class FakeCurrentUser : ICurrentUser
|
||||||
|
{
|
||||||
|
public Guid UserId { get; set; }
|
||||||
|
public bool IsAuthenticated => UserId != Guid.Empty;
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task Cursor_window_continues_exactly_after_the_previous_page()
|
||||||
|
{
|
||||||
|
var user = Guid.NewGuid();
|
||||||
|
var opts = new DbContextOptionsBuilder<AppDbContext>()
|
||||||
|
.UseInMemoryDatabase(nameof(Cursor_window_continues_exactly_after_the_previous_page)).Options;
|
||||||
|
|
||||||
|
var baseline = DateTimeOffset.UtcNow;
|
||||||
|
using (var seed = new AppDbContext(opts, new FakeCurrentUser()))
|
||||||
|
{
|
||||||
|
var sender = new Sender { UserId = user, Address = "s@x.x" };
|
||||||
|
seed.Senders.Add(sender);
|
||||||
|
for (var i = 0; i < 5; i++)
|
||||||
|
seed.Emails.Add(new Email { UserId = user, GmailMessageId = $"m{i}", Sender = sender, SentAtUtc = baseline.AddMinutes(-i) });
|
||||||
|
await seed.SaveChangesAsync();
|
||||||
|
}
|
||||||
|
|
||||||
|
using var ctx = new AppDbContext(opts, new FakeCurrentUser { UserId = user });
|
||||||
|
var svc = new SearchService(ctx);
|
||||||
|
|
||||||
|
// First window via offset (page 1, size 2): m0, m1 (newest first).
|
||||||
|
var page1 = await svc.SearchAsync(user, new SearchRequestDto(null, null, null, null, null, null, null, false, 1, 2));
|
||||||
|
page1.Items.Select(i => i.GmailMessageId).Should().Equal("m0", "m1");
|
||||||
|
|
||||||
|
// Next window via cursor from the last row of page 1.
|
||||||
|
var last = page1.Items[^1];
|
||||||
|
var page2 = await svc.SearchAsync(user, new SearchRequestDto(
|
||||||
|
null, null, null, null, null, null, null, false, 1, 2,
|
||||||
|
AfterSentAtUtc: last.SentAtUtc, AfterId: last.Id));
|
||||||
|
|
||||||
|
page2.Items.Select(i => i.GmailMessageId).Should().Equal("m2", "m3"); // no dupes, no skips
|
||||||
|
page2.TotalCount.Should().Be(-1, "cursor windows skip the COUNT — that's the perf win");
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user