From e5bcf9d5eab60bd44229093015d1de62e8541010 Mon Sep 17 00:00:00 2001 From: cesnimda Date: Wed, 1 Apr 2026 16:09:29 +0200 Subject: [PATCH 1/3] feat: harden gmail sync foundation --- JobTrackerApi.Tests/GmailControllerTests.cs | 33 +++ JobTrackerApi/Controllers/GmailController.cs | 32 ++- JobTrackerApi/Program.cs | 61 +++- .../Services/GmailCorrespondenceEnrichment.cs | 21 ++ JobTrackerApi/Services/GmailOAuthService.cs | 267 +++++++++++------- Models/GmailConnection.cs | 6 + SMART_GMAIL_PROGRESS.md | 40 +++ docs/gmail-correspondence-phase1.md | 55 ++++ .../src/components/Correspondence.tsx | 17 ++ .../src/correspondence-gmail-import.test.tsx | 12 +- job-tracker-ui/src/types.ts | 6 + 11 files changed, 435 insertions(+), 115 deletions(-) create mode 100644 JobTrackerApi/Services/GmailCorrespondenceEnrichment.cs create mode 100644 SMART_GMAIL_PROGRESS.md create mode 100644 docs/gmail-correspondence-phase1.md diff --git a/JobTrackerApi.Tests/GmailControllerTests.cs b/JobTrackerApi.Tests/GmailControllerTests.cs index bfde776..2515e07 100644 --- a/JobTrackerApi.Tests/GmailControllerTests.cs +++ b/JobTrackerApi.Tests/GmailControllerTests.cs @@ -15,6 +15,39 @@ namespace JobTrackerApi.Tests; public sealed class GmailControllerTests { + [Fact] + public async Task Status_returns_sync_state_fields_for_connected_account() + { + await using var db = CreateDb(); + var gmail = new Mock(); + gmail.Setup(service => service.GetConnectionAsync("user-1", It.IsAny())) + .ReturnsAsync(new GmailConnection + { + OwnerUserId = "user-1", + GmailAddress = "user@example.test", + ConnectedAt = DateTimeOffset.UtcNow.AddDays(-3), + LastSyncedAt = DateTimeOffset.UtcNow.AddMinutes(-10), + LastSyncAttemptedAt = DateTimeOffset.UtcNow.AddMinutes(-5), + LastSyncSucceededAt = DateTimeOffset.UtcNow.AddMinutes(-10), + LastSyncMode = "list-messages", + LastSyncSource = "custom-query", + LastSyncStatus = "error", + LastSyncError = "Token refresh failed" + }); + + var controller = CreateController(db, gmail.Object, "user-1"); + var result = await controller.Status(CancellationToken.None); + + var ok = Assert.IsType(result.Result); + var payload = Assert.IsType(ok.Value); + Assert.True(payload.Connected); + Assert.Equal("user@example.test", payload.GmailAddress); + Assert.Equal("list-messages", payload.LastSyncMode); + Assert.Equal("custom-query", payload.LastSyncSource); + Assert.Equal("error", payload.LastSyncStatus); + Assert.Equal("Token refresh failed", payload.LastSyncError); + } + [Fact] public async Task Import_thread_rejects_missing_message_ids() { diff --git a/JobTrackerApi/Controllers/GmailController.cs b/JobTrackerApi/Controllers/GmailController.cs index 45f7df5..c9f89f1 100644 --- a/JobTrackerApi/Controllers/GmailController.cs +++ b/JobTrackerApi/Controllers/GmailController.cs @@ -68,18 +68,34 @@ public sealed class GmailController : ControllerBase int CandidateThreadCount, IReadOnlyList Threads); + public sealed record GmailConnectionStatusDto( + bool Connected, + string? GmailAddress, + DateTimeOffset? ConnectedAt, + DateTimeOffset? LastSyncedAt, + DateTimeOffset? LastSyncAttemptedAt, + DateTimeOffset? LastSyncSucceededAt, + string? LastSyncMode, + string? LastSyncSource, + string? LastSyncStatus, + string? LastSyncError); + [HttpGet("status")] - public async Task Status(CancellationToken cancellationToken) + public async Task> Status(CancellationToken cancellationToken) { var ownerUserId = GetRequiredOwnerUserId(); var connection = await _gmail.GetConnectionAsync(ownerUserId, cancellationToken); - return Ok(new - { - connected = connection is not null, - gmailAddress = connection?.GmailAddress, - connectedAt = connection?.ConnectedAt, - lastSyncedAt = connection?.LastSyncedAt, - }); + return Ok(new GmailConnectionStatusDto( + connection is not null, + connection?.GmailAddress, + connection?.ConnectedAt, + connection?.LastSyncedAt, + connection?.LastSyncAttemptedAt, + connection?.LastSyncSucceededAt, + connection?.LastSyncMode, + connection?.LastSyncSource, + connection?.LastSyncStatus, + connection?.LastSyncError)); } [HttpGet("connect-url")] diff --git a/JobTrackerApi/Program.cs b/JobTrackerApi/Program.cs index 377f254..74c08ad 100644 --- a/JobTrackerApi/Program.cs +++ b/JobTrackerApi/Program.cs @@ -135,6 +135,7 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddScoped(); +builder.Services.AddSingleton(); builder.Services.AddIdentityCore(options => { @@ -628,10 +629,23 @@ CREATE TABLE IF NOT EXISTS "GmailConnections" ( "AccessTokenExpiresAt" TEXT NULL, "Scope" TEXT NOT NULL, "ConnectedAt" TEXT NOT NULL, - "LastSyncedAt" TEXT NULL + "LastSyncedAt" TEXT NULL, + "LastSyncAttemptedAt" TEXT NULL, + "LastSyncSucceededAt" TEXT NULL, + "LastSyncMode" TEXT NULL, + "LastSyncSource" TEXT NULL, + "LastSyncStatus" TEXT NULL, + "LastSyncError" TEXT NULL ); """); + EnsureColumn(c, "GmailConnections", "LastSyncAttemptedAt", "ALTER TABLE GmailConnections ADD COLUMN LastSyncAttemptedAt TEXT NULL;"); + EnsureColumn(c, "GmailConnections", "LastSyncSucceededAt", "ALTER TABLE GmailConnections ADD COLUMN LastSyncSucceededAt TEXT NULL;"); + EnsureColumn(c, "GmailConnections", "LastSyncMode", "ALTER TABLE GmailConnections ADD COLUMN LastSyncMode TEXT NULL;"); + EnsureColumn(c, "GmailConnections", "LastSyncSource", "ALTER TABLE GmailConnections ADD COLUMN LastSyncSource TEXT NULL;"); + EnsureColumn(c, "GmailConnections", "LastSyncStatus", "ALTER TABLE GmailConnections ADD COLUMN LastSyncStatus TEXT NULL;"); + EnsureColumn(c, "GmailConnections", "LastSyncError", "ALTER TABLE GmailConnections ADD COLUMN LastSyncError TEXT NULL;"); + Exec(c, """CREATE INDEX IF NOT EXISTS "IX_GmailConnections_OwnerUserId" ON "GmailConnections" ("OwnerUserId");"""); Exec(c, """CREATE UNIQUE INDEX IF NOT EXISTS "IX_GmailConnections_OwnerUserId_GmailAddress" ON "GmailConnections" ("OwnerUserId", "GmailAddress");"""); } @@ -939,6 +953,37 @@ CONSTRAINT `FK_CvExtractionRuns_CvUploadArtifacts_ArtifactId` FOREIGN KEY (`Arti cmd.ExecuteNonQuery(); } + if (!HasMySqlTable(conn, "GmailConnections")) + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = @"CREATE TABLE IF NOT EXISTS `GmailConnections` ( +`Id` int NOT NULL AUTO_INCREMENT, +`OwnerUserId` varchar(255) NOT NULL, +`GmailAddress` varchar(512) NOT NULL, +`EncryptedRefreshToken` longtext NOT NULL, +`EncryptedAccessToken` longtext NULL, +`AccessTokenExpiresAt` datetime(6) NULL, +`Scope` longtext NOT NULL, +`ConnectedAt` datetime(6) NOT NULL, +`LastSyncedAt` datetime(6) NULL, +`LastSyncAttemptedAt` datetime(6) NULL, +`LastSyncSucceededAt` datetime(6) NULL, +`LastSyncMode` varchar(255) NULL, +`LastSyncSource` varchar(255) NULL, +`LastSyncStatus` varchar(255) NULL, +`LastSyncError` longtext NULL, +PRIMARY KEY (`Id`) +);"; + cmd.ExecuteNonQuery(); + } + + EnsureMySqlColumn(conn, "GmailConnections", "LastSyncAttemptedAt", "ALTER TABLE `GmailConnections` ADD COLUMN `LastSyncAttemptedAt` datetime(6) NULL;"); + EnsureMySqlColumn(conn, "GmailConnections", "LastSyncSucceededAt", "ALTER TABLE `GmailConnections` ADD COLUMN `LastSyncSucceededAt` datetime(6) NULL;"); + EnsureMySqlColumn(conn, "GmailConnections", "LastSyncMode", "ALTER TABLE `GmailConnections` ADD COLUMN `LastSyncMode` varchar(255) NULL;"); + EnsureMySqlColumn(conn, "GmailConnections", "LastSyncSource", "ALTER TABLE `GmailConnections` ADD COLUMN `LastSyncSource` varchar(255) NULL;"); + EnsureMySqlColumn(conn, "GmailConnections", "LastSyncStatus", "ALTER TABLE `GmailConnections` ADD COLUMN `LastSyncStatus` varchar(255) NULL;"); + EnsureMySqlColumn(conn, "GmailConnections", "LastSyncError", "ALTER TABLE `GmailConnections` ADD COLUMN `LastSyncError` longtext NULL;"); + if (!HasMySqlTable(conn, "TailoredCvDrafts")) { using var cmd = conn.CreateCommand(); @@ -1000,6 +1045,20 @@ CONSTRAINT `FK_TailoredCvDrafts_JobApplications_JobApplicationId` FOREIGN KEY (` cmd.ExecuteNonQuery(); } + if (!MySqlIndexExists(conn, "GmailConnections", "IX_GmailConnections_OwnerUserId")) + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = "CREATE INDEX `IX_GmailConnections_OwnerUserId` ON `GmailConnections` (`OwnerUserId`);"; + cmd.ExecuteNonQuery(); + } + + if (!MySqlIndexExists(conn, "GmailConnections", "IX_GmailConnections_OwnerUserId_GmailAddress")) + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = "CREATE UNIQUE INDEX `IX_GmailConnections_OwnerUserId_GmailAddress` ON `GmailConnections` (`OwnerUserId`, `GmailAddress`);"; + cmd.ExecuteNonQuery(); + } + if (!MySqlIndexExists(conn, "TailoredCvDrafts", "IX_TailoredCvDrafts_OwnerUserId_JobApplicationId")) { using var cmd = conn.CreateCommand(); diff --git a/JobTrackerApi/Services/GmailCorrespondenceEnrichment.cs b/JobTrackerApi/Services/GmailCorrespondenceEnrichment.cs new file mode 100644 index 0000000..163ab15 --- /dev/null +++ b/JobTrackerApi/Services/GmailCorrespondenceEnrichment.cs @@ -0,0 +1,21 @@ +namespace JobTrackerApi.Services; + +public sealed record GmailSemanticMatchCandidate( + int? JobApplicationId, + string? Confidence, + string? Reason, + IReadOnlyList? ExtractedCompanies, + IReadOnlyList? ExtractedRecruiters, + IReadOnlyList? ExtractedRoles, + IReadOnlyList? ExtractedHints); + +public interface IGmailCorrespondenceEnrichmentService +{ + Task EnrichAsync(string threadSubject, string from, string to, string snippet, string? bodyText, CancellationToken cancellationToken = default); +} + +public sealed class NoOpGmailCorrespondenceEnrichmentService : IGmailCorrespondenceEnrichmentService +{ + public Task EnrichAsync(string threadSubject, string from, string to, string snippet, string? bodyText, CancellationToken cancellationToken = default) + => Task.FromResult(null); +} diff --git a/JobTrackerApi/Services/GmailOAuthService.cs b/JobTrackerApi/Services/GmailOAuthService.cs index f884e0a..01dae06 100644 --- a/JobTrackerApi/Services/GmailOAuthService.cs +++ b/JobTrackerApi/Services/GmailOAuthService.cs @@ -116,6 +116,12 @@ public sealed class GmailOAuthService : IGmailOAuthService existing.AccessTokenExpiresAt = DateTimeOffset.UtcNow.AddSeconds(Math.Max(tokens.expires_in - 60, 60)); existing.Scope = tokens.scope?.Trim() ?? Scope; existing.ConnectedAt = DateTimeOffset.UtcNow; + existing.LastSyncStatus = "connected"; + existing.LastSyncSource = "oauth-callback"; + existing.LastSyncMode = "connect"; + existing.LastSyncError = null; + existing.LastSyncAttemptedAt = DateTimeOffset.UtcNow; + existing.LastSyncSucceededAt = existing.LastSyncAttemptedAt; await _db.SaveChangesAsync(cancellationToken); return new GmailOAuthExchangeResult(existing.GmailAddress); @@ -148,40 +154,49 @@ public sealed class GmailOAuthService : IGmailOAuthService public async Task> ListMessagesAsync(string ownerUserId, string? query, int maxResults, CancellationToken cancellationToken) { maxResults = Math.Clamp(maxResults, 1, 25); - var accessToken = await GetValidAccessTokenAsync(ownerUserId, cancellationToken); - var client = _httpClientFactory.CreateClient(); - client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", accessToken); - - var url = $"https://gmail.googleapis.com/gmail/v1/users/me/messages?maxResults={maxResults}"; - if (!string.IsNullOrWhiteSpace(query)) + try { - url += $"&q={Uri.EscapeDataString(query.Trim())}"; + var accessToken = await GetValidAccessTokenAsync(ownerUserId, cancellationToken); + var client = _httpClientFactory.CreateClient(); + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", accessToken); + + var url = $"https://gmail.googleapis.com/gmail/v1/users/me/messages?maxResults={maxResults}"; + if (!string.IsNullOrWhiteSpace(query)) + { + url += $"&q={Uri.EscapeDataString(query.Trim())}"; + } + + using var response = await client.GetAsync(url, cancellationToken); + response.EnsureSuccessStatusCode(); + + using var doc = await JsonDocument.ParseAsync(await response.Content.ReadAsStreamAsync(cancellationToken), cancellationToken: cancellationToken); + if (!doc.RootElement.TryGetProperty("messages", out var messagesElement) || messagesElement.ValueKind != JsonValueKind.Array) + { + await TouchSyncStateAsync(ownerUserId, "list-messages", string.IsNullOrWhiteSpace(query) ? "default-query" : "custom-query", true, null, cancellationToken); + return Array.Empty(); + } + + var ids = messagesElement.EnumerateArray() + .Select(x => x.TryGetProperty("id", out var id) ? id.GetString() : null) + .Where(x => !string.IsNullOrWhiteSpace(x)) + .Cast() + .ToList(); + + var results = new List(ids.Count); + foreach (var id in ids) + { + var detail = await GetMessageAsync(ownerUserId, id, cancellationToken); + results.Add(new GmailMessageSummary(detail.Id, detail.ThreadId, detail.Subject, detail.From, detail.To, detail.Date, detail.Snippet)); + } + + await TouchSyncStateAsync(ownerUserId, "list-messages", string.IsNullOrWhiteSpace(query) ? "default-query" : "custom-query", true, null, cancellationToken); + return results; } - - using var response = await client.GetAsync(url, cancellationToken); - response.EnsureSuccessStatusCode(); - - using var doc = await JsonDocument.ParseAsync(await response.Content.ReadAsStreamAsync(cancellationToken), cancellationToken: cancellationToken); - if (!doc.RootElement.TryGetProperty("messages", out var messagesElement) || messagesElement.ValueKind != JsonValueKind.Array) + catch (Exception ex) { - return Array.Empty(); + await TouchSyncStateAsync(ownerUserId, "list-messages", string.IsNullOrWhiteSpace(query) ? "default-query" : "custom-query", false, ex.Message, cancellationToken); + throw; } - - var ids = messagesElement.EnumerateArray() - .Select(x => x.TryGetProperty("id", out var id) ? id.GetString() : null) - .Where(x => !string.IsNullOrWhiteSpace(x)) - .Cast() - .ToList(); - - var results = new List(ids.Count); - foreach (var id in ids) - { - var detail = await GetMessageAsync(ownerUserId, id, cancellationToken); - results.Add(new GmailMessageSummary(detail.Id, detail.ThreadId, detail.Subject, detail.From, detail.To, detail.Date, detail.Snippet)); - } - - await TouchSyncTimeAsync(ownerUserId, cancellationToken); - return results; } public async Task> ListMessagesForQueriesAsync(string ownerUserId, IEnumerable queries, int maxResultsPerQuery, CancellationToken cancellationToken) @@ -233,93 +248,111 @@ public sealed class GmailOAuthService : IGmailOAuthService return Array.Empty(); } - var accessToken = await GetValidAccessTokenAsync(ownerUserId, cancellationToken); - var client = _httpClientFactory.CreateClient(); - client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", accessToken); - - var url = $"https://gmail.googleapis.com/gmail/v1/users/me/threads/{Uri.EscapeDataString(threadId.Trim())}?format=metadata&metadataHeaders=Subject&metadataHeaders=From&metadataHeaders=To&metadataHeaders=Date"; - using var response = await client.GetAsync(url, cancellationToken); - response.EnsureSuccessStatusCode(); - - using var doc = await JsonDocument.ParseAsync(await response.Content.ReadAsStreamAsync(cancellationToken), cancellationToken: cancellationToken); - if (!doc.RootElement.TryGetProperty("messages", out var messagesElement) || messagesElement.ValueKind != JsonValueKind.Array) + try { - return Array.Empty(); - } + var accessToken = await GetValidAccessTokenAsync(ownerUserId, cancellationToken); + var client = _httpClientFactory.CreateClient(); + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", accessToken); - var results = new List(); - foreach (var messageElement in messagesElement.EnumerateArray()) - { - var id = messageElement.TryGetProperty("id", out var idEl) ? idEl.GetString() : null; - if (string.IsNullOrWhiteSpace(id)) continue; + var url = $"https://gmail.googleapis.com/gmail/v1/users/me/threads/{Uri.EscapeDataString(threadId.Trim())}?format=metadata&metadataHeaders=Subject&metadataHeaders=From&metadataHeaders=To&metadataHeaders=Date"; + using var response = await client.GetAsync(url, cancellationToken); + response.EnsureSuccessStatusCode(); - var messageThreadId = messageElement.TryGetProperty("threadId", out var messageThreadIdEl) - ? messageThreadIdEl.GetString() ?? threadId.Trim() - : threadId.Trim(); - var snippet = messageElement.TryGetProperty("snippet", out var snippetEl) ? snippetEl.GetString() ?? string.Empty : string.Empty; - var payload = messageElement.TryGetProperty("payload", out var payloadEl) ? payloadEl : default; - var headers = payload.ValueKind == JsonValueKind.Object ? ReadHeaders(payload) : new Dictionary(StringComparer.OrdinalIgnoreCase); - - DateTimeOffset? date = null; - if (headers.TryGetValue("date", out var dateHeader) && DateTimeOffset.TryParse(dateHeader, out var parsedDate)) + using var doc = await JsonDocument.ParseAsync(await response.Content.ReadAsStreamAsync(cancellationToken), cancellationToken: cancellationToken); + if (!doc.RootElement.TryGetProperty("messages", out var messagesElement) || messagesElement.ValueKind != JsonValueKind.Array) { - date = parsedDate; + await TouchSyncStateAsync(ownerUserId, "thread-refresh", "thread-metadata", true, null, cancellationToken); + return Array.Empty(); } - results.Add(new GmailMessageSummary( - id.Trim(), - messageThreadId, - headers.TryGetValue("subject", out var subject) ? subject : string.Empty, - headers.TryGetValue("from", out var from) ? from : string.Empty, - headers.TryGetValue("to", out var to) ? to : string.Empty, - date, - snippet)); - } + var results = new List(); + foreach (var messageElement in messagesElement.EnumerateArray()) + { + var id = messageElement.TryGetProperty("id", out var idEl) ? idEl.GetString() : null; + if (string.IsNullOrWhiteSpace(id)) continue; - await TouchSyncTimeAsync(ownerUserId, cancellationToken); - return results; + var messageThreadId = messageElement.TryGetProperty("threadId", out var messageThreadIdEl) + ? messageThreadIdEl.GetString() ?? threadId.Trim() + : threadId.Trim(); + var snippet = messageElement.TryGetProperty("snippet", out var snippetEl) ? snippetEl.GetString() ?? string.Empty : string.Empty; + var payload = messageElement.TryGetProperty("payload", out var payloadEl) ? payloadEl : default; + var headers = payload.ValueKind == JsonValueKind.Object ? ReadHeaders(payload) : new Dictionary(StringComparer.OrdinalIgnoreCase); + + DateTimeOffset? date = null; + if (headers.TryGetValue("date", out var dateHeader) && DateTimeOffset.TryParse(dateHeader, out var parsedDate)) + { + date = parsedDate; + } + + results.Add(new GmailMessageSummary( + id.Trim(), + messageThreadId, + headers.TryGetValue("subject", out var subject) ? subject : string.Empty, + headers.TryGetValue("from", out var from) ? from : string.Empty, + headers.TryGetValue("to", out var to) ? to : string.Empty, + date, + snippet)); + } + + await TouchSyncStateAsync(ownerUserId, "thread-refresh", "thread-metadata", true, null, cancellationToken); + return results; + } + catch (Exception ex) + { + await TouchSyncStateAsync(ownerUserId, "thread-refresh", "thread-metadata", false, ex.Message, cancellationToken); + throw; + } } public async Task GetMessageAsync(string ownerUserId, string messageId, CancellationToken cancellationToken) { - var accessToken = await GetValidAccessTokenAsync(ownerUserId, cancellationToken); - var client = _httpClientFactory.CreateClient(); - client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", accessToken); - - var url = $"https://gmail.googleapis.com/gmail/v1/users/me/messages/{Uri.EscapeDataString(messageId)}?format=full"; - using var response = await client.GetAsync(url, cancellationToken); - response.EnsureSuccessStatusCode(); - - using var doc = await JsonDocument.ParseAsync(await response.Content.ReadAsStreamAsync(cancellationToken), cancellationToken: cancellationToken); - var root = doc.RootElement; - - var threadId = root.TryGetProperty("threadId", out var threadEl) ? threadEl.GetString() ?? "" : ""; - var snippet = root.TryGetProperty("snippet", out var snippetEl) ? snippetEl.GetString() ?? "" : ""; - - var payload = root.GetProperty("payload"); - var headers = ReadHeaders(payload); - var bodyText = ExtractBody(payload, "text/plain"); - var bodyHtml = ExtractBody(payload, "text/html"); - if (string.IsNullOrWhiteSpace(bodyText) && !string.IsNullOrWhiteSpace(bodyHtml)) + try { - bodyText = StripHtml(bodyHtml); - } - else if (LooksLikeHtml(bodyText)) - { - bodyText = StripHtml(bodyText); - } + var accessToken = await GetValidAccessTokenAsync(ownerUserId, cancellationToken); + var client = _httpClientFactory.CreateClient(); + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", accessToken); - return new GmailMessageDetail( - messageId, - threadId, - headers.TryGetValue("subject", out var subject) ? subject : "", - headers.TryGetValue("from", out var from) ? from : "", - headers.TryGetValue("to", out var to) ? to : "", - headers.TryGetValue("date", out var dateRaw) && DateTimeOffset.TryParse(dateRaw, out var parsedDate) ? parsedDate : null, - snippet, - bodyText.Trim(), - string.IsNullOrWhiteSpace(bodyHtml) ? null : bodyHtml - ); + var url = $"https://gmail.googleapis.com/gmail/v1/users/me/messages/{Uri.EscapeDataString(messageId)}?format=full"; + using var response = await client.GetAsync(url, cancellationToken); + response.EnsureSuccessStatusCode(); + + using var doc = await JsonDocument.ParseAsync(await response.Content.ReadAsStreamAsync(cancellationToken), cancellationToken: cancellationToken); + var root = doc.RootElement; + + var threadId = root.TryGetProperty("threadId", out var threadEl) ? threadEl.GetString() ?? "" : ""; + var snippet = root.TryGetProperty("snippet", out var snippetEl) ? snippetEl.GetString() ?? "" : ""; + + var payload = root.GetProperty("payload"); + var headers = ReadHeaders(payload); + var bodyText = ExtractBody(payload, "text/plain"); + var bodyHtml = ExtractBody(payload, "text/html"); + if (string.IsNullOrWhiteSpace(bodyText) && !string.IsNullOrWhiteSpace(bodyHtml)) + { + bodyText = StripHtml(bodyHtml); + } + else if (LooksLikeHtml(bodyText)) + { + bodyText = StripHtml(bodyText); + } + + await TouchSyncStateAsync(ownerUserId, "message-detail", "gmail-message", true, null, cancellationToken); + return new GmailMessageDetail( + messageId, + threadId, + headers.TryGetValue("subject", out var subject) ? subject : "", + headers.TryGetValue("from", out var from) ? from : "", + headers.TryGetValue("to", out var to) ? to : "", + headers.TryGetValue("date", out var dateRaw) && DateTimeOffset.TryParse(dateRaw, out var parsedDate) ? parsedDate : null, + snippet, + bodyText.Trim(), + string.IsNullOrWhiteSpace(bodyHtml) ? null : bodyHtml + ); + } + catch (Exception ex) + { + await TouchSyncStateAsync(ownerUserId, "message-detail", "gmail-message", false, ex.Message, cancellationToken); + throw; + } } private async Task GetValidAccessTokenAsync(string ownerUserId, CancellationToken cancellationToken) @@ -435,13 +468,37 @@ public sealed class GmailOAuthService : IGmailOAuthService } private async Task TouchSyncTimeAsync(string ownerUserId, CancellationToken cancellationToken) + { + await TouchSyncStateAsync(ownerUserId, "sync", "gmail", true, null, cancellationToken); + } + + private async Task TouchSyncStateAsync(string ownerUserId, string mode, string source, bool succeeded, string? error, CancellationToken cancellationToken) { var connection = await _db.GmailConnections.FirstOrDefaultAsync(x => x.OwnerUserId == ownerUserId, cancellationToken); if (connection is null) return; - connection.LastSyncedAt = DateTimeOffset.UtcNow; + + var now = DateTimeOffset.UtcNow; + connection.LastSyncAttemptedAt = now; + connection.LastSyncMode = mode; + connection.LastSyncSource = source; + connection.LastSyncStatus = succeeded ? "success" : "error"; + connection.LastSyncError = succeeded ? null : TrimError(error); + if (succeeded) + { + connection.LastSyncedAt = now; + connection.LastSyncSucceededAt = now; + } + await _db.SaveChangesAsync(cancellationToken); } + private static string? TrimError(string? value) + { + if (string.IsNullOrWhiteSpace(value)) return null; + var trimmed = value.Trim(); + return trimmed.Length <= 300 ? trimmed : trimmed[..300]; + } + private string GetRequiredClientId() { return (_cfg["Google:ClientId"] ?? _cfg["Auth:GoogleClientId"] ?? "").Trim() switch diff --git a/Models/GmailConnection.cs b/Models/GmailConnection.cs index 024671a..ba93a99 100644 --- a/Models/GmailConnection.cs +++ b/Models/GmailConnection.cs @@ -11,4 +11,10 @@ public sealed class GmailConnection public string Scope { get; set; } = ""; public DateTimeOffset ConnectedAt { get; set; } = DateTimeOffset.UtcNow; public DateTimeOffset? LastSyncedAt { get; set; } + public DateTimeOffset? LastSyncAttemptedAt { get; set; } + public DateTimeOffset? LastSyncSucceededAt { get; set; } + public string? LastSyncMode { get; set; } + public string? LastSyncSource { get; set; } + public string? LastSyncStatus { get; set; } + public string? LastSyncError { get; set; } } diff --git a/SMART_GMAIL_PROGRESS.md b/SMART_GMAIL_PROGRESS.md new file mode 100644 index 0000000..80eb902 --- /dev/null +++ b/SMART_GMAIL_PROGRESS.md @@ -0,0 +1,40 @@ +# Smart Gmail Job Correspondence Integration Progress + +## Branch +- feat/gmail-job-correspondence + +## Status +- Workstream initialized. +- Milestones planned: M006-M010. +- Current focus: M006 / S01 foundation work. + +## Completed so far +- Created separate Gmail feature branch. +- Captured foundation context in `.gsd/milestones/M006/M006-CONTEXT.md`. +- Planned milestones M006-M010 for the Gmail workstream. +- Planned slice M006/S01. +- Confirmed existing architecture seams: + - Gmail OAuth/token flow already exists. + - Per-job Gmail candidate search/import/thread refresh already exists. + - Correspondence persistence already stores Gmail thread/message metadata. + - Current implementation is job-local, not global-inbox/review oriented. +- Implemented M006/S01 foundation changes: + - durable Gmail sync-state fields on `GmailConnection` + - SQLite/MySQL bootstrap support for new Gmail sync-state columns + - richer `GET /api/gmail/status` response + - per-job correspondence UI now shows sync diagnostics + - focused backend/frontend Gmail tests added and passing + - Phase 2 extension seam scaffolded with a no-op enrichment service + - Phase 1/2 design doc added at `docs/gmail-correspondence-phase1.md` + +## Next tasks +1. Implement M006/S01/T01: refactor Gmail connection foundation and sync-state model. +2. Implement M006/S01/T02: expose sync-state surfaces in UI without breaking current correspondence workflow. +3. Implement M006/S01/T03: prepare Phase 2 extension seam/docs. +4. Verify backend + frontend Gmail focused tests. +5. Commit and push incremental progress. + +## Resume notes +- Previous CV/parsing branch work is separate and already pushed. +- Local dev SQLite runtime still has missing-table drift in some unrelated surfaces (`RuleSettings`, `Companies`, etc.); avoid conflating that with the Gmail feature work. +- Existing per-job Gmail tests live in `JobTrackerApi.Tests/GmailControllerTests.cs` and `job-tracker-ui/src/correspondence-gmail-import.test.tsx`. diff --git a/docs/gmail-correspondence-phase1.md b/docs/gmail-correspondence-phase1.md new file mode 100644 index 0000000..ddb4a52 --- /dev/null +++ b/docs/gmail-correspondence-phase1.md @@ -0,0 +1,55 @@ +# Smart Gmail Job Correspondence Integration + +## Phase split + +### Phase 1 +Deterministic, high-trust Gmail job correspondence integration: +- OAuth/account connection +- token refresh lifecycle +- sync-state tracking +- manual sync/backfill +- dedup by Gmail message id +- deterministic job-linking + confidence routing +- review queue for medium-confidence items +- unmatched thread suggestions +- global inbox + per-job timeline + +### Phase 2 +Prepared, not deeply implemented in this slice: +- semantic Gmail-to-job disambiguation +- richer recruiter/company/role extraction +- stage/status hinting +- interview/rejection/offer extraction +- follow-up/reply suggestion generation + +## Foundation decisions + +- Phase 1 remains useful without AI/Ollama. +- Deterministic evidence remains the primary truth source. +- Future AI enrichment attaches reasons/confidence alongside deterministic evidence rather than replacing it. +- Gmail sync state is now durable on the Gmail connection record: + - `LastSyncAttemptedAt` + - `LastSyncSucceededAt` + - `LastSyncMode` + - `LastSyncSource` + - `LastSyncStatus` + - `LastSyncError` + +## Current code seams + +- Gmail OAuth and token lifecycle: `JobTrackerApi/Services/GmailOAuthService.cs` +- Gmail endpoints: `JobTrackerApi/Controllers/GmailController.cs` +- Gmail connection persistence: `Models/GmailConnection.cs` +- Correspondence persistence: `Models/Correspondence.cs` +- Per-job correspondence UX: `job-tracker-ui/src/components/Correspondence.tsx` +- Future Phase 2 AI seam: `JobTrackerApi/Services/GmailCorrespondenceEnrichment.cs` + +## What Phase 2 should plug into later + +The `IGmailCorrespondenceEnrichmentService` seam is intended to accept normalized Gmail message/thread context and return optional semantic hints: +- probable job match +- richer confidence rationale +- extracted recruiter/company/role entities +- lightweight stage hints + +Phase 1 should never require this service to return anything useful. The default runtime implementation remains a no-op. diff --git a/job-tracker-ui/src/components/Correspondence.tsx b/job-tracker-ui/src/components/Correspondence.tsx index 25302a8..6ea5066 100644 --- a/job-tracker-ui/src/components/Correspondence.tsx +++ b/job-tracker-ui/src/components/Correspondence.tsx @@ -413,6 +413,16 @@ export default function Correspondence({ jobId, job }: { jobId: number; job: Job 0 ? "success" : "default"} variant="outlined" label={linkedThreadIds.length > 0 ? `Linked threads: ${linkedThreadIds.length}` : "No linked threads yet"} /> + {gmailStatus?.lastSyncStatus ? ( + + ) : null} {linkedThreadRefresh ? ( ) : null} + {gmailStatus?.lastSyncError ? ( + + Latest Gmail sync issue: {gmailStatus.lastSyncError} + + ) : null} @@ -503,6 +518,8 @@ export default function Correspondence({ jobId, job }: { jobId: number; job: Job ) : null} {gmailStatus.lastSyncedAt ? : null} + {gmailStatus.lastSyncAttemptedAt ? : null} + {gmailStatus.lastSyncStatus === "error" && gmailStatus.lastSyncError ? : null} {linkedThreadIds.length > 0 ? ( diff --git a/job-tracker-ui/src/correspondence-gmail-import.test.tsx b/job-tracker-ui/src/correspondence-gmail-import.test.tsx index 7bb49ae..fa63a22 100644 --- a/job-tracker-ui/src/correspondence-gmail-import.test.tsx +++ b/job-tracker-ui/src/correspondence-gmail-import.test.tsx @@ -78,7 +78,7 @@ describe("correspondence Gmail import", () => { return Promise.resolve({ data: correspondenceMessages } as any); } if (url === "/gmail/status") { - return Promise.resolve({ data: { connected: true, gmailAddress: "user@example.test", lastSyncedAt: new Date().toISOString() } } as any); + return Promise.resolve({ data: { connected: true, gmailAddress: "user@example.test", lastSyncedAt: new Date().toISOString(), lastSyncAttemptedAt: new Date().toISOString(), lastSyncMode: "list-messages", lastSyncSource: "custom-query", lastSyncStatus: "error", lastSyncError: "Token refresh failed" } } as any); } if (url === "/gmail/job-candidates") { return Promise.resolve({ @@ -291,6 +291,16 @@ describe("correspondence Gmail import", () => { expect((await screen.findAllByText(/thread thread-1/i)).length).toBeGreaterThan(0); }); + test("shows Gmail sync state diagnostics alongside linked thread continuity", async () => { + renderDialog(); + + fireEvent.click(await screen.findByRole("button", { name: /import email/i })); + fireEvent.click(await screen.findByRole("tab", { name: /^google$/i })); + + expect(await screen.findByText(/sync checked/i)).toBeInTheDocument(); + expect((await screen.findAllByText(/token refresh failed/i)).length).toBeGreaterThan(0); + }); + test("manual Gmail search override reloads job candidates with queryOverride", async () => { renderDialog(); diff --git a/job-tracker-ui/src/types.ts b/job-tracker-ui/src/types.ts index 383ad42..1166b1b 100644 --- a/job-tracker-ui/src/types.ts +++ b/job-tracker-ui/src/types.ts @@ -293,6 +293,12 @@ export interface GmailStatus { gmailAddress?: string; connectedAt?: string; lastSyncedAt?: string; + lastSyncAttemptedAt?: string; + lastSyncSucceededAt?: string; + lastSyncMode?: string | null; + lastSyncSource?: string | null; + lastSyncStatus?: string | null; + lastSyncError?: string | null; } export interface GmailMessageSummary { From f48136f04c7de4dce91fa3dbcb5da9f873503911 Mon Sep 17 00:00:00 2001 From: cesnimda Date: Wed, 1 Apr 2026 16:27:34 +0200 Subject: [PATCH 2/3] feat: enrich gmail correspondence metadata --- Data/JobTrackerContext.cs | 2 + JobTrackerApi.Tests/GmailControllerTests.cs | 20 +++++-- .../Controllers/CorrespondenceController.cs | 8 ++- JobTrackerApi/Controllers/GmailController.cs | 11 ++++ JobTrackerApi/Program.cs | 9 ++++ JobTrackerApi/Services/GmailOAuthService.cs | 53 ++++++++++++++++++- Models/Correspondence.cs | 23 ++++++++ .../src/components/Correspondence.tsx | 4 +- job-tracker-ui/src/types.ts | 11 ++++ 9 files changed, 133 insertions(+), 8 deletions(-) diff --git a/Data/JobTrackerContext.cs b/Data/JobTrackerContext.cs index e1f8a9e..c621e56 100644 --- a/Data/JobTrackerContext.cs +++ b/Data/JobTrackerContext.cs @@ -66,6 +66,8 @@ namespace JobTrackerApi.Data modelBuilder.Entity() .HasQueryFilter(x => CurrentUserId == null || x.OwnerUserId == CurrentUserId); + modelBuilder.Ignore(); + modelBuilder.Entity() .HasIndex(x => new { x.OwnerUserId, x.GmailAddress }) .IsUnique(); diff --git a/JobTrackerApi.Tests/GmailControllerTests.cs b/JobTrackerApi.Tests/GmailControllerTests.cs index 2515e07..35779a5 100644 --- a/JobTrackerApi.Tests/GmailControllerTests.cs +++ b/JobTrackerApi.Tests/GmailControllerTests.cs @@ -258,7 +258,9 @@ public sealed class GmailControllerTests DateTimeOffset.UtcNow.AddDays(-1), "Snippet", "Body text", - null)); + null, + new[] { "INBOX", "IMPORTANT" }, + new[] { new GmailMessageAttachment("cv.pdf", "application/pdf", 2048, "att-1", false) })); var controller = CreateController(db, gmail.Object, "user-1"); @@ -272,6 +274,10 @@ public sealed class GmailControllerTests Assert.Equal("thread-1", firstPayload.Message!.ExternalThreadId); Assert.Equal("Maria Recruiter ", firstPayload.Message.ExternalFrom); Assert.Equal("user@example.test", firstPayload.Message.ExternalTo); + Assert.Equal("inbound", firstPayload.Message.Direction); + Assert.Contains("IMPORTANT", firstPayload.Message.ExternalLabels); + Assert.Single(firstPayload.Message.AttachmentMetadata); + Assert.Equal("cv.pdf", firstPayload.Message.AttachmentMetadata[0].FileName); var second = await controller.Import(new GmailController.ImportGmailMessageRequest(job.Id, "msg-1"), CancellationToken.None); var secondOk = Assert.IsType(second.Result); @@ -315,7 +321,9 @@ public sealed class GmailControllerTests DateTimeOffset.UtcNow.AddDays(-1), "Snippet 1", "Body text 1", - null)); + null, + Array.Empty(), + Array.Empty())); gmail.Setup(service => service.GetMessageAsync("user-1", "msg-2", It.IsAny())) .ReturnsAsync(new GmailMessageDetail( "msg-2", @@ -326,7 +334,9 @@ public sealed class GmailControllerTests DateTimeOffset.UtcNow, "Snippet 2", "Body text 2", - null)); + null, + Array.Empty(), + Array.Empty())); var controller = CreateController(db, gmail.Object, "user-1"); var request = new GmailController.ImportGmailThreadRequest(job.Id, "thread-1", new[] { "msg-1", "msg-2" }); @@ -398,7 +408,9 @@ public sealed class GmailControllerTests DateTimeOffset.UtcNow, "New reply", "Reply body", - null)); + null, + Array.Empty(), + Array.Empty())); var controller = CreateController(db, gmail.Object, "user-1"); var result = await controller.RefreshLinkedThreads(new GmailController.RefreshLinkedThreadsRequest(job.Id), CancellationToken.None); diff --git a/JobTrackerApi/Controllers/CorrespondenceController.cs b/JobTrackerApi/Controllers/CorrespondenceController.cs index c82c50d..a97ab20 100644 --- a/JobTrackerApi/Controllers/CorrespondenceController.cs +++ b/JobTrackerApi/Controllers/CorrespondenceController.cs @@ -48,10 +48,13 @@ namespace JobTrackerApi.Controllers string? Subject, string? Channel, DateTime? Date, + string? Direction, string? ExternalMessageId, string? ExternalThreadId, string? ExternalFrom, - string? ExternalTo + string? ExternalTo, + string? ExternalLabelsJson, + string? AttachmentMetadataJson ); // POST new message @@ -71,10 +74,13 @@ namespace JobTrackerApi.Controllers From = request.From.Trim(), Subject = string.IsNullOrWhiteSpace(request.Subject) ? null : request.Subject.Trim(), Channel = string.IsNullOrWhiteSpace(request.Channel) ? null : request.Channel.Trim(), + Direction = string.IsNullOrWhiteSpace(request.Direction) ? null : request.Direction.Trim(), ExternalMessageId = string.IsNullOrWhiteSpace(request.ExternalMessageId) ? null : request.ExternalMessageId.Trim(), ExternalThreadId = string.IsNullOrWhiteSpace(request.ExternalThreadId) ? null : request.ExternalThreadId.Trim(), ExternalFrom = string.IsNullOrWhiteSpace(request.ExternalFrom) ? null : request.ExternalFrom.Trim(), ExternalTo = string.IsNullOrWhiteSpace(request.ExternalTo) ? null : request.ExternalTo.Trim(), + ExternalLabelsJson = string.IsNullOrWhiteSpace(request.ExternalLabelsJson) ? null : request.ExternalLabelsJson.Trim(), + AttachmentMetadataJson = string.IsNullOrWhiteSpace(request.AttachmentMetadataJson) ? null : request.AttachmentMetadataJson.Trim(), Content = request.Content, Date = request.Date ?? DateTime.Now, }; diff --git a/JobTrackerApi/Controllers/GmailController.cs b/JobTrackerApi/Controllers/GmailController.cs index c9f89f1..bc01aac 100644 --- a/JobTrackerApi/Controllers/GmailController.cs +++ b/JobTrackerApi/Controllers/GmailController.cs @@ -1,4 +1,5 @@ using System.Security.Claims; +using System.Text.Json; using JobTrackerApi.Data; using JobTrackerApi.Models; using JobTrackerApi.Services; @@ -414,12 +415,22 @@ public sealed class GmailController : ControllerBase { JobApplicationId = job.Id, From = isMe ? "Me" : "Company", + Direction = isMe ? "outbound" : "inbound", Subject = string.IsNullOrWhiteSpace(detail.Subject) ? null : detail.Subject.Trim(), Channel = "Email", ExternalMessageId = detail.Id, ExternalThreadId = string.IsNullOrWhiteSpace(detail.ThreadId) ? null : detail.ThreadId.Trim(), ExternalFrom = string.IsNullOrWhiteSpace(detail.From) ? null : detail.From.Trim(), ExternalTo = string.IsNullOrWhiteSpace(detail.To) ? null : detail.To.Trim(), + ExternalLabelsJson = detail.Labels.Count == 0 ? null : JsonSerializer.Serialize(detail.Labels), + AttachmentMetadataJson = detail.Attachments.Count == 0 ? null : JsonSerializer.Serialize(detail.Attachments.Select(attachment => new CorrespondenceAttachmentMetadata + { + FileName = attachment.FileName, + MimeType = attachment.MimeType, + SizeBytes = attachment.SizeBytes, + GmailAttachmentId = attachment.GmailAttachmentId, + Inline = attachment.Inline, + })), Content = string.IsNullOrWhiteSpace(detail.BodyText) ? detail.Snippet : detail.BodyText, Date = messageDate, }; diff --git a/JobTrackerApi/Program.cs b/JobTrackerApi/Program.cs index 74c08ad..974aa87 100644 --- a/JobTrackerApi/Program.cs +++ b/JobTrackerApi/Program.cs @@ -753,6 +753,9 @@ CREATE TABLE IF NOT EXISTS "TailoredCvDrafts" ( EnsureColumn(conn, "Correspondences", "ExternalThreadId", "ALTER TABLE Correspondences ADD COLUMN ExternalThreadId TEXT NULL;"); EnsureColumn(conn, "Correspondences", "ExternalFrom", "ALTER TABLE Correspondences ADD COLUMN ExternalFrom TEXT NULL;"); EnsureColumn(conn, "Correspondences", "ExternalTo", "ALTER TABLE Correspondences ADD COLUMN ExternalTo TEXT NULL;"); + EnsureColumn(conn, "Correspondences", "Direction", "ALTER TABLE Correspondences ADD COLUMN Direction TEXT NULL;"); + EnsureColumn(conn, "Correspondences", "ExternalLabelsJson", "ALTER TABLE Correspondences ADD COLUMN ExternalLabelsJson TEXT NULL;"); + EnsureColumn(conn, "Correspondences", "AttachmentMetadataJson", "ALTER TABLE Correspondences ADD COLUMN AttachmentMetadataJson TEXT NULL;"); } // Record the migration as applied. @@ -780,6 +783,9 @@ CREATE TABLE IF NOT EXISTS "TailoredCvDrafts" ( EnsureColumn(conn, "Correspondences", "ExternalThreadId", "ALTER TABLE Correspondences ADD COLUMN ExternalThreadId TEXT NULL;"); EnsureColumn(conn, "Correspondences", "ExternalFrom", "ALTER TABLE Correspondences ADD COLUMN ExternalFrom TEXT NULL;"); EnsureColumn(conn, "Correspondences", "ExternalTo", "ALTER TABLE Correspondences ADD COLUMN ExternalTo TEXT NULL;"); + EnsureColumn(conn, "Correspondences", "Direction", "ALTER TABLE Correspondences ADD COLUMN Direction TEXT NULL;"); + EnsureColumn(conn, "Correspondences", "ExternalLabelsJson", "ALTER TABLE Correspondences ADD COLUMN ExternalLabelsJson TEXT NULL;"); + EnsureColumn(conn, "Correspondences", "AttachmentMetadataJson", "ALTER TABLE Correspondences ADD COLUMN AttachmentMetadataJson TEXT NULL;"); EnsureColumn(conn, "Attachments", "Purpose", "ALTER TABLE Attachments ADD COLUMN Purpose TEXT NULL;"); EnsureColumn(conn, "Attachments", "UseForAi", "ALTER TABLE Attachments ADD COLUMN UseForAi INTEGER NOT NULL DEFAULT 1;"); @@ -879,6 +885,9 @@ CREATE TABLE IF NOT EXISTS "TailoredCvDrafts" ( EnsureMySqlColumn(conn, "Correspondences", "ExternalThreadId", "ALTER TABLE `Correspondences` ADD COLUMN `ExternalThreadId` longtext NULL;"); EnsureMySqlColumn(conn, "Correspondences", "ExternalFrom", "ALTER TABLE `Correspondences` ADD COLUMN `ExternalFrom` longtext NULL;"); EnsureMySqlColumn(conn, "Correspondences", "ExternalTo", "ALTER TABLE `Correspondences` ADD COLUMN `ExternalTo` longtext NULL;"); + EnsureMySqlColumn(conn, "Correspondences", "Direction", "ALTER TABLE `Correspondences` ADD COLUMN `Direction` varchar(100) NULL;"); + EnsureMySqlColumn(conn, "Correspondences", "ExternalLabelsJson", "ALTER TABLE `Correspondences` ADD COLUMN `ExternalLabelsJson` longtext NULL;"); + EnsureMySqlColumn(conn, "Correspondences", "AttachmentMetadataJson", "ALTER TABLE `Correspondences` ADD COLUMN `AttachmentMetadataJson` longtext NULL;"); EnsureMySqlColumn(conn, "Attachments", "Purpose", "ALTER TABLE `Attachments` ADD COLUMN `Purpose` varchar(100) NULL;"); EnsureMySqlColumn(conn, "Attachments", "UseForAi", "ALTER TABLE `Attachments` ADD COLUMN `UseForAi` tinyint(1) NOT NULL DEFAULT 1;"); EnsureMySqlColumn(conn, "AspNetUsers", "ProfileCvText", "ALTER TABLE `AspNetUsers` ADD COLUMN `ProfileCvText` longtext NULL;"); diff --git a/JobTrackerApi/Services/GmailOAuthService.cs b/JobTrackerApi/Services/GmailOAuthService.cs index 01dae06..c6ef0a4 100644 --- a/JobTrackerApi/Services/GmailOAuthService.cs +++ b/JobTrackerApi/Services/GmailOAuthService.cs @@ -27,7 +27,8 @@ public interface IGmailOAuthService public sealed record GmailOAuthExchangeResult(string GmailAddress); public sealed record GmailMessageSummary(string Id, string ThreadId, string Subject, string From, string To, DateTimeOffset? Date, string Snippet); public sealed record GmailQueryMatchedMessage(GmailMessageSummary Message, IReadOnlyList MatchedQueries); -public sealed record GmailMessageDetail(string Id, string ThreadId, string Subject, string From, string To, DateTimeOffset? Date, string Snippet, string BodyText, string? BodyHtml); +public sealed record GmailMessageAttachment(string? FileName, string? MimeType, long? SizeBytes, string? GmailAttachmentId, bool Inline); +public sealed record GmailMessageDetail(string Id, string ThreadId, string Subject, string From, string To, DateTimeOffset? Date, string Snippet, string BodyText, string? BodyHtml, IReadOnlyList Labels, IReadOnlyList Attachments); internal sealed class GmailTokenResponse { @@ -321,9 +322,13 @@ public sealed class GmailOAuthService : IGmailOAuthService var threadId = root.TryGetProperty("threadId", out var threadEl) ? threadEl.GetString() ?? "" : ""; var snippet = root.TryGetProperty("snippet", out var snippetEl) ? snippetEl.GetString() ?? "" : ""; + var labels = root.TryGetProperty("labelIds", out var labelIdsEl) && labelIdsEl.ValueKind == JsonValueKind.Array + ? labelIdsEl.EnumerateArray().Where(x => x.ValueKind == JsonValueKind.String).Select(x => x.GetString()).Where(x => !string.IsNullOrWhiteSpace(x)).Cast().ToList() + : new List(); var payload = root.GetProperty("payload"); var headers = ReadHeaders(payload); + var attachments = ReadAttachments(payload); var bodyText = ExtractBody(payload, "text/plain"); var bodyHtml = ExtractBody(payload, "text/html"); if (string.IsNullOrWhiteSpace(bodyText) && !string.IsNullOrWhiteSpace(bodyHtml)) @@ -345,7 +350,9 @@ public sealed class GmailOAuthService : IGmailOAuthService headers.TryGetValue("date", out var dateRaw) && DateTimeOffset.TryParse(dateRaw, out var parsedDate) ? parsedDate : null, snippet, bodyText.Trim(), - string.IsNullOrWhiteSpace(bodyHtml) ? null : bodyHtml + string.IsNullOrWhiteSpace(bodyHtml) ? null : bodyHtml, + labels, + attachments ); } catch (Exception ex) @@ -538,6 +545,48 @@ public sealed class GmailOAuthService : IGmailOAuthService return result; } + private static List ReadAttachments(JsonElement payload) + { + var results = new List(); + ReadAttachmentsRecursive(payload, results); + return results; + } + + private static void ReadAttachmentsRecursive(JsonElement payload, List results) + { + var body = payload.TryGetProperty("body", out var bodyEl) && bodyEl.ValueKind == JsonValueKind.Object + ? bodyEl + : default; + var gmailAttachmentId = body.ValueKind == JsonValueKind.Object && body.TryGetProperty("attachmentId", out var attachmentIdEl) && attachmentIdEl.ValueKind == JsonValueKind.String + ? attachmentIdEl.GetString() + : null; + var filename = payload.TryGetProperty("filename", out var filenameEl) ? filenameEl.GetString() : null; + var mimeType = payload.TryGetProperty("mimeType", out var mimeTypeEl) ? mimeTypeEl.GetString() : null; + var sizeBytes = body.ValueKind == JsonValueKind.Object && body.TryGetProperty("size", out var sizeEl) && sizeEl.ValueKind == JsonValueKind.Number + ? sizeEl.GetInt64() + : (long?)null; + var disposition = payload.TryGetProperty("headers", out var headersEl) && headersEl.ValueKind == JsonValueKind.Array + ? headersEl.EnumerateArray() + .Where(h => h.TryGetProperty("name", out var n) && string.Equals(n.GetString(), "Content-Disposition", StringComparison.OrdinalIgnoreCase)) + .Select(h => h.TryGetProperty("value", out var v) ? v.GetString() : null) + .FirstOrDefault() + : null; + var isInline = !string.IsNullOrWhiteSpace(disposition) && disposition.Contains("inline", StringComparison.OrdinalIgnoreCase); + + if (!string.IsNullOrWhiteSpace(gmailAttachmentId) || !string.IsNullOrWhiteSpace(filename)) + { + results.Add(new GmailMessageAttachment(filename, mimeType, sizeBytes, gmailAttachmentId, isInline)); + } + + if (payload.TryGetProperty("parts", out var partsEl) && partsEl.ValueKind == JsonValueKind.Array) + { + foreach (var part in partsEl.EnumerateArray()) + { + ReadAttachmentsRecursive(part, results); + } + } + } + private static string ExtractBody(JsonElement payload, string mimeType) { if (payload.TryGetProperty("mimeType", out var mimeTypeEl) && diff --git a/Models/Correspondence.cs b/Models/Correspondence.cs index 002be6d..7ddc96d 100644 --- a/Models/Correspondence.cs +++ b/Models/Correspondence.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Text.Json.Serialization; namespace JobTrackerApi.Models @@ -11,13 +12,35 @@ namespace JobTrackerApi.Models [JsonIgnore] public JobApplication JobApplication { get; set; } = null!; public string From { get; set; } = ""; // "Me" or "Company" + public string? Direction { get; set; } // inbound, outbound, internal, unknown public string? Subject { get; set; } public string? Channel { get; set; } // e.g. Email, Call, Note public string? ExternalMessageId { get; set; } public string? ExternalThreadId { get; set; } public string? ExternalFrom { get; set; } public string? ExternalTo { get; set; } + public string? ExternalLabelsJson { get; set; } + public string? AttachmentMetadataJson { get; set; } public string Content { get; set; } = ""; public DateTime Date { get; set; } = DateTime.Now; + + [JsonIgnore] + public IReadOnlyList ExternalLabels => string.IsNullOrWhiteSpace(ExternalLabelsJson) + ? Array.Empty() + : (System.Text.Json.JsonSerializer.Deserialize>(ExternalLabelsJson) ?? new List()); + + [JsonIgnore] + public IReadOnlyList AttachmentMetadata => string.IsNullOrWhiteSpace(AttachmentMetadataJson) + ? Array.Empty() + : (System.Text.Json.JsonSerializer.Deserialize>(AttachmentMetadataJson) ?? new List()); + } + + public sealed class CorrespondenceAttachmentMetadata + { + public string? FileName { get; set; } + public string? MimeType { get; set; } + public long? SizeBytes { get; set; } + public string? GmailAttachmentId { get; set; } + public bool Inline { get; set; } } } diff --git a/job-tracker-ui/src/components/Correspondence.tsx b/job-tracker-ui/src/components/Correspondence.tsx index 6ea5066..c51c795 100644 --- a/job-tracker-ui/src/components/Correspondence.tsx +++ b/job-tracker-ui/src/components/Correspondence.tsx @@ -382,11 +382,13 @@ export default function Correspondence({ jobId, job }: { jobId: number; job: Job {m.subject ? {m.subject} : null} {m.content} - {(m.externalThreadId || m.externalFrom || m.externalTo) ? ( + {(m.externalThreadId || m.externalFrom || m.externalTo || m.externalLabelsJson || m.attachmentMetadataJson) ? ( {m.externalThreadId ? : null} {m.externalFrom ? : null} {m.externalTo ? : null} + {m.externalLabelsJson ? : null} + {m.attachmentMetadataJson ? : null} ) : null} diff --git a/job-tracker-ui/src/types.ts b/job-tracker-ui/src/types.ts index 1166b1b..9c4dad8 100644 --- a/job-tracker-ui/src/types.ts +++ b/job-tracker-ui/src/types.ts @@ -200,10 +200,19 @@ export interface SaveApplicationDraftsRequest { recruiterMessageDraft?: string | null; } +export interface CorrespondenceAttachmentMetadata { + fileName?: string | null; + mimeType?: string | null; + sizeBytes?: number | null; + gmailAttachmentId?: string | null; + inline?: boolean; +} + export interface CorrespondenceMessage { id: number; jobApplicationId: number; from: string; + direction?: string | null; content: string; subject?: string; channel?: string; @@ -212,6 +221,8 @@ export interface CorrespondenceMessage { externalThreadId?: string | null; externalFrom?: string | null; externalTo?: string | null; + externalLabelsJson?: string | null; + attachmentMetadataJson?: string | null; } export interface GmailJobMatchReason { From fd3527776a9ae492f81abfffe2a5a8a3df4ad858 Mon Sep 17 00:00:00 2001 From: cesnimda Date: Wed, 1 Apr 2026 16:38:03 +0200 Subject: [PATCH 3/3] docs: update gmail workstream progress --- SMART_GMAIL_PROGRESS.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/SMART_GMAIL_PROGRESS.md b/SMART_GMAIL_PROGRESS.md index 80eb902..694f208 100644 --- a/SMART_GMAIL_PROGRESS.md +++ b/SMART_GMAIL_PROGRESS.md @@ -26,6 +26,10 @@ - focused backend/frontend Gmail tests added and passing - Phase 2 extension seam scaffolded with a no-op enrichment service - Phase 1/2 design doc added at `docs/gmail-correspondence-phase1.md` +- Started M007 ingestion/storage work: + - imported correspondence now stores direction, Gmail labels JSON, and attachment metadata JSON + - Gmail message detail extraction now reads labels and attachment metadata from Gmail payloads + - focused Gmail backend/frontend tests pass against the richer import contract ## Next tasks 1. Implement M006/S01/T01: refactor Gmail connection foundation and sync-state model.