diff --git a/JobTrackerApi.Tests/AccountDataExportTests.cs b/JobTrackerApi.Tests/AccountDataExportTests.cs new file mode 100644 index 0000000..9276825 --- /dev/null +++ b/JobTrackerApi.Tests/AccountDataExportTests.cs @@ -0,0 +1,210 @@ +using System.IO.Compression; +using System.Security.Claims; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using JobTrackerApi.Controllers; +using JobTrackerApi.Data; +using JobTrackerApi.Models; +using JobTrackerApi.Services; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Identity; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.RateLimiting; +using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Hosting; +using Moq; +using Xunit; + +namespace JobTrackerApi.Tests; + +public sealed class AccountDataExportTests +{ + [Fact] + public async Task Readable_zip_is_complete_checksummed_owner_isolated_and_redacted() + { + var root = Path.Combine(Path.GetTempPath(), $"jobtracker-account-export-{Guid.NewGuid():N}"); + Directory.CreateDirectory(root); + try + { + var (db, paths, service) = CreateService(root); + await using (db) + { + var owner = new ApplicationUser + { + Id = "user-1", + UserName = "owner@example.test", + Email = "owner@example.test", + DisplayName = "Synthetic Owner", + PasswordHash = "PASSWORD_HASH_MUST_NOT_EXPORT", + SecurityStamp = "SECURITY_STAMP_MUST_NOT_EXPORT", + TotpSecretEncrypted = "TOTP_SECRET_MUST_NOT_EXPORT", + GoogleEmail = "owner@gmail.test", + AiEnabled = true, + }; + var other = new ApplicationUser { Id = "user-2", UserName = "other@example.test", Email = "other@example.test", DisplayName = "OTHER_TENANT_PRIVATE" }; + var role = new IdentityRole("Pro") { Id = "role-pro", NormalizedName = "PRO" }; + db.Users.AddRange(owner, other); + db.Roles.Add(role); + db.UserRoles.Add(new IdentityUserRole { UserId = owner.Id, RoleId = role.Id }); + var company = new Company { OwnerUserId = owner.Id, Name = "Owner Company" }; + db.Companies.Add(company); + await db.SaveChangesAsync(); + var application = new JobApplication { OwnerUserId = owner.Id, CompanyId = company.Id, JobTitle = "Owner Role", Notes = "Readable application note" }; + var otherApplication = new JobApplication { OwnerUserId = other.Id, CompanyId = company.Id, JobTitle = "OTHER_TENANT_PRIVATE" }; + db.JobApplications.AddRange(application, otherApplication); + await db.SaveChangesAsync(); + + var attachmentPath = Path.Combine(paths.AttachmentsRoot, application.Id.ToString(), "evidence.txt"); + Directory.CreateDirectory(Path.GetDirectoryName(attachmentPath)!); + await File.WriteAllTextAsync(attachmentPath, "owned attachment bytes"); + db.Attachments.Add(new Attachment { JobApplicationId = application.Id, FileName = "evidence.txt", FilePath = attachmentPath, FileType = "text/plain", FileSize = new FileInfo(attachmentPath).Length }); + + var artifactRoot = Path.Combine(paths.CvArtifactsRoot, AppPaths.GetOwnerStorageKey(owner.Id)); + Directory.CreateDirectory(artifactRoot); + var artifactPath = Path.Combine(artifactRoot, "resume.txt"); + await File.WriteAllTextAsync(artifactPath, "owned CV artifact bytes"); + db.CvUploadArtifacts.Add(new CvUploadArtifact { OwnerUserId = owner.Id, OriginalFileName = "resume.txt", StoredFileName = "resume.txt", MimeType = "text/plain", ByteSize = new FileInfo(artifactPath).Length, Sha256 = "synthetic", StoragePath = artifactPath }); + db.GmailConnections.Add(new GmailConnection { OwnerUserId = owner.Id, GmailAddress = owner.GoogleEmail!, Scope = "mail.read", EncryptedRefreshToken = "GMAIL_REFRESH_SECRET_MUST_NOT_EXPORT", EncryptedAccessToken = "GMAIL_ACCESS_SECRET_MUST_NOT_EXPORT" }); + db.ImapConnections.Add(new ImapConnection { OwnerUserId = owner.Id, Host = "mail.example.test", Username = "owner", EncryptedPassword = "IMAP_SECRET_MUST_NOT_EXPORT" }); + db.UserOperations.Add(new UserOperation { Id = Guid.NewGuid(), OwnerUserId = owner.Id, TaskType = "synthetic", Status = OperationStatuses.Running, LeaseToken = "LEASE_SECRET_MUST_NOT_EXPORT", CreatedAtUtc = DateTime.UtcNow, AvailableAtUtc = DateTime.UtcNow }); + db.TwoFactorRecoveryCodes.Add(new TwoFactorRecoveryCode { UserId = owner.Id, CodeHash = "RECOVERY_HASH_MUST_NOT_EXPORT", CreatedAtUtc = DateTimeOffset.UtcNow }); + await db.SaveChangesAsync(); + + var avatar = await AvatarStorage.StoreAsync(paths.DataRoot, owner.Id, [1, 2, 3, 4], "image/png", CancellationToken.None); + owner.AvatarImageDataUrl = avatar; + db.Users.Update(owner); + var generatedCvRoot = Path.Combine(paths.GetOwnerCvExportsRoot(owner.Id), "20260815"); + Directory.CreateDirectory(generatedCvRoot); + await File.WriteAllTextAsync(Path.Combine(generatedCvRoot, "generated.pdf"), "synthetic generated PDF"); + var dailyRoot = paths.GetOwnerDailyExportsRoot(null, owner.Id); + Directory.CreateDirectory(dailyRoot); + await File.WriteAllTextAsync(Path.Combine(dailyRoot, "daily_export_20260815.json"), "{\"owner\":true}"); + await db.SaveChangesAsync(); + + var artifact = await service.CreateAsync(owner.Id, CancellationToken.None); + Assert.True(File.Exists(artifact.StoragePath)); + using var archive = ZipFile.OpenRead(artifact.StoragePath); + Assert.NotNull(archive.GetEntry("manifest.json")); + Assert.NotNull(archive.GetEntry("README.txt")); + Assert.NotNull(archive.GetEntry("data/account.json")); + Assert.NotNull(archive.GetEntry("data/applications.json")); + Assert.NotNull(archive.GetEntry($"files/attachments/{application.Id}/evidence.txt")); + Assert.NotNull(archive.GetEntry("files/cv-artifacts/1/resume.txt")); + Assert.Contains(archive.Entries, item => item.FullName.StartsWith("files/avatar/", StringComparison.Ordinal)); + Assert.NotNull(archive.GetEntry("files/generated-cv/20260815/generated.pdf")); + Assert.NotNull(archive.GetEntry("files/daily-exports/daily_export_20260815.json")); + + var readable = string.Join('\n', archive.Entries + .Where(item => item.FullName.EndsWith(".json", StringComparison.OrdinalIgnoreCase) || item.FullName.EndsWith(".txt", StringComparison.OrdinalIgnoreCase)) + .Select(ReadText)); + Assert.Contains("Synthetic Owner", readable); + Assert.Contains("Readable application note", readable); + Assert.DoesNotContain("OTHER_TENANT_PRIVATE", readable); + Assert.DoesNotContain("PASSWORD_HASH_MUST_NOT_EXPORT", readable); + Assert.DoesNotContain("SECURITY_STAMP_MUST_NOT_EXPORT", readable); + Assert.DoesNotContain("TOTP_SECRET_MUST_NOT_EXPORT", readable); + Assert.DoesNotContain("GMAIL_REFRESH_SECRET_MUST_NOT_EXPORT", readable); + Assert.DoesNotContain("GMAIL_ACCESS_SECRET_MUST_NOT_EXPORT", readable); + Assert.DoesNotContain("IMAP_SECRET_MUST_NOT_EXPORT", readable); + Assert.DoesNotContain("LEASE_SECRET_MUST_NOT_EXPORT", readable); + Assert.DoesNotContain("RECOVERY_HASH_MUST_NOT_EXPORT", readable); + + using var manifestDocument = JsonDocument.Parse(ReadText(archive.GetEntry("manifest.json")!)); + var manifestEntries = manifestDocument.RootElement.GetProperty("entries").EnumerateArray().ToList(); + Assert.Equal(archive.Entries.Count - 1, manifestEntries.Count); + foreach (var manifestEntry in manifestEntries) + { + var zipEntry = archive.GetEntry(manifestEntry.GetProperty("path").GetString()!); + Assert.NotNull(zipEntry); + using var source = zipEntry!.Open(); + var hash = Convert.ToHexString(SHA256.HashData(source)).ToLowerInvariant(); + Assert.Equal(manifestEntry.GetProperty("sha256").GetString(), hash); + Assert.Equal(manifestEntry.GetProperty("bytes").GetInt64(), zipEntry.Length); + } + } + } + finally + { + SqliteConnection.ClearAllPools(); + if (Directory.Exists(root)) Directory.Delete(root, recursive: true); + } + } + + [Fact] + public async Task Account_export_requires_a_session_created_within_fifteen_minutes() + { + var root = Path.Combine(Path.GetTempPath(), $"jobtracker-account-export-auth-{Guid.NewGuid():N}"); + Directory.CreateDirectory(root); + try + { + var (db, _, service) = CreateService(root); + await using (db) + { + var user = new ApplicationUser { Id = "user-1", UserName = "owner@example.test", Email = "owner@example.test" }; + db.Users.Add(user); + db.UserSessions.AddRange( + new UserSession { Id = "fresh", UserId = user.Id, CreatedAtUtc = DateTimeOffset.UtcNow, LastSeenAtUtc = DateTimeOffset.UtcNow, ExpiresAtUtc = DateTimeOffset.UtcNow.AddHours(1) }, + new UserSession { Id = "stale", UserId = user.Id, CreatedAtUtc = DateTimeOffset.UtcNow.AddMinutes(-16), LastSeenAtUtc = DateTimeOffset.UtcNow, ExpiresAtUtc = DateTimeOffset.UtcNow.AddHours(1) }); + await db.SaveChangesAsync(); + + var stale = Controller(db, service, user.Id, "stale"); + var staleResult = Assert.IsType(await stale.ExportAccount(CancellationToken.None)); + Assert.Equal(StatusCodes.Status403Forbidden, staleResult.StatusCode); + + var fresh = Controller(db, service, user.Id, "fresh"); + var file = Assert.IsType(await fresh.ExportAccount(CancellationToken.None)); + Assert.Equal("application/zip", file.ContentType); + Assert.EndsWith(".zip", file.FileDownloadName); + await file.FileStream.DisposeAsync(); + } + } + finally + { + SqliteConnection.ClearAllPools(); + if (Directory.Exists(root)) Directory.Delete(root, recursive: true); + } + } + + [Fact] + public void Account_export_is_rate_limited() + { + var attribute = typeof(ExportController).GetMethod(nameof(ExportController.ExportAccount))!.GetCustomAttributes(typeof(EnableRateLimitingAttribute), inherit: true).Cast().Single(); + Assert.Equal("account-data", attribute.PolicyName); + } + + private static ExportController Controller(JobTrackerApi.Data.JobTrackerContext db, AccountDataExportService service, string userId, string sessionId) + { + var controller = new ExportController(db, service) { ControllerContext = new ControllerContext { HttpContext = new DefaultHttpContext() } }; + controller.ControllerContext.HttpContext.User = new ClaimsPrincipal(new ClaimsIdentity( + [ + new Claim(ClaimTypes.NameIdentifier, userId), + new Claim("sid", sessionId), + ], "local")); + return controller; + } + + private static (JobTrackerApi.Data.JobTrackerContext Db, AppPaths Paths, AccountDataExportService Service) CreateService(string root) + { + var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary { ["Data:Root"] = root }).Build(); + var environment = new Mock(); + environment.SetupGet(item => item.ContentRootPath).Returns(root); + var paths = new AppPaths(configuration, environment.Object); + var currentUser = new Mock(); + currentUser.SetupGet(item => item.UserId).Returns("user-1"); + var options = new DbContextOptionsBuilder() + .UseSqlite($"Data Source={Path.Combine(root, "account-export-tests.db")}") + .Options; + var db = new JobTrackerContext(options, currentUser.Object); + db.Database.EnsureCreated(); + return (db, paths, new AccountDataExportService(db, paths, new AttachmentStorage(paths), TimeProvider.System)); + } + + private static string ReadText(ZipArchiveEntry entry) + { + using var reader = new StreamReader(entry.Open(), Encoding.UTF8); + return reader.ReadToEnd(); + } +} diff --git a/JobTrackerApi/Controllers/ExportController.cs b/JobTrackerApi/Controllers/ExportController.cs index cf91796..58a56b1 100644 --- a/JobTrackerApi/Controllers/ExportController.cs +++ b/JobTrackerApi/Controllers/ExportController.cs @@ -1,8 +1,11 @@ using System.Text; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.RateLimiting; using Microsoft.EntityFrameworkCore; using JobTrackerApi.Data; +using JobTrackerApi.Services; +using System.Security.Claims; namespace JobTrackerApi.Controllers { @@ -12,10 +15,43 @@ namespace JobTrackerApi.Controllers public class ExportController : ControllerBase { private readonly JobTrackerContext _db; + private readonly AccountDataExportService? _accountExport; + private readonly TimeProvider _timeProvider; - public ExportController(JobTrackerContext db) + public ExportController(JobTrackerContext db, AccountDataExportService? accountExport = null, TimeProvider? timeProvider = null) { _db = db; + _accountExport = accountExport; + _timeProvider = timeProvider ?? TimeProvider.System; + } + + [HttpPost("account")] + [EnableRateLimiting("account-data")] + public async Task ExportAccount(CancellationToken cancellationToken) + { + var ownerUserId = LocalAuthIdentity.GetRequiredUserId(User); + if (ownerUserId is null) return Unauthorized(); + if (_accountExport is null) throw new InvalidOperationException("Account export is not configured."); + + var sessionId = User.FindFirstValue("sid"); + var recentCutoff = _timeProvider.GetUtcNow().AddMinutes(-15); + var currentSession = string.IsNullOrWhiteSpace(sessionId) + ? null + : await _db.UserSessions.IgnoreQueryFilters().AsNoTracking() + .FirstOrDefaultAsync(item => item.Id == sessionId && item.UserId == ownerUserId, cancellationToken); + var recentlyAuthenticated = currentSession is { RevokedAtUtc: null } && currentSession.CreatedAtUtc >= recentCutoff; + if (!recentlyAuthenticated) + { + return StatusCode(StatusCodes.Status403Forbidden, new ProblemDetails + { + Title = "Recent sign-in required", + Detail = "Sign in again before downloading a complete account export.", + }); + } + + var artifact = await _accountExport.CreateAsync(ownerUserId, cancellationToken); + var stream = new FileStream(artifact.StoragePath, FileMode.Open, FileAccess.Read, FileShare.Read, 64 * 1024, FileOptions.Asynchronous | FileOptions.DeleteOnClose); + return File(stream, "application/zip", artifact.DownloadFileName); } [HttpGet("jobs")] diff --git a/JobTrackerApi/Program.cs b/JobTrackerApi/Program.cs index e9c8109..1024ac9 100644 --- a/JobTrackerApi/Program.cs +++ b/JobTrackerApi/Program.cs @@ -46,6 +46,7 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(TimeProvider.System); builder.Services.AddScoped(); builder.Services.AddScoped(); +builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddSingleton(); @@ -452,6 +453,17 @@ builder.Services.AddRateLimiter(options => QueueLimit = 0, })); + options.AddPolicy("account-data", context => + RateLimitPartition.GetFixedWindowLimiter( + partitionKey: $"account-data:{context.User.FindFirst(System.Security.Claims.ClaimTypes.NameIdentifier)?.Value ?? context.User.FindFirst("sub")?.Value ?? context.Connection.RemoteIpAddress?.ToString() ?? "unknown"}", + factory: _ => new FixedWindowRateLimiterOptions + { + PermitLimit = 2, + Window = TimeSpan.FromHours(1), + QueueProcessingOrder = QueueProcessingOrder.OldestFirst, + QueueLimit = 0, + })); + // Brute-forcing a 6-digit TOTP code (1e6 space) is far more feasible than a password, so // this gets a tighter window than auth-login. options.AddPolicy("auth-2fa-challenge", context => diff --git a/JobTrackerApi/Services/AccountDataExportService.cs b/JobTrackerApi/Services/AccountDataExportService.cs new file mode 100644 index 0000000..680c28e --- /dev/null +++ b/JobTrackerApi/Services/AccountDataExportService.cs @@ -0,0 +1,442 @@ +using System.IO.Compression; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using JobTrackerApi.Data; +using JobTrackerApi.Models; +using Microsoft.EntityFrameworkCore; + +namespace JobTrackerApi.Services; + +public sealed record AccountDataExportArtifact(string StoragePath, string DownloadFileName); + +public sealed class AccountDataExportService( + JobTrackerContext db, + AppPaths paths, + IAttachmentStorage attachmentStorage, + TimeProvider timeProvider) +{ + private const string SchemaVersion = "jobtracker.user-export.v1"; + private static readonly JsonSerializerOptions JsonOptions = new() + { + WriteIndented = true, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + ReferenceHandler = ReferenceHandler.IgnoreCycles, + }; + + public async Task CreateAsync(string ownerUserId, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(ownerUserId); + var user = await db.Users.AsNoTracking().FirstOrDefaultAsync(item => item.Id == ownerUserId, cancellationToken) + ?? throw new InvalidOperationException("The account no longer exists."); + var generatedAt = timeProvider.GetUtcNow(); + var ownerKey = AppPaths.GetOwnerStorageKey(ownerUserId); + var outputRoot = Path.Combine(paths.DataRoot, "AccountExports", ownerKey); + Directory.CreateDirectory(outputRoot); + var outputPath = Path.Combine(outputRoot, $"{Guid.NewGuid():N}.zip"); + var warnings = new List(); + var entries = new List(); + + try + { + await using (var output = new FileStream(outputPath, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None, 64 * 1024, FileOptions.Asynchronous | FileOptions.SequentialScan)) + using (var archive = new ZipArchive(output, ZipArchiveMode.Create, leaveOpen: false, Encoding.UTF8)) + { + async Task AddJsonAsync(string entryName, object value, int itemCount) + { + var bytes = JsonSerializer.SerializeToUtf8Bytes(value, value.GetType(), JsonOptions); + await AddBytesAsync(archive, entries, entryName, bytes, "data", itemCount, cancellationToken); + } + + var roles = await (from userRole in db.UserRoles.AsNoTracking() + join role in db.Roles.AsNoTracking() on userRole.RoleId equals role.Id + where userRole.UserId == ownerUserId + orderby role.Name + select role.Name!).ToListAsync(cancellationToken); + var claims = await db.UserClaims.AsNoTracking() + .Where(item => item.UserId == ownerUserId) + .OrderBy(item => item.ClaimType) + .Select(item => new { item.ClaimType, item.ClaimValue }) + .ToListAsync(cancellationToken); + var logins = await db.UserLogins.AsNoTracking() + .Where(item => item.UserId == ownerUserId) + .OrderBy(item => item.LoginProvider) + .Select(item => new { item.LoginProvider, item.ProviderDisplayName }) + .ToListAsync(cancellationToken); + + await AddJsonAsync("data/account.json", new + { + user.Id, + user.UserName, + user.Email, + user.EmailConfirmed, + user.PhoneNumber, + user.PhoneNumberConfirmed, + user.FirstName, + user.LastName, + user.DisplayName, + user.PendingEmail, + user.PendingEmailRequestedAtUtc, + user.ProfileCvText, + user.ProfileCvStructureJson, + user.CurrentCvUploadArtifactId, + user.CurrentCvExtractionRunId, + user.CurrentCvProfileVersion, + user.GoogleSubject, + user.GoogleEmail, + user.GoogleLinkedAt, + user.MicrosoftSubject, + user.MicrosoftTenantId, + user.MicrosoftObjectId, + user.MicrosoftEmail, + user.MicrosoftLinkedAt, + user.TotpEnabledAtUtc, + user.StripeCustomerId, + user.StripeSubscriptionId, + user.StripeSubscriptionStatus, + user.StripeLastEventCreatedUtc, + user.AiEnabled, + user.ExternalAiProcessingAllowed, + Roles = roles, + Claims = claims, + ExternalLogins = logins, + }, 1); + + var companies = await db.Companies.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken); + var opportunities = await db.Jobs.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken); + var applications = await db.JobApplications.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken); + var applicationIds = applications.Select(item => item.Id).ToList(); + await AddJsonAsync("data/companies.json", companies, companies.Count); + await AddJsonAsync("data/opportunities.json", opportunities, opportunities.Count); + await AddJsonAsync("data/applications.json", applications, applications.Count); + + var correspondence = await db.Correspondences.IgnoreQueryFilters().AsNoTracking().Where(item => applicationIds.Contains(item.JobApplicationId)).OrderBy(item => item.Id).ToListAsync(cancellationToken); + var events = await db.JobEvents.IgnoreQueryFilters().AsNoTracking().Where(item => applicationIds.Contains(item.JobApplicationId)).OrderBy(item => item.Id).ToListAsync(cancellationToken); + var attachments = await db.Attachments.IgnoreQueryFilters().AsNoTracking().Where(item => applicationIds.Contains(item.JobApplicationId)).OrderBy(item => item.Id).ToListAsync(cancellationToken); + await AddJsonAsync("data/correspondence.json", correspondence, correspondence.Count); + await AddJsonAsync("data/job-events.json", events, events.Count); + await AddJsonAsync("data/attachments.json", attachments.Select(item => new + { + item.Id, + item.JobApplicationId, + item.FileName, + item.UploadDate, + item.FileType, + item.FileSize, + item.Purpose, + item.UseForAi, + }).ToList(), attachments.Count); + + var careerProfile = await db.CareerProfiles.IgnoreQueryFilters().AsNoTracking().FirstOrDefaultAsync(item => item.OwnerUserId == ownerUserId, cancellationToken); + var careerProfileId = careerProfile?.Id; + var careerVersions = await db.CareerProfileVersions.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Version).ToListAsync(cancellationToken); + var careerExperiences = await db.CareerExperiences.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.SortOrder).ToListAsync(cancellationToken); + var careerEducation = await db.CareerEducations.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.SortOrder).ToListAsync(cancellationToken); + var careerSkills = await db.CareerSkills.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.SortOrder).ToListAsync(cancellationToken); + var careerProjects = await db.CareerProjects.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.SortOrder).ToListAsync(cancellationToken); + var careerCertifications = await db.CareerCertifications.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.SortOrder).ToListAsync(cancellationToken); + var careerLanguages = await db.CareerLanguages.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.SortOrder).ToListAsync(cancellationToken); + await AddJsonAsync("data/career.json", new + { + Profile = careerProfile, + Versions = careerVersions, + Experiences = careerExperiences, + Education = careerEducation, + Skills = careerSkills, + Projects = careerProjects, + Certifications = careerCertifications, + Languages = careerLanguages, + }, (careerProfileId is null ? 0 : 1) + careerVersions.Count + careerExperiences.Count + careerEducation.Count + careerSkills.Count + careerProjects.Count + careerCertifications.Count + careerLanguages.Count); + + var variants = await db.CvVariants.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken); + var variantVersions = await db.CvVariantVersions.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken); + var artifacts = await db.CvUploadArtifacts.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken); + var extractionRuns = await db.CvExtractionRuns.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken); + await AddJsonAsync("data/cv.json", new + { + Variants = variants, + VariantVersions = variantVersions, + UploadArtifacts = artifacts.Select(item => new + { + item.Id, + item.OriginalFileName, + item.StoredFileName, + item.MimeType, + item.ByteSize, + item.Sha256, + item.UploadedAtUtc, + }), + ExtractionRuns = extractionRuns, + }, variants.Count + variantVersions.Count + artifacts.Count + extractionRuns.Count); + + var tailoredDrafts = await db.TailoredCvDrafts.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken); + var interviewNotes = await db.InterviewPrepNotes.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken); + var aiNotes = await db.AiWorkspaceNotes.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken); + var aiInteractions = await db.AiInteractions.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken); + var checklist = await db.ApplicationChecklistItems.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken); + var coverLetters = await db.CoverLetterVersions.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken); + var interviewItems = await db.InterviewPrepItems.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken); + var emailDrafts = await db.EmailDrafts.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.CreatedAtUtc).ToListAsync(cancellationToken); + var emailAttempts = await db.EmailSendAttempts.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.CreatedAtUtc) + .Select(item => new EmailSendAttemptExport(item.Id, item.JobApplicationId, item.Provider, item.ClientRequestId, item.Status, item.ProviderMessageId, item.FailureCategory, item.CreatedAtUtc, item.StartedAtUtc, item.CompletedAtUtc)) + .ToListAsync(cancellationToken); + await AddJsonAsync("data/application-workspace.json", new + { + TailoredCvDrafts = tailoredDrafts, + InterviewPrepNotes = interviewNotes, + AiWorkspaceNotes = aiNotes, + AiInteractions = aiInteractions, + ChecklistItems = checklist, + CoverLetterVersions = coverLetters, + InterviewPrepItems = interviewItems, + EmailDrafts = emailDrafts, + EmailSendAttempts = emailAttempts, + }, tailoredDrafts.Count + interviewNotes.Count + aiNotes.Count + aiInteractions.Count + checklist.Count + coverLetters.Count + interviewItems.Count + emailDrafts.Count + emailAttempts.Count); + + var operations = await db.UserOperations.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.CreatedAtUtc) + .Select(item => new + { + item.Id, + item.TaskType, + item.Status, + item.Priority, + item.EntitlementDecision, + item.PrivacyPolicy, + item.SubjectType, + item.SubjectId, + item.Provider, + item.Model, + item.AttemptCount, + item.MaxAttempts, + item.CreatedAtUtc, + item.AvailableAtUtc, + item.StartedAtUtc, + item.CompletedAtUtc, + item.DeadlineAtUtc, + item.CancellationRequestedAtUtc, + item.ProgressStage, + item.ProgressPercent, + item.FailureCategory, + item.FailureMessage, + item.ResultReference, + }).ToListAsync(cancellationToken); + var notifications = await db.UserNotifications.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.CreatedAtUtc).ToListAsync(cancellationToken); + await AddJsonAsync("data/operations.json", new { Operations = operations, Notifications = notifications }, operations.Count + notifications.Count); + + var gmail = await db.GmailConnections.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId) + .Select(item => new { item.GmailAddress, item.Scope, item.ConnectedAt, item.LastSyncedAt, item.LastSyncAttemptedAt, item.LastSyncSucceededAt, item.LastSyncMode, item.LastSyncSource, item.LastSyncStatus, item.LastSyncError }).ToListAsync(cancellationToken); + var microsoft = await db.MicrosoftGraphConnections.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId) + .Select(item => new { item.MailAddress, item.Scope, item.ConnectedAt, item.LastSyncedAt, item.LastSyncAttemptedAt, item.LastSyncSucceededAt, item.LastSyncMode, item.LastSyncSource, item.LastSyncStatus, item.LastSyncError }).ToListAsync(cancellationToken); + var imap = await db.ImapConnections.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId) + .Select(item => new { item.Host, item.Port, item.UseSsl, item.Username, item.ConnectedAt, item.LastSyncedAt, item.LastSyncAttemptedAt, item.LastSyncSucceededAt, item.LastSyncMode, item.LastSyncSource, item.LastSyncStatus, item.LastSyncError }).ToListAsync(cancellationToken); + var reviewDecisions = await db.GmailReviewDecisions.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken); + var userRules = await db.UserRuleSettings.IgnoreQueryFilters().AsNoTracking().FirstOrDefaultAsync(item => item.OwnerUserId == ownerUserId, cancellationToken); + await AddJsonAsync("data/settings-and-providers.json", new + { + UserRules = userRules, + GmailConnections = gmail, + MicrosoftConnections = microsoft, + ImapConnections = imap, + GmailReviewDecisions = reviewDecisions, + }, (userRules is null ? 0 : 1) + gmail.Count + microsoft.Count + imap.Count + reviewDecisions.Count); + + var sessions = await db.UserSessions.IgnoreQueryFilters().AsNoTracking().Where(item => item.UserId == ownerUserId).OrderBy(item => item.Id) + .Select(item => new { item.DeviceLabel, item.CreatedAtUtc, item.LastSeenAtUtc, item.ExpiresAtUtc, item.RevokedAtUtc }).ToListAsync(cancellationToken); + var trustedDevices = await db.TrustedDevices.IgnoreQueryFilters().AsNoTracking().Where(item => item.UserId == ownerUserId).OrderBy(item => item.Id) + .Select(item => new { item.DeviceLabel, item.CreatedAtUtc, item.LastSeenAtUtc, item.ExpiresAtUtc }).ToListAsync(cancellationToken); + var recoveryCodeCount = await db.TwoFactorRecoveryCodes.IgnoreQueryFilters().AsNoTracking().CountAsync(item => item.UserId == ownerUserId, cancellationToken); + await AddJsonAsync("data/security-metadata.json", new + { + TwoFactorEnabled = !string.IsNullOrWhiteSpace(user.TotpSecretEncrypted), + RecoveryCodeCount = recoveryCodeCount, + Sessions = sessions, + TrustedDevices = trustedDevices, + }, sessions.Count + trustedDevices.Count + recoveryCodeCount); + + foreach (var attachment in attachments) + { + await AddOwnedFileAsync(archive, entries, warnings, attachment.FilePath, $"files/attachments/{attachment.Id}/{SafeSegment(attachment.FileName)}", "attachment", attachmentStorage.IsManagedPath, cancellationToken); + } + foreach (var artifact in artifacts) + { + await AddOwnedFileAsync(archive, entries, warnings, artifact.StoragePath, $"files/cv-artifacts/{artifact.Id}/{SafeSegment(artifact.OriginalFileName)}", "cv-artifact", path => IsManagedPath(paths.CvArtifactsRoot, path), cancellationToken); + } + + await AddAvatarAsync(archive, entries, warnings, paths, ownerUserId, user.AvatarImageDataUrl, cancellationToken); + await AddDirectoryAsync(archive, entries, warnings, paths.GetOwnerCvExportsRoot(ownerUserId), "files/generated-cv", "generated-cv", cancellationToken); + await AddDirectoryAsync(archive, entries, warnings, paths.GetOwnerDailyExportsRoot(null, ownerUserId), "files/daily-exports", "daily-export", cancellationToken); + + const string readme = """ + Jobjakt readable account export + + This ZIP contains user-readable JSON plus owned files. manifest.json lists every entry, + byte size, SHA-256 checksum and any unavailable file warnings. + + Excluded secrets: password/security/concurrency hashes, TOTP secrets, recovery-code and + trusted-device token hashes, session IDs, OAuth access/refresh tokens, IMAP passwords, + operation lease tokens, email payload hashes, data-protection keys and global settings. + + External providers may retain mailbox, billing or model-service data under their own + policies. Application logs and immutable backups are not edited by this export. Their + exact retention remains an operator/legal decision documented in BLOCKERS.md. + """; + await AddBytesAsync(archive, entries, "README.txt", Encoding.UTF8.GetBytes(readme), "documentation", 1, cancellationToken); + + var manifest = new + { + SchemaVersion, + GeneratedAtUtc = generatedAt, + AccountId = ownerUserId, + EntryCount = entries.Count, + Entries = entries, + Warnings = warnings, + Exclusions = new[] + { + "authentication secrets and hashes", + "provider credentials and access/refresh tokens", + "global application settings and data-protection keys", + "other users' data", + "application logs, immutable backups and external-provider retained data", + "unattributable legacy generated files", + }, + }; + var manifestBytes = JsonSerializer.SerializeToUtf8Bytes(manifest, JsonOptions); + var manifestEntry = archive.CreateEntry("manifest.json", CompressionLevel.Optimal); + await using var manifestStream = manifestEntry.Open(); + await manifestStream.WriteAsync(manifestBytes, cancellationToken); + } + + return new AccountDataExportArtifact(outputPath, $"jobjakt-account-export-{generatedAt:yyyyMMdd-HHmmss}.zip"); + } + catch + { + try { File.Delete(outputPath); } catch { } + throw; + } + } + + private static async Task AddOwnedFileAsync( + ZipArchive archive, + ICollection entries, + ICollection warnings, + string sourcePath, + string entryName, + string category, + Func isManaged, + CancellationToken cancellationToken) + { + if (!isManaged(sourcePath)) + { + warnings.Add($"Excluded unsafe {category} path for {entryName}."); + return; + } + if (!File.Exists(sourcePath)) + { + warnings.Add($"Owned {category} file was unavailable: {entryName}."); + return; + } + await AddFileAsync(archive, entries, sourcePath, entryName, category, cancellationToken); + } + + private static async Task AddDirectoryAsync(ZipArchive archive, ICollection entries, ICollection warnings, string root, string entryRoot, string category, CancellationToken cancellationToken) + { + if (!Directory.Exists(root)) return; + foreach (var path in Directory.EnumerateFiles(root, "*", new EnumerationOptions + { + RecurseSubdirectories = true, + IgnoreInaccessible = true, + AttributesToSkip = FileAttributes.ReparsePoint, + })) + { + var relative = Path.GetRelativePath(root, path); + if (relative.StartsWith("..", StringComparison.Ordinal)) + { + warnings.Add($"Excluded unsafe {category} path."); + continue; + } + var entryName = $"{entryRoot}/{string.Join('/', relative.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar).Select(SafeSegment))}"; + await AddFileAsync(archive, entries, path, entryName, category, cancellationToken); + } + } + + private static async Task AddAvatarAsync(ZipArchive archive, ICollection entries, ICollection warnings, AppPaths paths, string ownerUserId, string? storedAvatar, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(storedAvatar)) return; + if (storedAvatar.StartsWith("file:", StringComparison.Ordinal)) + { + var path = storedAvatar[5..]; + var root = Path.Combine(paths.DataRoot, "Avatars", AppPaths.GetOwnerStorageKey(ownerUserId)); + await AddOwnedFileAsync(archive, entries, warnings, path, $"files/avatar/{SafeSegment(Path.GetFileName(path))}", "avatar", candidate => IsManagedPath(root, candidate), cancellationToken); + return; + } + if (storedAvatar.StartsWith("data:", StringComparison.OrdinalIgnoreCase)) + { + var comma = storedAvatar.IndexOf(','); + if (comma > 0 && storedAvatar[..comma].Contains(";base64", StringComparison.OrdinalIgnoreCase)) + { + try + { + var bytes = Convert.FromBase64String(storedAvatar[(comma + 1)..]); + await AddBytesAsync(archive, entries, "files/avatar/avatar", bytes, "avatar", 1, cancellationToken); + return; + } + catch (FormatException) { } + } + } + warnings.Add("The profile avatar was stored in an unsupported format and could not be included."); + } + + private static async Task AddFileAsync(ZipArchive archive, ICollection entries, string sourcePath, string entryName, string category, CancellationToken cancellationToken) + { + await using var source = new FileStream(sourcePath, FileMode.Open, FileAccess.Read, FileShare.Read, 64 * 1024, FileOptions.Asynchronous | FileOptions.SequentialScan); + var checksum = Convert.ToHexString(await SHA256.HashDataAsync(source, cancellationToken)).ToLowerInvariant(); + source.Position = 0; + var entry = archive.CreateEntry(entryName.Replace('\\', '/'), CompressionLevel.Optimal); + await using var target = entry.Open(); + await source.CopyToAsync(target, cancellationToken); + entries.Add(new ManifestEntry(entry.FullName, source.Length, checksum, category, 1)); + } + + private static async Task AddBytesAsync(ZipArchive archive, ICollection entries, string entryName, byte[] bytes, string category, int itemCount, CancellationToken cancellationToken) + { + var entry = archive.CreateEntry(entryName, CompressionLevel.Optimal); + await using var target = entry.Open(); + await target.WriteAsync(bytes, cancellationToken); + entries.Add(new ManifestEntry(entry.FullName, bytes.LongLength, Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant(), category, itemCount)); + } + + private static bool IsManagedPath(string root, string path) + { + if (string.IsNullOrWhiteSpace(path)) return false; + try + { + var comparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; + var fullRoot = Path.GetFullPath(root).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + var fullPath = Path.GetFullPath(path); + if (!fullPath.StartsWith(fullRoot + Path.DirectorySeparatorChar, comparison)) return false; + var current = Path.GetDirectoryName(fullPath); + while (!string.IsNullOrWhiteSpace(current) && !string.Equals(current, fullRoot, comparison)) + { + if (Directory.Exists(current) && (File.GetAttributes(current) & FileAttributes.ReparsePoint) != 0) return false; + current = Path.GetDirectoryName(current); + } + return !File.Exists(fullPath) || (File.GetAttributes(fullPath) & FileAttributes.ReparsePoint) == 0; + } + catch + { + return false; + } + } + + private static string SafeSegment(string? value) + { + var candidate = Path.GetFileName(value ?? string.Empty).Trim(); + foreach (var invalid in Path.GetInvalidFileNameChars()) candidate = candidate.Replace(invalid, '_'); + if (candidate.Length > 120) candidate = candidate[..120]; + return string.IsNullOrWhiteSpace(candidate) || candidate is "." or ".." ? "file" : candidate; + } + + private sealed record ManifestEntry(string Path, long Bytes, string Sha256, string Category, int ItemCount); +} diff --git a/docs/audits/verification-log.md b/docs/audits/verification-log.md index ae7d59e..266aa7c 100644 --- a/docs/audits/verification-log.md +++ b/docs/audits/verification-log.md @@ -205,3 +205,4 @@ Output was reduced to filenames and commit counts. The token artifact appears un | V-171 | Public/product claim inventory; plan/notice/usage focused Jest; current entitlement/billing-policy backend slice; full frontend; optimized build; full Playwright | Repository root / `job-tracker-ui` | Replace contradictory plan/commercial claims and prove respectful Free/Pro promotion without changing billing or enforcement | PASS — exactly Free/Pro comes from one catalogue; focused frontend 7 suites/30 tests, policy/billing backend 30/30, full frontend 57 suites/232 tests, build and Playwright 8/8. Chromium proves retired claims absent, explicit Light/Dark, 375/768/1440 no overflow and keyboard Free/Pro actions | Local synthetic account only; no Stripe checkout/webhook/portal, native AT or production. Commercial terms remain intentionally absent until configured Checkout; existing Jest/GSI warnings remain | PRODUCT-001 repository/browser scope verified; configured billing lifecycle and production remain | | V-172 | Action-matrix reconciliation; full backend/frontend/sidecar/build/Compose/preflight gates; expanded authenticated and anonymous Playwright | Repository root / `job-tracker-ui` / `tools/summarizer` | Complete VER-001 local release regression without promoting mocked/provider/external checks | PASS — backend 647/647, frontend 57 suites/232 tests, sidecar 22/22, production build, Compose config, API-down/wrong-base/malformed-JSON preflight and Chromium 9/9. Browser covers admin deployment identity/normal-user absence, notifications, honest Free, jobs, Career/CV, Kanban, responsive themes and public PDF | No external provider, private data, native AT or production mutation. Optional Compose variables remain unset; existing Jest/GSI/SWIG warnings remain. Windows CRLF materialization was normalized for shell execution; indexed LF policy was already correct | VER-001 verified locally; remote CI/provider/native-AT/production cells remain | | V-173 | Owner-path trace; focused CV/export/controller/background tests; full backend; build and diff hygiene | Repository root | Establish attributable generated-file ownership before SEC-009 export/deletion | PASS — CV PDFs use opaque owner/date/UUID storage while preserving download names; daily exports use opaque owner directories and atomic writes; legacy/new retention paths are covered. Focused 77/77 and backend 647/647 | No existing file moved or deleted. Legacy shared-date generated files are intentionally not guessed. No migration, production path or private data used | SEC-009 owner-scoped generated-output prerequisite verified locally | +| V-174 | Real-SQLite two-owner export fixture; manifest/checksum/file/redaction assertions; recent-session/rate-limit API tests; focused/full frontend and backend; optimized build; Chromium ZIP response | Repository root / `job-tracker-ui` | Deliver a complete user-readable export without exposing secrets or another tenant | PASS — focused backend/API 11/11, backend 650/650, frontend focused 4/4 and full 58 suites/234 tests, builds pass. Every manifest checksum/size matches; owned attachment/CV/avatar/generated/daily files are included; secret and other-owner sentinels are absent; Chromium receives HTTP 200 `application/zip` with `PK` signature | Synthetic data/files only; no production/private/provider access. Export reports backups/logs/external retention instead of claiming erasure. One initial InMemory-only test missed SQLite DateTimeOffset translation; the test moved to real SQLite and the query boundary was corrected | SEC-009 readable export verified locally; deletion/retention/restore remain | diff --git a/docs/verification/application-action-matrix.md b/docs/verification/application-action-matrix.md index 0f6d343..4310ab9 100644 --- a/docs/verification/application-action-matrix.md +++ b/docs/verification/application-action-matrix.md @@ -50,6 +50,7 @@ This is the rolling action-level evidence index. `PASS (automated/runtime)` is n | Public CV | responsive A4/multi-page framing without inner or outer overflow | PASS (component) | PASS — 375px Chromium | NOT RUN | `accessibility-evidence.md`, V-170 | | Public plans | exactly Free/Pro; no invented tier, price, interval, trial or unlimited claim | PASS (catalogue + landing components) | PASS — Light/Dark at 375/768/1440 | NOT RUN | `product-001-honest-plans.md`, V-171 | | Public plans | Free registration and Pro sign-in-to-Settings actions | PASS (component) | PASS — keyboard activation | NOT RUN | `product-001-honest-plans.md`, V-171 | +| Account export | recent-sign-in/rate-limit gate; complete redacted owner ZIP, files, warnings and checksums | PASS (real SQLite + components) | PASS — fresh synthetic Free download response and success state | NOT RUN with production data | `sec-009-account-lifecycle.md`, V-174 | | Pro promotion | benefit-specific locked notice, preserved-data copy and session dismissal | PASS (components) | NOT RUN on every contextual surface | NOT RUN | `product-001-honest-plans.md`, V-171 | | Billing presentation | checkout/portal only when server status permits; unconfigured deployment disclosed | PASS (components + policy slice) | NOT RUN with configured Stripe | NOT RUN | `product-001-honest-plans.md`, V-171 | | Admin safety | self/other Admin demotion confirmation and final-admin API protection | PASS (controller + components) | NOT RUN | NOT RUN | V-161 | diff --git a/docs/verification/sec-009-account-lifecycle.md b/docs/verification/sec-009-account-lifecycle.md index afe6610..27197f8 100644 --- a/docs/verification/sec-009-account-lifecycle.md +++ b/docs/verification/sec-009-account-lifecycle.md @@ -2,7 +2,7 @@ Updated: 2026-08-15 -Status: `IN PROGRESS`. Generated-output ownership is now explicit. Readable export and the disabled deletion lifecycle remain to be implemented. +Status: `IN PROGRESS`. Generated-output ownership and the readable export are implemented. The disabled deletion lifecycle remains to be implemented. ## Owner inventory boundary @@ -18,19 +18,33 @@ The authoritative inventory must include Identity-safe account/profile fields an No existing generated file is moved or guessed. Legacy shared-date outputs stay a separately reviewed rollout concern because they cannot be attributed safely. +## Checkpoint 2 — complete readable export + +- Authenticated `POST /api/export/account` requires the current local session to have been created within the last 15 minutes and is limited to two requests per user per hour. +- One service owns both the authoritative row inventory and file inventory. It queries with explicit owner predicates and `IgnoreQueryFilters`, so soft-deleted applications remain portable and an absent/requestless tenant scope cannot silently empty the export. +- The ZIP contains readable account, company, opportunity, application, correspondence, event, attachment, Career, CV, workspace, AI operation, notification, settings/provider and security-metadata JSON categories. +- Owned attachment, CV upload, avatar, generated-CV and daily-export bytes are included only after managed-root/reparse-point checks. Missing or unsafe files produce manifest warnings rather than cross-root reads. +- `manifest.json` records schema version, generated time, category/item counts, byte sizes and SHA-256 checksums for every included entry. `README.txt` explains formats, exclusions and retention limits. +- Password/security/concurrency hashes, TOTP secrets, recovery/trusted-device hashes, session IDs, provider access/refresh tokens, IMAP passwords, operation leases, email payload hashes, global settings and data-protection keys are never serialized. +- The Settings Backup tab presents the readable export separately from the application-key-encrypted operational backup and explains recent sign-in without weakening the API rule. +- Temporary ZIPs live under an opaque owner root and are opened with delete-on-close when returned by the controller. + ## Verification -- Focused CV/export/controller/background tests: 77/77. -- Full backend: 647/647. +- Owner-storage focused CV/export/controller/background tests: 77/77. +- Readable-export focused backend/API tests: 11/11, including real SQLite, two-owner isolation, file inclusion, every checksum and secret-redaction sentinels. +- Full backend: 650/650. +- Frontend export/Settings tests: 4/4; full frontend 58 suites/234 tests. - Backend build: pass, zero warnings/errors. +- Optimized frontend build/TypeScript: pass. +- Chromium: fresh Free account receives a real ZIP response with a `PK` signature and readable-export success state. - `git diff --check`: pass aside from line-ending notices. ## Remaining repository work -1. Implement one owner inventory used by both readable ZIP export and deletion. -2. Add manifest/checksums/warnings and include safely owned binary files without exposing storage paths. -3. Add the additive deletion state/request/file schema and disabled coordinator. -4. Add pending-account authentication/mutation gates, session/queue cancellation, provider cleanup and idempotent file quarantine/database purge. -5. Add separate tombstone storage/replay and settings/admin UX while keeping production activation disabled. +1. Reuse the completed owner inventory in the deletion coordinator. +2. Add the additive deletion state/request/file schema and disabled coordinator. +3. Add pending-account authentication/mutation gates, session/queue cancellation, provider cleanup and idempotent file quarantine/database purge. +4. Add separate tombstone storage/replay and settings/admin UX while keeping production activation disabled. Production retention, legal hold and restored-backup decisions remain recorded in `BLOCKERS.md`. diff --git a/docs/work-programmes/decisions.md b/docs/work-programmes/decisions.md index 5b94786..a08b6da 100644 --- a/docs/work-programmes/decisions.md +++ b/docs/work-programmes/decisions.md @@ -739,3 +739,13 @@ - **Consequences:** all new generated outputs have an exact owner boundary and collision-resistant storage path. Existing legacy files age out under retention and remain excluded from user deletion unless independently attributed. - **User approval required:** No; additive storage hardening within the requested account lifecycle, with no existing data mutation. - **Reversible:** Restore shared date paths for future files. Existing owner-scoped files remain valid retention artifacts and must not be bulk-moved or deleted during rollback. + +## DEC-075 — Separate readable portability export from operational backup + +- **Date:** 2026-08-15 +- **Decision:** Add a recent-authenticated, per-user-rate-limited readable ZIP export alongside—not in place of—the existing application-key-encrypted backup. Build one explicit redacted owner inventory with checksum manifest and reuse it as the future deletion inventory boundary. +- **Reason/evidence:** the encrypted backup is useful for application recovery but unreadable without the deployment key and omits many owned categories. A portability export must be readable, complete, tenant-isolated and secret-free; it must not be mislabeled as backup erasure. +- **Alternatives considered:** expose the encrypted backup as user export; serialize the whole EF graph; reuse the jobs-only export; include provider/token/security rows verbatim. These are unreadable, partial, cycle-prone or credential disclosures. +- **Consequences:** users can download JSON and owned files with independently verifiable SHA-256 checksums. Missing/legacy/external/backup categories are disclosed truthfully. The service becomes the authoritative inventory seam for deletion without coupling export to deletion activation. +- **User approval required:** No; this is the requested repository-side data lifecycle, using synthetic tests and no production data. +- **Reversible:** Remove the endpoint/UI and service. Existing downloaded ZIPs remain user-owned files; no stored schema or data changed. diff --git a/docs/work-programmes/master-progress.md b/docs/work-programmes/master-progress.md index 1ede4ec..310cb86 100644 --- a/docs/work-programmes/master-progress.md +++ b/docs/work-programmes/master-progress.md @@ -3,7 +3,7 @@ Updated: 2026-08-15 - **Overall programme status:** Active. Eight packages are locally verified; twenty-four are implemented with verification incomplete; SEC-009 is in progress. The prioritized admin-only version indicator and every immediate repository/browser item are implemented on the release branch; remote and production verification remain. -- **Current work package:** `SEC-009` — complete readable export and account deletion lifecycle (`IN PROGRESS`). Generated CV/daily outputs now use opaque owner directories; proceed with the shared owner inventory/readable ZIP, then the disabled deletion lifecycle while retention/restore policy blocks production activation. +- **Current work package:** `SEC-009` — complete readable export and account deletion lifecycle (`IN PROGRESS`). Owner-scoped generated paths and the real-SQLite-verified readable ZIP are complete; proceed with the disabled deletion lifecycle while retention/restore policy blocks production activation. - **Completed work packages:** None are `DONE`; all repository security/AI packages still have applicable browser, provider and/or production gates. - **Locally verified work:** SEC-001, SEC-002, SEC-003, SEC-005A, CORE-001, PROD-002, DEP-001 and VER-001 (`VERIFIED LOCALLY`). - **Implemented, verification incomplete:** SEC-004, SEC-005B, SEC-008, CORE-002, BG-001, OPS-001A/B/C, POL-001/002, AI-001/002/003/004, UX-001/002/003, QA-001, CAREER-001/002, MAIL-001, JOBS-001/002 and PRODUCT-001 (`IMPLEMENTED — NOT VERIFIED`). Their safe repository/browser scope is implemented; production/native-device/provider gates remain where recorded. @@ -12,7 +12,7 @@ Updated: 2026-08-15 - **Deferred work:** None. Conditional multi-replica coordination, model deletion, realtime operation delivery and unrelated production changes remain outside current packages. - **Immediate order:** SEC-009 owner inventory/export first, then its disabled deletion lifecycle. The eight-item immediate queue is complete locally: admin version (`a6cffe0`), Career persistence (`f0b9b22`), CV contrast (`3b86ea2`), JOBS-002 (`deed948`), accessibility (`a7c2549`), PRODUCT-001 (`a25c31b`), VER-001 and tracking reconciliation. External-only work remains skipped, not allowed to stall repository progress. - **Status counts:** 8 `VERIFIED LOCALLY`; 24 `IMPLEMENTED — NOT VERIFIED`; 1 `IN PROGRESS`; 1 `NOT STARTED`; 5 `BLOCKED`; 0 `DONE`; 0 `DEFERRED`. -- **Test status:** backend 647/647; frontend 57/57 suites and 232/232 tests; AI sidecar 22/22; optimized production build/TypeScript; Docker Compose config; safe-failure deployment preflight; and Playwright 9/9 pass. Chromium covers the admin deployment badge/normal-user absence, notification popover, honest Free behavior, explicit light/dark at 375/768/1440, application workspace, Career/CV, discovery, Kanban and public CV/PDF. npm audit 0 evidence remains current because the lockfile did not change. Historical JT-019 and Jest force-exit/open-handle behavior remain recorded. +- **Test status:** backend 650/650; frontend 58/58 suites and 234/234 tests; AI sidecar 22/22; optimized production build/TypeScript; Docker Compose config; safe-failure deployment preflight; and the preceding Playwright 9/9 pass. A fresh Chromium Free account now also downloads the real readable ZIP response. npm audit 0 evidence remains current because the lockfile did not change. Historical JT-019 and Jest force-exit/open-handle behavior remain recorded. - **Deployment status:** Gitea pull-request run 609 passed the complete CI job in 4m20s. Deploy was intentionally skipped because the workflow deploys only a `push` to `main`; live remains unchanged. No merge/deployment was performed directly, no production migrations were run and the AI operation worker remains disabled by default. - **Production status:** Unchanged and unverified. No provider/model call, model pull, external request or paid API occurred. - **Known regressions:** None found by automated/local browser checks. Jest still needs `--forceExit` and reports its existing open-handle notice. Email-provider/send tests are fake/local only; real delivery is not claimed. Current MAIL browser evidence is 1280×720 only because the browser surface could not resize or perform native Tab traversal. Interrupted attempts are aged after 15 minutes and notified without retry; the five-minute scan is unmeasured on a large ledger. Direct clean EF-only SQLite migration still hits the pre-existing historical blank-chain defect before later migrations; normal startup owns reconciliation. Cross-feature monthly AI usage accounting remains a rollout gap. diff --git a/docs/work-programmes/master-work-plan.md b/docs/work-programmes/master-work-plan.md index 1b3de7f..ffa1443 100644 --- a/docs/work-programmes/master-work-plan.md +++ b/docs/work-programmes/master-work-plan.md @@ -270,9 +270,9 @@ This queue records the highest-value work that can proceed without production cr - **Required production verification:** backup retention/tombstone rehearsal before self-service enablement. - **Status:** `IN PROGRESS`. - **Blocker:** legal/operator retention and production restore decisions block activation, not the repository-side disabled/dark launch. -- **Evidence:** audit JT-009 inventory/design; `docs/verification/sec-009-account-lifecycle.md`; V-173 owner-scoped generated-output checkpoint, focused 77/77 and backend 647/647. +- **Evidence:** audit JT-009 inventory/design; `docs/verification/sec-009-account-lifecycle.md`; V-173/V-174. Owner-scoped generated storage and complete redacted readable ZIP pass real-SQLite two-owner, focused API/UI, full backend/frontend, build and Chromium checks. - **Commit:** none. -- **Remaining work:** owner inventory/readable ZIP export next; then additive disabled deletion coordinator, tombstone replay, UI and failure/restart verification. Production activation remains blocked by retention/restore policy. +- **Remaining work:** reuse the completed inventory for the additive disabled deletion coordinator, tombstone replay, UI and failure/restart verification. Production activation remains blocked by retention/restore policy. ### CORE-001 — Restore default SQLite/MariaDB behavior parity diff --git a/docs/work-programmes/session-handoff.md b/docs/work-programmes/session-handoff.md index de12534..7e61114 100644 --- a/docs/work-programmes/session-handoff.md +++ b/docs/work-programmes/session-handoff.md @@ -3,16 +3,16 @@ Updated: 2026-08-15 - **Exact current task:** begin SEC-009 with the owner inventory/readable export, then implement the deletion lifecycle behind a disabled production gate. -- **Last completed step:** established owner-scoped storage for every newly generated CV PDF and daily export without moving unattributable legacy files. -- **Files currently modified:** `AppPaths`, CV PDF exporter/controller callers, daily export worker, focused tests and SEC-009 evidence. -- **Commands already run:** SEC-009 storage slice 77/77; full backend 647/647; backend build; diff hygiene. The preceding VER-001 frontend/sidecar/build/Compose/preflight/Playwright 9/9 evidence remains current. +- **Last completed step:** implemented a recent-authenticated, rate-limited, redacted readable account ZIP with complete safe row/file inventory, checksums, warnings and Settings UX. +- **Files currently modified:** account export service/API/rate policy, Backup Settings UI/translations, real-SQLite/backend/frontend/browser tests and SEC-009 evidence. +- **Commands already run:** readable-export backend/API 11/11, backend 650/650, frontend focused 4/4 and full 234/234, backend/frontend builds and targeted Chromium ZIP response. The preceding VER-001 sidecar/Compose/preflight/full Playwright evidence remains current. - **Test results:** all listed local gates pass. Provider/native-AT/production cells remain explicitly partial, not run or blocked. Jest retains the documented force-exit/open-handle notice. - **Services currently running:** none on task-owned ports 3000/5202. Playwright stopped its disposable API/Next servers. Pre-existing Docker services were not changed. - **Temporary files or processes:** no task-owned process is running and the failed disposable migration database was removed. Existing synthetic browser evidence/account and startup-created local backup remain documented. No provider account, real email, private content, paid service or production service was accessed. - **Production changes currently active:** none. No deployment, migration, provider connection/sync/send or production payload occurred. - **Rollback status:** downgrade `20260810080858_AddEmailDraftClientRequestId`, then `20260810075206_AddEmailDrafts`, before reverting draft commits; then follow the existing MAIL rollback order (`ee5ef7e`, `449faeb`, `123fc55`/`e9937ac`, ledger downgrade before `653f011`). No production migration/deploy/provider grant occurred. -- **Uncommitted changes:** V-173 owner-storage code/tests/docs; no dependency, schema or production configuration change. V-172 is pushed as `0d48712`. +- **Uncommitted changes:** V-174 readable-export code/tests/docs; no dependency, schema or production configuration change. V-173 is pushed as `cdcc716`. - **Known failures:** live deployment is not verified because PR deploy is intentionally skipped and the active branch is not approved for merge. Draft export/API/UI, full thread/category actions and non-Gmail review remain; existing accounts need re-consent and IMAP stays read-only. A clean full-chain SQLite apply fails in the pre-existing JT-019 migration before the new draft migration. Browser/provider/MariaDB/production unavailable or unverified; recovery scan performance is unmeasured at large ledger scale; Jest open handles; SEC-006 parser dependency work is still separately gated; parser isolation remains SEC-007. -- **Exact next action:** commit/push V-173, then implement one redacted owner inventory and readable ZIP export with manifest/checksums/missing-file warnings. +- **Exact next action:** complete full Chromium after the export addition, commit/push V-174, then add the disabled additive deletion state and coordinator using the same owner inventory. - **Work that can continue independently:** SEC-009 repository-side owner inventory/export and disabled deletion lifecycle. UX/JOBS/PRODUCT production, MAIL provider mutations, SEC-006/007 and PROD packages retain their recorded external gates. - **Decisions still required from the user:** none for synthetic/code-inspected repository work. Any provider connection or send test, internet/package upgrades, private data, external/paid providers and production actions retain explicit approval/safety gates; SEC-009 retention/legal policy remains unresolved. diff --git a/job-tracker-ui/e2e/smoke.spec.ts b/job-tracker-ui/e2e/smoke.spec.ts index 41905e4..a9168d8 100644 --- a/job-tracker-ui/e2e/smoke.spec.ts +++ b/job-tracker-ui/e2e/smoke.spec.ts @@ -139,6 +139,16 @@ test("a Free account keeps manual work available while AI actions stay honestly await expect(page.getByRole("button", { name: "Upgrade to Pro" })).toHaveCount(0); const overflow = await page.evaluate(() => document.documentElement.scrollWidth - document.documentElement.clientWidth); expect(overflow).toBeLessThanOrEqual(1); + + await page.getByRole("tab", { name: "Backup" }).click(); + await expect(page.getByText(/requires a sign-in from the last 15 minutes/i)).toBeVisible(); + const exportResponsePromise = page.waitForResponse((response) => response.url().endsWith("/api/export/account") && response.request().method() === "POST"); + await page.getByRole("button", { name: "Download readable account export" }).click(); + const exportResponse = await exportResponsePromise; + expect(exportResponse.status()).toBe(200); + expect(exportResponse.headers()["content-type"]).toContain("application/zip"); + expect((await exportResponse.body()).subarray(0, 2).toString()).toBe("PK"); + await expect(page.getByText(/Readable account export downloaded/i)).toBeVisible(); }); test("a saved job can be created through the reviewed UI flow", async ({ page }) => { diff --git a/job-tracker-ui/src/account-data-export.test.tsx b/job-tracker-ui/src/account-data-export.test.tsx new file mode 100644 index 0000000..609b1da --- /dev/null +++ b/job-tracker-ui/src/account-data-export.test.tsx @@ -0,0 +1,54 @@ +import React from 'react'; +import '@testing-library/jest-dom'; +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; + +import BackupCard from './components/BackupCard'; +import { I18nProvider } from './i18n/I18nProvider'; +import { ToastProvider } from './toast'; +import { api } from './api'; + +jest.mock('./api', () => ({ + api: { + get: jest.fn(), + post: jest.fn(), + put: jest.fn(), + patch: jest.fn(), + delete: jest.fn(), + interceptors: { request: { use: jest.fn() }, response: { use: jest.fn() } }, + }, + getApiErrorMessage: (error: any, fallback?: string) => error?.response?.data?.detail || fallback || 'Request failed.', +})); + +const mockedApi = api as jest.Mocked; + +beforeEach(() => { + mockedApi.post.mockReset(); + Object.defineProperty(URL, 'createObjectURL', { configurable: true, value: jest.fn(() => 'blob:account-export') }); + Object.defineProperty(URL, 'revokeObjectURL', { configurable: true, value: jest.fn() }); + jest.spyOn(HTMLAnchorElement.prototype, 'click').mockImplementation(() => undefined); +}); + +afterEach(() => jest.restoreAllMocks()); + +test('downloads the readable account ZIP from the protected endpoint', async () => { + mockedApi.post.mockResolvedValue({ + data: new Blob(['zip']), + headers: { 'content-disposition': 'attachment; filename="jobjakt-account-export.zip"' }, + } as any); + render(); + + fireEvent.click(screen.getByRole('button', { name: /download readable account export/i })); + + await waitFor(() => expect(mockedApi.post).toHaveBeenCalledWith('/export/account', null, { responseType: 'blob' })); + expect(HTMLAnchorElement.prototype.click).toHaveBeenCalled(); + expect(await screen.findByText(/readable account export downloaded/i)).toBeInTheDocument(); +}); + +test('explains the recent-sign-in requirement without weakening it', async () => { + mockedApi.post.mockRejectedValue({ response: { status: 403, data: { detail: 'Sign in again before downloading a complete account export.' } } }); + render(); + + expect(screen.getByText(/requires a sign-in from the last 15 minutes/i)).toBeInTheDocument(); + fireEvent.click(screen.getByRole('button', { name: /download readable account export/i })); + expect(await screen.findByText(/sign in again before downloading/i)).toBeInTheDocument(); +}); diff --git a/job-tracker-ui/src/components/BackupCard.tsx b/job-tracker-ui/src/components/BackupCard.tsx index 304879a..30a7780 100644 --- a/job-tracker-ui/src/components/BackupCard.tsx +++ b/job-tracker-ui/src/components/BackupCard.tsx @@ -1,6 +1,6 @@ import React, { useState } from "react"; -import { Box, Button, Paper, Typography } from "@mui/material"; +import { Alert, Box, Button, Divider, Paper, Typography } from "@mui/material"; import { api, getApiErrorMessage } from "../api"; import { useToast } from "../toast"; import { useI18n } from "../i18n/I18nProvider"; @@ -9,24 +9,38 @@ export default function BackupCard() { const { toast } = useToast(); const { t } = useI18n(); const [downloading, setDownloading] = useState(false); + const [exportingAccount, setExportingAccount] = useState(false); + + const downloadBlob = (blob: Blob, contentDisposition: string | undefined, fallbackName: string) => { + const url = URL.createObjectURL(blob); + const match = /filename="?([^";]+)"?/i.exec(contentDisposition || ""); + const link = document.createElement("a"); + link.href = url; + link.download = match?.[1] ?? fallbackName; + document.body.appendChild(link); + link.click(); + link.remove(); + window.setTimeout(() => URL.revokeObjectURL(url), 5000); + }; + + const downloadAccountExport = async () => { + setExportingAccount(true); + try { + const res = await api.post("/export/account", null, { responseType: "blob" }); + downloadBlob(res.data as Blob, res.headers?.["content-disposition"] as string | undefined, `jobjakt-account-export-${new Date().toISOString().slice(0, 10)}.zip`); + toast(t("accountExportDownloaded"), "success"); + } catch (error: any) { + toast(getApiErrorMessage(error, t("accountExportFailed")), "error"); + } finally { + setExportingAccount(false); + } + }; const downloadEncrypted = async () => { setDownloading(true); try { const res = await api.post("/backup/encrypted", null, { responseType: "blob" }); - const blob: Blob = res.data; - const url = URL.createObjectURL(blob); - const cd = (res.headers?.["content-disposition"] as string) || ""; - const m = /filename="?([^";]+)"?/i.exec(cd); - const filename = m?.[1] ?? `jobtracker_backup_${new Date().toISOString().slice(0, 10)}.jtbackup`; - - const link = document.createElement("a"); - link.href = url; - link.download = filename; - document.body.appendChild(link); - link.click(); - link.remove(); - window.setTimeout(() => URL.revokeObjectURL(url), 5000); + downloadBlob(res.data as Blob, res.headers?.["content-disposition"] as string | undefined, `jobtracker_backup_${new Date().toISOString().slice(0, 10)}.jtbackup`); toast(t("backupDownloaded"), "success"); } catch (error: any) { toast(getApiErrorMessage(error, t("backupFailed")), "error"); @@ -40,11 +54,22 @@ export default function BackupCard() { {t("backupTitle")} + + {t("accountExportBody")} + + + + + {t("accountExportRecentSignIn")} + + {t("backupEncryptedTitle")} {t("backupBody")} - diff --git a/job-tracker-ui/src/i18n/translations.ts b/job-tracker-ui/src/i18n/translations.ts index 23d2261..63805ae 100644 --- a/job-tracker-ui/src/i18n/translations.ts +++ b/job-tracker-ui/src/i18n/translations.ts @@ -809,7 +809,14 @@ export const translations = { signedInAs: "Signed in as {name}.", unlinkGoogle: "Unlink Google", backupTitle: "Data safety", - backupBody: "One-click encrypted backup of your current data.", + accountExportBody: "Download a readable ZIP of your account data and owned files, with a checksum manifest and clear exclusions.", + accountExportPreparing: "Preparing account export...", + accountExportDownload: "Download readable account export", + accountExportDownloaded: "Readable account export downloaded.", + accountExportFailed: "Account export failed.", + accountExportRecentSignIn: "For your security, a complete account export requires a sign-in from the last 15 minutes. Sign out and sign in again if requested.", + backupEncryptedTitle: "Encrypted application backup", + backupBody: "Download an application-key-encrypted operational backup. Use the readable export above for personal data portability.", backupPreparing: "Preparing...", backupDownload: "Download encrypted backup", backupDownloaded: "Backup downloaded.", @@ -1962,7 +1969,14 @@ export const translations = { signedInAs: "Logget inn som {name}.", unlinkGoogle: "Koble fra Google", backupTitle: "Datasikkerhet", - backupBody: "Kryptert sikkerhetskopi av gjeldende data med ett klikk.", + accountExportBody: "Last ned en lesbar ZIP med kontodata og egne filer, med kontrollsummer og tydelige unntak.", + accountExportPreparing: "Forbereder kontoeksport...", + accountExportDownload: "Last ned lesbar kontoeksport", + accountExportDownloaded: "Lesbar kontoeksport lastet ned.", + accountExportFailed: "Kontoeksport mislyktes.", + accountExportRecentSignIn: "Av sikkerhetsgrunner krever en full kontoeksport at du logget inn de siste 15 minuttene. Logg ut og inn igjen hvis du blir bedt om det.", + backupEncryptedTitle: "Kryptert systemsikkerhetskopi", + backupBody: "Last ned en operativ sikkerhetskopi kryptert med applikasjonsnøkkelen. Bruk den lesbare eksporten over for dataportabilitet.", backupPreparing: "Forbereder...", backupDownload: "Last ned kryptert sikkerhetskopi", backupDownloaded: "Sikkerhetskopi lastet ned.",