using System.IO.Compression; using System.Security.Cryptography; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; using JobTrackerApi.Data; using JobTrackerApi.Models; using Microsoft.EntityFrameworkCore; namespace JobTrackerApi.Services; public sealed record AccountDataExportArtifact(string StoragePath, string DownloadFileName); public sealed class AccountDataExportService( JobTrackerContext db, AppPaths paths, AccountOwnedFileInventory fileInventory, TimeProvider timeProvider) { private const string SchemaVersion = "jobtracker.user-export.v1"; private static readonly JsonSerializerOptions JsonOptions = new() { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, ReferenceHandler = ReferenceHandler.IgnoreCycles, }; public async Task CreateAsync(string ownerUserId, CancellationToken cancellationToken) { ArgumentException.ThrowIfNullOrWhiteSpace(ownerUserId); var user = await db.Users.AsNoTracking().FirstOrDefaultAsync(item => item.Id == ownerUserId, cancellationToken) ?? throw new InvalidOperationException("The account no longer exists."); var generatedAt = timeProvider.GetUtcNow(); var outputRoot = paths.GetOwnerAccountExportsRoot(ownerUserId); Directory.CreateDirectory(outputRoot); var outputPath = Path.Combine(outputRoot, $"{Guid.NewGuid():N}.zip"); var warnings = new List(); var entries = new List(); try { await using (var output = new FileStream(outputPath, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None, 64 * 1024, FileOptions.Asynchronous | FileOptions.SequentialScan)) using (var archive = new ZipArchive(output, ZipArchiveMode.Create, leaveOpen: false, Encoding.UTF8)) { async Task AddJsonAsync(string entryName, object value, int itemCount) { var bytes = JsonSerializer.SerializeToUtf8Bytes(value, value.GetType(), JsonOptions); await AddBytesAsync(archive, entries, entryName, bytes, "data", itemCount, cancellationToken); } var roles = await (from userRole in db.UserRoles.AsNoTracking() join role in db.Roles.AsNoTracking() on userRole.RoleId equals role.Id where userRole.UserId == ownerUserId orderby role.Name select role.Name!).ToListAsync(cancellationToken); var claims = await db.UserClaims.AsNoTracking() .Where(item => item.UserId == ownerUserId) .OrderBy(item => item.ClaimType) .Select(item => new { item.ClaimType, item.ClaimValue }) .ToListAsync(cancellationToken); var logins = await db.UserLogins.AsNoTracking() .Where(item => item.UserId == ownerUserId) .OrderBy(item => item.LoginProvider) .Select(item => new { item.LoginProvider, item.ProviderDisplayName }) .ToListAsync(cancellationToken); await AddJsonAsync("data/account.json", new { user.Id, user.UserName, user.Email, user.EmailConfirmed, user.PhoneNumber, user.PhoneNumberConfirmed, user.FirstName, user.LastName, user.DisplayName, user.PendingEmail, user.PendingEmailRequestedAtUtc, user.ProfileCvText, user.ProfileCvStructureJson, user.CurrentCvUploadArtifactId, user.CurrentCvExtractionRunId, user.CurrentCvProfileVersion, user.GoogleSubject, user.GoogleEmail, user.GoogleLinkedAt, user.MicrosoftSubject, user.MicrosoftTenantId, user.MicrosoftObjectId, user.MicrosoftEmail, user.MicrosoftLinkedAt, user.TotpEnabledAtUtc, user.StripeCustomerId, user.StripeSubscriptionId, user.StripeSubscriptionStatus, user.StripeLastEventCreatedUtc, user.AiEnabled, user.ExternalAiProcessingAllowed, Roles = roles, Claims = claims, ExternalLogins = logins, }, 1); var companies = await db.Companies.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken); var opportunities = await db.Jobs.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken); var applications = await db.JobApplications.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken); var applicationIds = applications.Select(item => item.Id).ToList(); await AddJsonAsync("data/companies.json", companies, companies.Count); await AddJsonAsync("data/opportunities.json", opportunities, opportunities.Count); await AddJsonAsync("data/applications.json", applications, applications.Count); var correspondence = await db.Correspondences.IgnoreQueryFilters().AsNoTracking().Where(item => applicationIds.Contains(item.JobApplicationId)).OrderBy(item => item.Id).ToListAsync(cancellationToken); var events = await db.JobEvents.IgnoreQueryFilters().AsNoTracking().Where(item => applicationIds.Contains(item.JobApplicationId)).OrderBy(item => item.Id).ToListAsync(cancellationToken); var attachments = await db.Attachments.IgnoreQueryFilters().AsNoTracking().Where(item => applicationIds.Contains(item.JobApplicationId)).OrderBy(item => item.Id).ToListAsync(cancellationToken); await AddJsonAsync("data/correspondence.json", correspondence, correspondence.Count); await AddJsonAsync("data/job-events.json", events, events.Count); await AddJsonAsync("data/attachments.json", attachments.Select(item => new { item.Id, item.JobApplicationId, item.FileName, item.UploadDate, item.FileType, item.FileSize, item.Purpose, item.UseForAi, }).ToList(), attachments.Count); var careerProfile = await db.CareerProfiles.IgnoreQueryFilters().AsNoTracking().FirstOrDefaultAsync(item => item.OwnerUserId == ownerUserId, cancellationToken); var careerProfileId = careerProfile?.Id; var careerVersions = await db.CareerProfileVersions.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Version).ToListAsync(cancellationToken); var careerExperiences = await db.CareerExperiences.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.SortOrder).ToListAsync(cancellationToken); var careerEducation = await db.CareerEducations.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.SortOrder).ToListAsync(cancellationToken); var careerSkills = await db.CareerSkills.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.SortOrder).ToListAsync(cancellationToken); var careerProjects = await db.CareerProjects.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.SortOrder).ToListAsync(cancellationToken); var careerCertifications = await db.CareerCertifications.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.SortOrder).ToListAsync(cancellationToken); var careerLanguages = await db.CareerLanguages.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.SortOrder).ToListAsync(cancellationToken); await AddJsonAsync("data/career.json", new { Profile = careerProfile, Versions = careerVersions, Experiences = careerExperiences, Education = careerEducation, Skills = careerSkills, Projects = careerProjects, Certifications = careerCertifications, Languages = careerLanguages, }, (careerProfileId is null ? 0 : 1) + careerVersions.Count + careerExperiences.Count + careerEducation.Count + careerSkills.Count + careerProjects.Count + careerCertifications.Count + careerLanguages.Count); var variants = await db.CvVariants.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken); var variantVersions = await db.CvVariantVersions.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken); var artifacts = await db.CvUploadArtifacts.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken); var extractionRuns = await db.CvExtractionRuns.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken); await AddJsonAsync("data/cv.json", new { Variants = variants, VariantVersions = variantVersions, UploadArtifacts = artifacts.Select(item => new { item.Id, item.OriginalFileName, item.StoredFileName, item.MimeType, item.ByteSize, item.Sha256, item.UploadedAtUtc, }), ExtractionRuns = extractionRuns, }, variants.Count + variantVersions.Count + artifacts.Count + extractionRuns.Count); var tailoredDrafts = await db.TailoredCvDrafts.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken); var interviewNotes = await db.InterviewPrepNotes.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken); var aiNotes = await db.AiWorkspaceNotes.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken); var aiInteractions = await db.AiInteractions.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken); var checklist = await db.ApplicationChecklistItems.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken); var coverLetters = await db.CoverLetterVersions.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken); var interviewItems = await db.InterviewPrepItems.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken); var emailDrafts = await db.EmailDrafts.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.CreatedAtUtc).ToListAsync(cancellationToken); var emailAttempts = await db.EmailSendAttempts.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.CreatedAtUtc) .Select(item => new EmailSendAttemptExport(item.Id, item.JobApplicationId, item.Provider, item.ClientRequestId, item.Status, item.ProviderMessageId, item.FailureCategory, item.CreatedAtUtc, item.StartedAtUtc, item.CompletedAtUtc)) .ToListAsync(cancellationToken); await AddJsonAsync("data/application-workspace.json", new { TailoredCvDrafts = tailoredDrafts, InterviewPrepNotes = interviewNotes, AiWorkspaceNotes = aiNotes, AiInteractions = aiInteractions, ChecklistItems = checklist, CoverLetterVersions = coverLetters, InterviewPrepItems = interviewItems, EmailDrafts = emailDrafts, EmailSendAttempts = emailAttempts, }, tailoredDrafts.Count + interviewNotes.Count + aiNotes.Count + aiInteractions.Count + checklist.Count + coverLetters.Count + interviewItems.Count + emailDrafts.Count + emailAttempts.Count); var operations = await db.UserOperations.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.CreatedAtUtc) .Select(item => new { item.Id, item.TaskType, item.Status, item.Priority, item.EntitlementDecision, item.PrivacyPolicy, item.SubjectType, item.SubjectId, item.Provider, item.Model, item.AttemptCount, item.MaxAttempts, item.CreatedAtUtc, item.AvailableAtUtc, item.StartedAtUtc, item.CompletedAtUtc, item.DeadlineAtUtc, item.CancellationRequestedAtUtc, item.ProgressStage, item.ProgressPercent, item.FailureCategory, item.FailureMessage, item.ResultReference, }).ToListAsync(cancellationToken); var notifications = await db.UserNotifications.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.CreatedAtUtc).ToListAsync(cancellationToken); await AddJsonAsync("data/operations.json", new { Operations = operations, Notifications = notifications }, operations.Count + notifications.Count); var gmail = await db.GmailConnections.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId) .Select(item => new { item.GmailAddress, item.Scope, item.ConnectedAt, item.LastSyncedAt, item.LastSyncAttemptedAt, item.LastSyncSucceededAt, item.LastSyncMode, item.LastSyncSource, item.LastSyncStatus, item.LastSyncError }).ToListAsync(cancellationToken); var microsoft = await db.MicrosoftGraphConnections.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId) .Select(item => new { item.MailAddress, item.Scope, item.ConnectedAt, item.LastSyncedAt, item.LastSyncAttemptedAt, item.LastSyncSucceededAt, item.LastSyncMode, item.LastSyncSource, item.LastSyncStatus, item.LastSyncError }).ToListAsync(cancellationToken); var imap = await db.ImapConnections.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId) .Select(item => new { item.Host, item.Port, item.UseSsl, item.Username, item.ConnectedAt, item.LastSyncedAt, item.LastSyncAttemptedAt, item.LastSyncSucceededAt, item.LastSyncMode, item.LastSyncSource, item.LastSyncStatus, item.LastSyncError }).ToListAsync(cancellationToken); var reviewDecisions = await db.GmailReviewDecisions.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken); var userRules = await db.UserRuleSettings.IgnoreQueryFilters().AsNoTracking().FirstOrDefaultAsync(item => item.OwnerUserId == ownerUserId, cancellationToken); await AddJsonAsync("data/settings-and-providers.json", new { UserRules = userRules, GmailConnections = gmail, MicrosoftConnections = microsoft, ImapConnections = imap, GmailReviewDecisions = reviewDecisions, }, (userRules is null ? 0 : 1) + gmail.Count + microsoft.Count + imap.Count + reviewDecisions.Count); var sessions = await db.UserSessions.IgnoreQueryFilters().AsNoTracking().Where(item => item.UserId == ownerUserId).OrderBy(item => item.Id) .Select(item => new { item.DeviceLabel, item.CreatedAtUtc, item.LastSeenAtUtc, item.ExpiresAtUtc, item.RevokedAtUtc }).ToListAsync(cancellationToken); var trustedDevices = await db.TrustedDevices.IgnoreQueryFilters().AsNoTracking().Where(item => item.UserId == ownerUserId).OrderBy(item => item.Id) .Select(item => new { item.DeviceLabel, item.CreatedAtUtc, item.LastSeenAtUtc, item.ExpiresAtUtc }).ToListAsync(cancellationToken); var recoveryCodeCount = await db.TwoFactorRecoveryCodes.IgnoreQueryFilters().AsNoTracking().CountAsync(item => item.UserId == ownerUserId, cancellationToken); await AddJsonAsync("data/security-metadata.json", new { TwoFactorEnabled = !string.IsNullOrWhiteSpace(user.TotpSecretEncrypted), RecoveryCodeCount = recoveryCodeCount, Sessions = sessions, TrustedDevices = trustedDevices, }, sessions.Count + trustedDevices.Count + recoveryCodeCount); var ownedFiles = await fileInventory.BuildAsync(ownerUserId, cancellationToken); warnings.AddRange(ownedFiles.Warnings); foreach (var ownedFile in ownedFiles.Files) { if (ownedFile.InlineBytes is not null) await AddBytesAsync(archive, entries, ownedFile.ExportPath, ownedFile.InlineBytes, ownedFile.Category, 1, cancellationToken); else await AddFileAsync(archive, entries, ownedFile.SourcePath!, ownedFile.ExportPath, ownedFile.Category, cancellationToken); } const string readme = """ Jobjakt readable account export This ZIP contains user-readable JSON plus owned files. manifest.json lists every entry, byte size, SHA-256 checksum and any unavailable file warnings. Excluded secrets: password/security/concurrency hashes, TOTP secrets, recovery-code and trusted-device token hashes, session IDs, OAuth access/refresh tokens, IMAP passwords, operation lease tokens, email payload hashes, data-protection keys and global settings. External providers may retain mailbox, billing or model-service data under their own policies. Application logs and immutable backups are not edited by this export. Their exact retention remains an operator/legal decision documented in BLOCKERS.md. """; await AddBytesAsync(archive, entries, "README.txt", Encoding.UTF8.GetBytes(readme), "documentation", 1, cancellationToken); var manifest = new { SchemaVersion, GeneratedAtUtc = generatedAt, AccountId = ownerUserId, EntryCount = entries.Count, Entries = entries, Warnings = warnings, Exclusions = new[] { "authentication secrets and hashes", "provider credentials and access/refresh tokens", "global application settings and data-protection keys", "other users' data", "application logs, immutable backups and external-provider retained data", "unattributable legacy generated files", }, }; var manifestBytes = JsonSerializer.SerializeToUtf8Bytes(manifest, JsonOptions); var manifestEntry = archive.CreateEntry("manifest.json", CompressionLevel.Optimal); await using var manifestStream = manifestEntry.Open(); await manifestStream.WriteAsync(manifestBytes, cancellationToken); } return new AccountDataExportArtifact(outputPath, $"jobjakt-account-export-{generatedAt:yyyyMMdd-HHmmss}.zip"); } catch { try { File.Delete(outputPath); } catch { } throw; } } private static async Task AddFileAsync(ZipArchive archive, ICollection entries, string sourcePath, string entryName, string category, CancellationToken cancellationToken) { await using var source = new FileStream(sourcePath, FileMode.Open, FileAccess.Read, FileShare.Read, 64 * 1024, FileOptions.Asynchronous | FileOptions.SequentialScan); var checksum = Convert.ToHexString(await SHA256.HashDataAsync(source, cancellationToken)).ToLowerInvariant(); source.Position = 0; var entry = archive.CreateEntry(entryName.Replace('\\', '/'), CompressionLevel.Optimal); await using var target = entry.Open(); await source.CopyToAsync(target, cancellationToken); entries.Add(new ManifestEntry(entry.FullName, source.Length, checksum, category, 1)); } private static async Task AddBytesAsync(ZipArchive archive, ICollection entries, string entryName, byte[] bytes, string category, int itemCount, CancellationToken cancellationToken) { var entry = archive.CreateEntry(entryName, CompressionLevel.Optimal); await using var target = entry.Open(); await target.WriteAsync(bytes, cancellationToken); entries.Add(new ManifestEntry(entry.FullName, bytes.LongLength, Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant(), category, itemCount)); } private sealed record ManifestEntry(string Path, long Bytes, string Sha256, string Category, int ItemCount); }