jobtrackingapp/JobTrackerApi/Services/AccountDataExportService.cs

using System.IO.Compression;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using JobTrackerApi.Data;
using JobTrackerApi.Models;
using Microsoft.EntityFrameworkCore;

namespace JobTrackerApi.Services;

public sealed record AccountDataExportArtifact(string StoragePath, string DownloadFileName);

public sealed class AccountDataExportService(
    JobTrackerContext db,
    AppPaths paths,
    AccountOwnedFileInventory fileInventory,
    TimeProvider timeProvider)
{
    private const string SchemaVersion = "jobtracker.user-export.v1";
    private static readonly JsonSerializerOptions JsonOptions = new()
    {
        WriteIndented = true,
        PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
        DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
        ReferenceHandler = ReferenceHandler.IgnoreCycles,
    };

    public async Task<AccountDataExportArtifact> CreateAsync(string ownerUserId, CancellationToken cancellationToken)
    {
        ArgumentException.ThrowIfNullOrWhiteSpace(ownerUserId);
        var user = await db.Users.AsNoTracking().FirstOrDefaultAsync(item => item.Id == ownerUserId, cancellationToken)
            ?? throw new InvalidOperationException("The account no longer exists.");
        var generatedAt = timeProvider.GetUtcNow();
        var outputRoot = paths.GetOwnerAccountExportsRoot(ownerUserId);
        Directory.CreateDirectory(outputRoot);
        var outputPath = Path.Combine(outputRoot, $"{Guid.NewGuid():N}.zip");
        var warnings = new List<string>();
        var entries = new List<ManifestEntry>();

        try
        {
            await using (var output = new FileStream(outputPath, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None, 64 * 1024, FileOptions.Asynchronous | FileOptions.SequentialScan))
            using (var archive = new ZipArchive(output, ZipArchiveMode.Create, leaveOpen: false, Encoding.UTF8))
            {
                async Task AddJsonAsync(string entryName, object value, int itemCount)
                {
                    var bytes = JsonSerializer.SerializeToUtf8Bytes(value, value.GetType(), JsonOptions);
                    await AddBytesAsync(archive, entries, entryName, bytes, "data", itemCount, cancellationToken);
                }

                var roles = await (from userRole in db.UserRoles.AsNoTracking()
                                   join role in db.Roles.AsNoTracking() on userRole.RoleId equals role.Id
                                   where userRole.UserId == ownerUserId
                                   orderby role.Name
                                   select role.Name!).ToListAsync(cancellationToken);
                var claims = await db.UserClaims.AsNoTracking()
                    .Where(item => item.UserId == ownerUserId)
                    .OrderBy(item => item.ClaimType)
                    .Select(item => new { item.ClaimType, item.ClaimValue })
                    .ToListAsync(cancellationToken);
                var logins = await db.UserLogins.AsNoTracking()
                    .Where(item => item.UserId == ownerUserId)
                    .OrderBy(item => item.LoginProvider)
                    .Select(item => new { item.LoginProvider, item.ProviderDisplayName })
                    .ToListAsync(cancellationToken);

                await AddJsonAsync("data/account.json", new
                {
                    user.Id,
                    user.UserName,
                    user.Email,
                    user.EmailConfirmed,
                    user.PhoneNumber,
                    user.PhoneNumberConfirmed,
                    user.FirstName,
                    user.LastName,
                    user.DisplayName,
                    user.PendingEmail,
                    user.PendingEmailRequestedAtUtc,
                    user.ProfileCvText,
                    user.ProfileCvStructureJson,
                    user.CurrentCvUploadArtifactId,
                    user.CurrentCvExtractionRunId,
                    user.CurrentCvProfileVersion,
                    user.GoogleSubject,
                    user.GoogleEmail,
                    user.GoogleLinkedAt,
                    user.MicrosoftSubject,
                    user.MicrosoftTenantId,
                    user.MicrosoftObjectId,
                    user.MicrosoftEmail,
                    user.MicrosoftLinkedAt,
                    user.TotpEnabledAtUtc,
                    user.StripeCustomerId,
                    user.StripeSubscriptionId,
                    user.StripeSubscriptionStatus,
                    user.StripeLastEventCreatedUtc,
                    user.AiEnabled,
                    user.ExternalAiProcessingAllowed,
                    Roles = roles,
                    Claims = claims,
                    ExternalLogins = logins,
                }, 1);

                var companies = await db.Companies.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken);
                var opportunities = await db.Jobs.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken);
                var applications = await db.JobApplications.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken);
                var applicationIds = applications.Select(item => item.Id).ToList();
                await AddJsonAsync("data/companies.json", companies, companies.Count);
                await AddJsonAsync("data/opportunities.json", opportunities, opportunities.Count);
                await AddJsonAsync("data/applications.json", applications, applications.Count);

                var correspondence = await db.Correspondences.IgnoreQueryFilters().AsNoTracking().Where(item => applicationIds.Contains(item.JobApplicationId)).OrderBy(item => item.Id).ToListAsync(cancellationToken);
                var events = await db.JobEvents.IgnoreQueryFilters().AsNoTracking().Where(item => applicationIds.Contains(item.JobApplicationId)).OrderBy(item => item.Id).ToListAsync(cancellationToken);
                var attachments = await db.Attachments.IgnoreQueryFilters().AsNoTracking().Where(item => applicationIds.Contains(item.JobApplicationId)).OrderBy(item => item.Id).ToListAsync(cancellationToken);
                await AddJsonAsync("data/correspondence.json", correspondence, correspondence.Count);
                await AddJsonAsync("data/job-events.json", events, events.Count);
                await AddJsonAsync("data/attachments.json", attachments.Select(item => new
                {
                    item.Id,
                    item.JobApplicationId,
                    item.FileName,
                    item.UploadDate,
                    item.FileType,
                    item.FileSize,
                    item.Purpose,
                    item.UseForAi,
                }).ToList(), attachments.Count);

                var careerProfile = await db.CareerProfiles.IgnoreQueryFilters().AsNoTracking().FirstOrDefaultAsync(item => item.OwnerUserId == ownerUserId, cancellationToken);
                var careerProfileId = careerProfile?.Id;
                var careerVersions = await db.CareerProfileVersions.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Version).ToListAsync(cancellationToken);
                var careerExperiences = await db.CareerExperiences.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.SortOrder).ToListAsync(cancellationToken);
                var careerEducation = await db.CareerEducations.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.SortOrder).ToListAsync(cancellationToken);
                var careerSkills = await db.CareerSkills.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.SortOrder).ToListAsync(cancellationToken);
                var careerProjects = await db.CareerProjects.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.SortOrder).ToListAsync(cancellationToken);
                var careerCertifications = await db.CareerCertifications.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.SortOrder).ToListAsync(cancellationToken);
                var careerLanguages = await db.CareerLanguages.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.SortOrder).ToListAsync(cancellationToken);
                await AddJsonAsync("data/career.json", new
                {
                    Profile = careerProfile,
                    Versions = careerVersions,
                    Experiences = careerExperiences,
                    Education = careerEducation,
                    Skills = careerSkills,
                    Projects = careerProjects,
                    Certifications = careerCertifications,
                    Languages = careerLanguages,
                }, (careerProfileId is null ? 0 : 1) + careerVersions.Count + careerExperiences.Count + careerEducation.Count + careerSkills.Count + careerProjects.Count + careerCertifications.Count + careerLanguages.Count);

                var variants = await db.CvVariants.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken);
                var variantVersions = await db.CvVariantVersions.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken);
                var artifacts = await db.CvUploadArtifacts.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken);
                var extractionRuns = await db.CvExtractionRuns.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken);
                await AddJsonAsync("data/cv.json", new
                {
                    Variants = variants,
                    VariantVersions = variantVersions,
                    UploadArtifacts = artifacts.Select(item => new
                    {
                        item.Id,
                        item.OriginalFileName,
                        item.StoredFileName,
                        item.MimeType,
                        item.ByteSize,
                        item.Sha256,
                        item.UploadedAtUtc,
                    }),
                    ExtractionRuns = extractionRuns,
                }, variants.Count + variantVersions.Count + artifacts.Count + extractionRuns.Count);

                var tailoredDrafts = await db.TailoredCvDrafts.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken);
                var interviewNotes = await db.InterviewPrepNotes.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken);
                var aiNotes = await db.AiWorkspaceNotes.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken);
                var aiInteractions = await db.AiInteractions.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken);
                var aiUsage = await db.AiUsageRecords.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken);
                var checklist = await db.ApplicationChecklistItems.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken);
                var coverLetters = await db.CoverLetterVersions.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken);
                var interviewItems = await db.InterviewPrepItems.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken);
                var emailDrafts = await db.EmailDrafts.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.CreatedAtUtc).ToListAsync(cancellationToken);
                var emailAttempts = await db.EmailSendAttempts.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.CreatedAtUtc)
                    .Select(item => new EmailSendAttemptExport(item.Id, item.JobApplicationId, item.Provider, item.ClientRequestId, item.Status, item.ProviderMessageId, item.FailureCategory, item.CreatedAtUtc, item.StartedAtUtc, item.CompletedAtUtc))
                    .ToListAsync(cancellationToken);
                await AddJsonAsync("data/application-workspace.json", new
                {
                    TailoredCvDrafts = tailoredDrafts,
                    InterviewPrepNotes = interviewNotes,
                    AiWorkspaceNotes = aiNotes,
                    AiInteractions = aiInteractions,
                    AiUsage = aiUsage,
                    ChecklistItems = checklist,
                    CoverLetterVersions = coverLetters,
                    InterviewPrepItems = interviewItems,
                    EmailDrafts = emailDrafts,
                    EmailSendAttempts = emailAttempts,
                }, tailoredDrafts.Count + interviewNotes.Count + aiNotes.Count + aiInteractions.Count + aiUsage.Count + checklist.Count + coverLetters.Count + interviewItems.Count + emailDrafts.Count + emailAttempts.Count);

                var operations = await db.UserOperations.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.CreatedAtUtc)
                    .Select(item => new
                    {
                        item.Id,
                        item.TaskType,
                        item.Status,
                        item.Priority,
                        item.EntitlementDecision,
                        item.PrivacyPolicy,
                        item.SubjectType,
                        item.SubjectId,
                        item.Provider,
                        item.Model,
                        item.AttemptCount,
                        item.MaxAttempts,
                        item.CreatedAtUtc,
                        item.AvailableAtUtc,
                        item.StartedAtUtc,
                        item.CompletedAtUtc,
                        item.DeadlineAtUtc,
                        item.CancellationRequestedAtUtc,
                        item.ProgressStage,
                        item.ProgressPercent,
                        item.FailureCategory,
                        item.FailureMessage,
                        item.ResultReference,
                    }).ToListAsync(cancellationToken);
                var notifications = await db.UserNotifications.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.CreatedAtUtc).ToListAsync(cancellationToken);
                await AddJsonAsync("data/operations.json", new { Operations = operations, Notifications = notifications }, operations.Count + notifications.Count);

                var gmail = await db.GmailConnections.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId)
                    .Select(item => new { item.GmailAddress, item.Scope, item.ConnectedAt, item.LastSyncedAt, item.LastSyncAttemptedAt, item.LastSyncSucceededAt, item.LastSyncMode, item.LastSyncSource, item.LastSyncStatus, item.LastSyncError }).ToListAsync(cancellationToken);
                var microsoft = await db.MicrosoftGraphConnections.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId)
                    .Select(item => new { item.MailAddress, item.Scope, item.ConnectedAt, item.LastSyncedAt, item.LastSyncAttemptedAt, item.LastSyncSucceededAt, item.LastSyncMode, item.LastSyncSource, item.LastSyncStatus, item.LastSyncError }).ToListAsync(cancellationToken);
                var imap = await db.ImapConnections.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId)
                    .Select(item => new { item.Host, item.Port, item.UseSsl, item.Username, item.ConnectedAt, item.LastSyncedAt, item.LastSyncAttemptedAt, item.LastSyncSucceededAt, item.LastSyncMode, item.LastSyncSource, item.LastSyncStatus, item.LastSyncError }).ToListAsync(cancellationToken);
                var reviewDecisions = await db.GmailReviewDecisions.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.Id).ToListAsync(cancellationToken);
                var userRules = await db.UserRuleSettings.IgnoreQueryFilters().AsNoTracking().FirstOrDefaultAsync(item => item.OwnerUserId == ownerUserId, cancellationToken);
                await AddJsonAsync("data/settings-and-providers.json", new
                {
                    UserRules = userRules,
                    GmailConnections = gmail,
                    MicrosoftConnections = microsoft,
                    ImapConnections = imap,
                    GmailReviewDecisions = reviewDecisions,
                }, (userRules is null ? 0 : 1) + gmail.Count + microsoft.Count + imap.Count + reviewDecisions.Count);

                var sessions = await db.UserSessions.IgnoreQueryFilters().AsNoTracking().Where(item => item.UserId == ownerUserId).OrderBy(item => item.Id)
                    .Select(item => new { item.DeviceLabel, item.CreatedAtUtc, item.LastSeenAtUtc, item.ExpiresAtUtc, item.RevokedAtUtc }).ToListAsync(cancellationToken);
                var trustedDevices = await db.TrustedDevices.IgnoreQueryFilters().AsNoTracking().Where(item => item.UserId == ownerUserId).OrderBy(item => item.Id)
                    .Select(item => new { item.DeviceLabel, item.CreatedAtUtc, item.LastSeenAtUtc, item.ExpiresAtUtc }).ToListAsync(cancellationToken);
                var recoveryCodeCount = await db.TwoFactorRecoveryCodes.IgnoreQueryFilters().AsNoTracking().CountAsync(item => item.UserId == ownerUserId, cancellationToken);
                await AddJsonAsync("data/security-metadata.json", new
                {
                    TwoFactorEnabled = !string.IsNullOrWhiteSpace(user.TotpSecretEncrypted),
                    RecoveryCodeCount = recoveryCodeCount,
                    Sessions = sessions,
                    TrustedDevices = trustedDevices,
                }, sessions.Count + trustedDevices.Count + recoveryCodeCount);

                var ownedFiles = await fileInventory.BuildAsync(ownerUserId, cancellationToken);
                warnings.AddRange(ownedFiles.Warnings);
                foreach (var ownedFile in ownedFiles.Files)
                {
                    if (ownedFile.InlineBytes is not null)
                        await AddBytesAsync(archive, entries, ownedFile.ExportPath, ownedFile.InlineBytes, ownedFile.Category, 1, cancellationToken);
                    else
                        await AddFileAsync(archive, entries, ownedFile.SourcePath!, ownedFile.ExportPath, ownedFile.Category, cancellationToken);
                }

                const string readme = """
                    Jobjakt readable account export

                    This ZIP contains user-readable JSON plus owned files. manifest.json lists every entry,
                    byte size, SHA-256 checksum and any unavailable file warnings.

                    Excluded secrets: password/security/concurrency hashes, TOTP secrets, recovery-code and
                    trusted-device token hashes, session IDs, OAuth access/refresh tokens, IMAP passwords,
                    operation lease tokens, email payload hashes, data-protection keys and global settings.

                    External providers may retain mailbox, billing or model-service data under their own
                    policies. Application logs and immutable backups are not edited by this export. Their
                    exact retention remains an operator/legal decision documented in BLOCKERS.md.
                    """;
                await AddBytesAsync(archive, entries, "README.txt", Encoding.UTF8.GetBytes(readme), "documentation", 1, cancellationToken);

                var manifest = new
                {
                    SchemaVersion,
                    GeneratedAtUtc = generatedAt,
                    AccountId = ownerUserId,
                    EntryCount = entries.Count,
                    Entries = entries,
                    Warnings = warnings,
                    Exclusions = new[]
                    {
                        "authentication secrets and hashes",
                        "provider credentials and access/refresh tokens",
                        "global application settings and data-protection keys",
                        "other users' data",
                        "application logs, immutable backups and external-provider retained data",
                        "unattributable legacy generated files",
                    },
                };
                var manifestBytes = JsonSerializer.SerializeToUtf8Bytes(manifest, JsonOptions);
                var manifestEntry = archive.CreateEntry("manifest.json", CompressionLevel.Optimal);
                await using var manifestStream = manifestEntry.Open();
                await manifestStream.WriteAsync(manifestBytes, cancellationToken);
            }

            return new AccountDataExportArtifact(outputPath, $"jobjakt-account-export-{generatedAt:yyyyMMdd-HHmmss}.zip");
        }
        catch
        {
            try { File.Delete(outputPath); } catch { }
            throw;
        }
    }

    private static async Task AddFileAsync(ZipArchive archive, ICollection<ManifestEntry> entries, string sourcePath, string entryName, string category, CancellationToken cancellationToken)
    {
        await using var source = new FileStream(sourcePath, FileMode.Open, FileAccess.Read, FileShare.Read, 64 * 1024, FileOptions.Asynchronous | FileOptions.SequentialScan);
        var checksum = Convert.ToHexString(await SHA256.HashDataAsync(source, cancellationToken)).ToLowerInvariant();
        source.Position = 0;
        var entry = archive.CreateEntry(entryName.Replace('\\', '/'), CompressionLevel.Optimal);
        await using var target = entry.Open();
        await source.CopyToAsync(target, cancellationToken);
        entries.Add(new ManifestEntry(entry.FullName, source.Length, checksum, category, 1));
    }

    private static async Task AddBytesAsync(ZipArchive archive, ICollection<ManifestEntry> entries, string entryName, byte[] bytes, string category, int itemCount, CancellationToken cancellationToken)
    {
        var entry = archive.CreateEntry(entryName, CompressionLevel.Optimal);
        await using var target = entry.Open();
        await target.WriteAsync(bytes, cancellationToken);
        entries.Add(new ManifestEntry(entry.FullName, bytes.LongLength, Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant(), category, itemCount));
    }

    private sealed record ManifestEntry(string Path, long Bytes, string Sha256, string Category, int ItemCount);
}