jobtrackingapp/JobTrackerApi/Services/AccountDeletionService.cs

using System.Security.Cryptography;
using System.Text.Json;
using JobTrackerApi.Data;
using JobTrackerApi.Models;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Caching.Memory;

namespace JobTrackerApi.Services;

public sealed record AccountDeletionRequestResult(Guid RequestId, string Status, string Stage);

public sealed class AccountDeletionService(
    JobTrackerContext db,
    AccountOwnedFileInventory fileInventory,
    AccountDeletionTombstoneStore tombstones,
    IAiSidecarCachePurger aiSidecarCache,
    IConfiguration configuration,
    IMemoryCache memoryCache,
    TimeProvider timeProvider,
    ILogger<AccountDeletionService> logger)
{
    private const int MaxAttempts = 20;
    public bool CanAcceptRequests => configuration.GetValue("AccountLifecycle:DeletionEnabled", false);

    public async Task<AccountDeletionRequestResult?> RequestAsync(string ownerUserId, string requestedByUserId, CancellationToken cancellationToken)
    {
        if (!CanAcceptRequests) return null;
        var request = await RequestCoreAsync(ownerUserId, requestedByUserId, cancellationToken);
        return new AccountDeletionRequestResult(request.Id, request.Status, request.Stage);
    }

    public async Task<int> StageRestoredAccountsAsync(CancellationToken cancellationToken)
    {
        var ownerKeys = (await tombstones.ReadAsync(cancellationToken)).Select(item => item.OwnerKey).ToHashSet(StringComparer.Ordinal);
        if (ownerKeys.Count == 0) return 0;
        var users = await db.Users.AsNoTracking().Select(item => item.Id).ToListAsync(cancellationToken);
        var restored = users.Where(item => ownerKeys.Contains(AppPaths.GetOwnerStorageKey(item))).ToList();
        foreach (var ownerUserId in restored)
        {
            await RequestCoreAsync(ownerUserId, "tombstone-replay", cancellationToken);
        }
        return restored.Count;
    }

    public async Task<int> ProcessPendingAsync(CancellationToken cancellationToken)
    {
        var requestIds = await db.AccountDeletionRequests.AsNoTracking()
            .Where(item => item.Status != AccountDeletionRequestStatuses.Completed && item.AttemptCount < MaxAttempts)
            .OrderBy(item => item.Id)
            .Select(item => item.Id)
            .ToListAsync(cancellationToken);
        var completed = 0;
        foreach (var requestId in requestIds)
        {
            if (await ProcessAsync(requestId, cancellationToken)) completed++;
        }
        return completed;
    }

    public async Task<bool> ProcessAsync(Guid requestId, CancellationToken cancellationToken)
    {
        // Requests are normally processed in a fresh background scope. Clearing here also makes
        // direct retries safe when the same scoped service accepted the request: ExecuteDelete
        // must not leave a previously tracked ApplicationUser pending for a later SaveChanges.
        db.ChangeTracker.Clear();
        var request = await db.AccountDeletionRequests.Include(item => item.Files).FirstOrDefaultAsync(item => item.Id == requestId, cancellationToken);
        if (request is null) return false;
        if (request.Status == AccountDeletionRequestStatuses.Completed) return true;
        request.Status = AccountDeletionRequestStatuses.Processing;
        request.AttemptCount++;
        request.StartedAtUtc ??= timeProvider.GetUtcNow();
        request.LastErrorCategory = null;
        request.LastErrorMessage = null;
        await db.SaveChangesAsync(cancellationToken);

        try
        {
            if (request.Stage == AccountDeletionStages.Requested)
                await PrepareFilesAsync(request, cancellationToken);
            if (request.Stage == AccountDeletionStages.QuarantiningFiles)
                await QuarantineFilesAsync(request, cancellationToken);
            if (request.Stage == AccountDeletionStages.DeletingDatabase)
                await DeleteDatabaseRowsAsync(request, cancellationToken);
            if (request.Stage == AccountDeletionStages.PurgingFiles)
                await PurgeFilesAsync(request, cancellationToken);
            if (request.Stage == AccountDeletionStages.RecordingTombstone)
                await CompleteAsync(request, cancellationToken);
            return request.Status == AccountDeletionRequestStatuses.Completed;
        }
        catch (Exception ex)
        {
            request.Status = AccountDeletionRequestStatuses.RetryRequired;
            request.LastErrorCategory = Classify(ex);
            request.LastErrorMessage = Sanitize(ex.Message);
            try { await db.SaveChangesAsync(cancellationToken); }
            catch (Exception saveError) { logger.LogError(saveError, "Could not persist account deletion failure for {RequestId}", request.Id); }
            logger.LogWarning(ex, "Account deletion request {RequestId} stopped at {Stage}; it remains retryable", request.Id, request.Stage);
            return false;
        }
    }

    private async Task<AccountDeletionRequest> RequestCoreAsync(string ownerUserId, string requestedByUserId, CancellationToken cancellationToken)
    {
        var existing = await db.AccountDeletionRequests
            .Where(item => item.OwnerUserId == ownerUserId && item.Status != AccountDeletionRequestStatuses.Completed)
            .OrderBy(item => item.Id)
            .FirstOrDefaultAsync(cancellationToken);
        if (existing is not null) return existing;

        var user = await db.Users.FirstOrDefaultAsync(item => item.Id == ownerUserId, cancellationToken)
            ?? throw new InvalidOperationException("The account no longer exists.");
        var now = timeProvider.GetUtcNow();
        var request = new AccountDeletionRequest
        {
            Id = Guid.NewGuid(),
            OwnerUserId = ownerUserId,
            OwnerKey = AppPaths.GetOwnerStorageKey(ownerUserId),
            RequestedByUserId = requestedByUserId,
            Status = AccountDeletionRequestStatuses.Pending,
            Stage = AccountDeletionStages.Requested,
            RequestedAtUtc = now,
        };
        user.DeletionStatus = AccountDeletionStatuses.Pending;
        user.DeletionRequestedAtUtc = now;
        user.SecurityStamp = Guid.NewGuid().ToString();
        foreach (var variant in await db.CvVariants.IgnoreQueryFilters().Where(item => item.OwnerUserId == ownerUserId && item.IsPublic).ToListAsync(cancellationToken))
            variant.IsPublic = false;
        foreach (var session in await db.UserSessions.IgnoreQueryFilters().Where(item => item.UserId == ownerUserId && item.RevokedAtUtc == null).ToListAsync(cancellationToken))
            session.RevokedAtUtc = now;
        db.TrustedDevices.RemoveRange(await db.TrustedDevices.IgnoreQueryFilters().Where(item => item.UserId == ownerUserId).ToListAsync(cancellationToken));
        var operations = await db.UserOperations.IgnoreQueryFilters().Where(item => item.OwnerUserId == ownerUserId
            && item.Status != OperationStatuses.Succeeded
            && item.Status != OperationStatuses.Failed
            && item.Status != OperationStatuses.Cancelled).ToListAsync(cancellationToken);
        foreach (var operation in operations)
        {
            if (operation.Status == OperationStatuses.Running) operation.CancellationRequestedAtUtc = now.UtcDateTime;
            else
            {
                operation.Status = OperationStatuses.Cancelled;
                operation.CompletedAtUtc = now.UtcDateTime;
                operation.LeaseToken = null;
                operation.LeaseExpiresAtUtc = null;
            }
        }
        db.AccountDeletionRequests.Add(request);
        await db.SaveChangesAsync(cancellationToken);
        return request;
    }

    private async Task PrepareFilesAsync(AccountDeletionRequest request, CancellationToken cancellationToken)
    {
        var inventory = await fileInventory.BuildAsync(request.OwnerUserId, cancellationToken, includeAccountExports: true);
        if (inventory.Warnings.Any(item => item.StartsWith("Excluded unsafe", StringComparison.Ordinal)))
            throw new InvalidOperationException("One or more owned file paths failed the managed-root safety check.");
        request.WarningJson = JsonSerializer.Serialize(inventory.Warnings);
        foreach (var ownedFile in inventory.Files.Where(item => item.SourcePath is not null))
        {
            var sourcePath = ownedFile.SourcePath!;
            if (request.Files.Any(item => string.Equals(item.OriginalPath, sourcePath, PathComparison))) continue;
            await using var source = new FileStream(sourcePath, FileMode.Open, FileAccess.Read, FileShare.Read, 64 * 1024, FileOptions.Asynchronous | FileOptions.SequentialScan);
            var hash = Convert.ToHexString(await SHA256.HashDataAsync(source, cancellationToken)).ToLowerInvariant();
            request.Files.Add(new AccountDeletionFile
            {
                AccountDeletionRequestId = request.Id,
                Category = ownedFile.Category,
                OriginalPath = sourcePath,
                QuarantinePath = sourcePath + $".{request.Id:N}.account-deleting",
                Status = "planned",
                ByteSize = source.Length,
                Sha256 = hash,
            });
        }
        request.FileCount = request.Files.Count;
        request.Stage = AccountDeletionStages.QuarantiningFiles;
        await db.SaveChangesAsync(cancellationToken);
    }

    private async Task QuarantineFilesAsync(AccountDeletionRequest request, CancellationToken cancellationToken)
    {
        try
        {
            foreach (var file in request.Files)
            {
                cancellationToken.ThrowIfCancellationRequested();
                if (File.Exists(file.QuarantinePath))
                {
                    file.Status = "quarantined";
                    continue;
                }
                if (!File.Exists(file.OriginalPath))
                {
                    file.Status = "missing";
                    AppendWarning(request, $"Owned {file.Category} file disappeared before quarantine.");
                    continue;
                }
                File.Move(file.OriginalPath, file.QuarantinePath, overwrite: false);
                file.Status = "quarantined";
            }
            request.Stage = AccountDeletionStages.DeletingDatabase;
            await db.SaveChangesAsync(cancellationToken);
        }
        catch
        {
            RestoreQuarantinedFiles(request);
            await db.SaveChangesAsync(cancellationToken);
            throw;
        }
    }

    private async Task DeleteDatabaseRowsAsync(AccountDeletionRequest request, CancellationToken cancellationToken)
    {
        foreach (var file in request.Files)
        {
            if (File.Exists(file.OriginalPath) || (file.Status != "missing" && !File.Exists(file.QuarantinePath)))
                throw new InvalidOperationException("Owned files are not fully quarantined; database deletion was not started.");
        }

        var transaction = db.Database.IsRelational() ? await db.Database.BeginTransactionAsync(cancellationToken) : null;
        try
        {
            var owner = request.OwnerUserId;
            var applicationIds = await db.JobApplications.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).Select(item => item.Id).ToListAsync(cancellationToken);
            var variantIds = await db.CvVariants.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).Select(item => item.Id).ToListAsync(cancellationToken);
            var deleted = 0;
            deleted += await db.EmailDrafts.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.EmailSendAttempts.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.ApplicationChecklistItems.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.CoverLetterVersions.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.InterviewPrepItems.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.InterviewPrepNotes.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.AiWorkspaceNotes.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.AiInteractions.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.AiUsageRecords.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.TailoredCvDrafts.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.Correspondences.IgnoreQueryFilters().Where(item => applicationIds.Contains(item.JobApplicationId)).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.JobEvents.IgnoreQueryFilters().Where(item => applicationIds.Contains(item.JobApplicationId)).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.Attachments.IgnoreQueryFilters().Where(item => applicationIds.Contains(item.JobApplicationId)).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.GmailReviewDecisions.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.CvVariantVersions.IgnoreQueryFilters().Where(item => variantIds.Contains(item.CvVariantId)).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.CvVariants.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.CareerExperiences.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.CareerEducations.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.CareerSkills.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.CareerProjects.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.CareerCertifications.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.CareerLanguages.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.CareerProfileVersions.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.CareerProfiles.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.CvExtractionRuns.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.CvUploadArtifacts.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
            if (await db.GmailConnections.IgnoreQueryFilters().AnyAsync(item => item.OwnerUserId == owner, cancellationToken))
                AppendWarning(request, "Google consent was not revoked remotely; local Gmail credentials were deleted.");
            deleted += await db.GmailConnections.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.MicrosoftGraphConnections.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.ImapConnections.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.UserRuleSettings.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.UserNotifications.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.UserOperations.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.JobApplications.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.Jobs.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.Companies.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.TwoFactorRecoveryCodes.IgnoreQueryFilters().Where(item => item.UserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.TrustedDevices.IgnoreQueryFilters().Where(item => item.UserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.UserSessions.IgnoreQueryFilters().Where(item => item.UserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.UserClaims.Where(item => item.UserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.UserLogins.Where(item => item.UserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.UserTokens.Where(item => item.UserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.UserRoles.Where(item => item.UserId == owner).ExecuteDeleteAsync(cancellationToken);
            deleted += await db.Users.Where(item => item.Id == owner).ExecuteDeleteAsync(cancellationToken);
            request.DatabaseRowCount = deleted;
            request.Stage = AccountDeletionStages.PurgingFiles;
            request.Status = AccountDeletionRequestStatuses.Processing;
            await db.SaveChangesAsync(cancellationToken);
            if (transaction is not null) await transaction.CommitAsync(cancellationToken);
        }
        catch
        {
            if (transaction is not null)
            {
                try { await transaction.RollbackAsync(CancellationToken.None); }
                catch (Exception rollbackError) { logger.LogError(rollbackError, "Account deletion transaction rollback outcome is uncertain for {RequestId}", request.Id); }
            }

            // A commit can succeed at the database and still lose the acknowledgement. Restore
            // quarantined files only when the owner row proves the database deletion rolled back.
            // When the outcome cannot be read, leave files quarantined and safely replay deletion.
            try
            {
                if (await db.Users.AsNoTracking().AnyAsync(item => item.Id == request.OwnerUserId, CancellationToken.None))
                {
                    RestoreQuarantinedFiles(request);
                    request.Stage = AccountDeletionStages.DeletingDatabase;
                }
            }
            catch (Exception verificationError)
            {
                logger.LogError(verificationError, "Could not verify database deletion outcome for {RequestId}; files remain quarantined", request.Id);
                request.Stage = AccountDeletionStages.DeletingDatabase;
            }
            throw;
        }
        finally
        {
            if (transaction is not null) await transaction.DisposeAsync();
        }
    }

    private async Task PurgeFilesAsync(AccountDeletionRequest request, CancellationToken cancellationToken)
    {
        foreach (var file in request.Files)
        {
            cancellationToken.ThrowIfCancellationRequested();
            if (File.Exists(file.QuarantinePath)) File.Delete(file.QuarantinePath);
            file.Status = "purged";
        }
        if (memoryCache is MemoryCache cache) cache.Compact(1.0);
        // The sidecar cache is content-keyed rather than owner-keyed, so deletion clears it
        // globally. Keep this inside the durable stage: an unavailable sidecar leaves the request
        // retryable and prevents a false completion/tombstone acknowledgement.
        await aiSidecarCache.PurgeAsync(cancellationToken);
        request.Stage = AccountDeletionStages.RecordingTombstone;
        await db.SaveChangesAsync(cancellationToken);
    }

    private async Task CompleteAsync(AccountDeletionRequest request, CancellationToken cancellationToken)
    {
        var completedAt = timeProvider.GetUtcNow();
        await tombstones.AppendAsync(request.Id, request.OwnerKey, completedAt, cancellationToken);
        request.Stage = AccountDeletionStages.Completed;
        request.Status = AccountDeletionRequestStatuses.Completed;
        request.CompletedAtUtc = completedAt;
        await db.SaveChangesAsync(cancellationToken);
    }

    private static void RestoreQuarantinedFiles(AccountDeletionRequest request)
    {
        foreach (var file in request.Files.Where(item => File.Exists(item.QuarantinePath) && !File.Exists(item.OriginalPath)))
        {
            try
            {
                File.Move(file.QuarantinePath, file.OriginalPath, overwrite: false);
                file.Status = "planned";
            }
            catch { }
        }
    }

    private static void AppendWarning(AccountDeletionRequest request, string warning)
    {
        var warnings = string.IsNullOrWhiteSpace(request.WarningJson)
            ? new List<string>()
            : JsonSerializer.Deserialize<List<string>>(request.WarningJson) ?? new List<string>();
        if (!warnings.Contains(warning, StringComparer.Ordinal)) warnings.Add(warning);
        request.WarningJson = JsonSerializer.Serialize(warnings);
    }

    private static string Classify(Exception exception) => exception switch
    {
        IOException => "file_io",
        UnauthorizedAccessException => "file_access",
        DbUpdateException => "database",
        OperationCanceledException => "cancelled",
        _ => "unexpected",
    };

    private static string Sanitize(string value)
    {
        var message = value.Replace('\r', ' ').Replace('\n', ' ').Trim();
        return message.Length > 500 ? message[..500] : message;
    }

    private static StringComparison PathComparison => OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal;
}

public sealed class AccountDeletionHostedService(
    IServiceScopeFactory scopes,
    IStartupReadiness startupReadiness,
    ILogger<AccountDeletionHostedService> logger) : BackgroundService
{
    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        await startupReadiness.WaitUntilReadyAsync(stoppingToken);
        while (!stoppingToken.IsCancellationRequested)
        {
            try
            {
                await using var scope = scopes.CreateAsyncScope();
                await scope.ServiceProvider.GetRequiredService<AccountDeletionService>().ProcessPendingAsync(stoppingToken);
            }
            catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { }
            catch (Exception ex) { logger.LogError(ex, "Account deletion reconciliation failed; durable requests remain retryable"); }
            await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken);
        }
    }
}