Inboxintel/src/InboxIntel.Infrastructure/Sync/SyncService.cs

using InboxIntel.Application.Abstractions;
using InboxIntel.Application.DTOs;
using InboxIntel.Domain.Entities;
using InboxIntel.Domain.Enums;
using InboxIntel.Infrastructure.Configuration;
using InboxIntel.Infrastructure.Gmail;
using InboxIntel.Infrastructure.Persistence;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;

namespace InboxIntel.Infrastructure.Sync;

/// <summary>
/// Orchestrates Gmail synchronisation. Full sync pages through every message;
/// incremental sync replays the Gmail history feed since the last historyId.
/// Progress is checkpointed to <see cref="SyncState"/> so an interrupted run
/// resumes from its last page token instead of restarting.
/// </summary>
public class SyncService : ISyncService
{
    private readonly AppDbContext _db;
    private readonly IGmailService _gmail;
    private readonly ILogger<SyncService> _logger;
    private readonly GmailSyncOptions _options;
    private readonly ISyncQueue _queue;
    private readonly IAiService _ai;

    public SyncService(AppDbContext db, IGmailService gmail, ILogger<SyncService> logger, IOptions<GmailSyncOptions> options, ISyncQueue queue, IAiService ai)
    {
        _db = db;
        _gmail = gmail;
        _logger = logger;
        _options = options.Value;
        _queue = queue;
        _ai = ai;
    }

    public async Task QueueSyncAsync(Guid userId, bool fullSync, CancellationToken ct = default)
    {
        // Flip to Running synchronously so the UI splash shows immediately,
        // then hand the actual work to the background worker.
        var state = await GetOrCreateStateAsync(userId, ct);
        state.Status = SyncStatus.Running;
        state.LastSyncType = fullSync ? SyncType.Full : SyncType.Incremental;
        state.StartedUtc = DateTimeOffset.UtcNow;
        state.LastError = null;
        if (fullSync) state.MessagesProcessed = 0;
        await _db.SaveChangesAsync(ct);
        _queue.Enqueue(userId, fullSync);
    }

    public async Task<SyncStatus> GetStatusAsync(Guid userId, CancellationToken ct = default)
        => (await GetOrCreateStateAsync(userId, ct)).Status;

    public async Task<SyncProgressDto> GetProgressAsync(Guid userId, CancellationToken ct = default)
    {
        var s = await GetOrCreateStateAsync(userId, ct);
        // Total is unknown (0 -> indeterminate bar) until Gmail returns the real
        // count; then it's the mailbox total, capped by MaxMessages in dev.
        var total = s.TotalMessagesEstimate;
        if (_options.MaxMessages > 0 && total > 0)
            total = Math.Min(total, _options.MaxMessages);
        return new SyncProgressDto(
            s.Status.ToString(), s.MessagesProcessed, total,
            s.Status == SyncStatus.Running, s.LastSuccessfulSyncUtc, s.LastError);
    }

    public async Task RunFullSyncAsync(Guid userId, CancellationToken ct = default)
    {
        var state = await GetOrCreateStateAsync(userId, ct);
        state.Status = SyncStatus.Running;
        state.LastSyncType = SyncType.Full;
        state.StartedUtc = DateTimeOffset.UtcNow;
        state.LastError = null;
        await _db.SaveChangesAsync(ct);

        try
        {
            await SyncLabelsAsync(userId, ct);

            // Dev cap: stop after MaxMessages (most recent first). 0 = unlimited.
            var maxMessages = _options.MaxMessages;

            // Phase 1 — enumerate message ids (cheap, ids only) to get an ACCURATE
            // total. Gmail's resultSizeEstimate is unreliable, so we count instead.
            var ids = new List<string>();
            string? listToken = null;
            do
            {
                var page = await _gmail.ListMessageIdsAsync(userId, listToken, ct);
                foreach (var id in page.MessageIds)
                {
                    ids.Add(id);
                    if (maxMessages > 0 && ids.Count >= maxMessages) break;
                }
                listToken = page.NextPageToken;
            }
            while (listToken is not null && (maxMessages == 0 || ids.Count < maxMessages) && !ct.IsCancellationRequested);

            state.TotalMessagesEstimate = ids.Count;   // the true target (capped in dev)
            await _db.SaveChangesAsync(ct);

            // Phase 2 — fetch bodies in chunks, in parallel, skipping ones we already
            // have. The DbContext is not thread-safe, so upserts run sequentially.
            const int chunkSize = 100;
            for (var i = 0; i < ids.Count && !ct.IsCancellationRequested; i += chunkSize)
            {
                var chunk = ids.GetRange(i, Math.Min(chunkSize, ids.Count - i));
                var have = (await _db.Emails
                    .Where(e => e.UserId == userId && chunk.Contains(e.GmailMessageId))
                    .Select(e => e.GmailMessageId).ToListAsync(ct)).ToHashSet();
                state.MessagesProcessed += have.Count;  // already-synced count toward progress

                var toFetch = chunk.Where(id => !have.Contains(id)).ToList();
                var details = await FetchDetailsParallelAsync(userId, toFetch, ct);
                foreach (var detail in details)
                {
                    await UpsertMessageAsync(userId, detail, ct);
                    state.MessagesProcessed++;
                    if (state.MessagesProcessed % 25 == 0)
                        await _db.SaveChangesAsync(ct); // smooth progress for the splash
                }
                await _db.SaveChangesAsync(ct);
            }

            state.LastHistoryId = await _gmail.GetProfileHistoryIdAsync(userId, ct);
            state.ResumePageToken = null;
            state.Status = SyncStatus.Completed;
            state.CompletedUtc = DateTimeOffset.UtcNow;
            state.LastSuccessfulSyncUtc = DateTimeOffset.UtcNow;
            state.ConsecutiveFailures = 0;
            await _db.SaveChangesAsync(ct);
        }
        catch (Exception ex)
        {
            await MarkFailedAsync(state, ex, ct);
            throw;
        }
    }

    public async Task RunIncrementalSyncAsync(Guid userId, CancellationToken ct = default)
    {
        var state = await GetOrCreateStateAsync(userId, ct);
        if (string.IsNullOrEmpty(state.LastHistoryId))
        {
            await RunFullSyncAsync(userId, ct); // no watermark yet -> full sync
            return;
        }

        state.Status = SyncStatus.Running;
        state.LastSyncType = SyncType.Incremental;
        state.StartedUtc = DateTimeOffset.UtcNow;
        await _db.SaveChangesAsync(ct);

        try
        {
            string? pageToken = null;
            string? newHistoryId = state.LastHistoryId;
            do
            {
                var page = await _gmail.ListHistoryAsync(userId, state.LastHistoryId!, pageToken, ct);
                foreach (var id in page.ChangedMessageIds.Distinct())
                {
                    var detail = await _gmail.GetMessageAsync(userId, id, ct);
                    await UpsertMessageAsync(userId, detail, ct);
                }
                foreach (var id in page.DeletedMessageIds.Distinct())
                {
                    var existing = await _db.Emails.FirstOrDefaultAsync(e => e.UserId == userId && e.GmailMessageId == id, ct);
                    if (existing is not null) _db.Emails.Remove(existing);
                }
                if (page.NewHistoryId is not null) newHistoryId = page.NewHistoryId;
                pageToken = page.NextPageToken;
                await _db.SaveChangesAsync(ct);
            }
            while (pageToken is not null && !ct.IsCancellationRequested);

            state.LastHistoryId = newHistoryId;
            state.Status = SyncStatus.Completed;
            state.CompletedUtc = DateTimeOffset.UtcNow;
            state.LastSuccessfulSyncUtc = DateTimeOffset.UtcNow;
            state.ConsecutiveFailures = 0;
            await _db.SaveChangesAsync(ct);
        }
        catch (Exception ex)
        {
            await MarkFailedAsync(state, ex, ct);
            throw;
        }
    }

    private async Task MarkFailedAsync(SyncState state, Exception ex, CancellationToken ct)
    {
        _logger.LogError(ex, "Sync failed for user {UserId}", state.UserId);
        // Discard the failed batch's pending changes; otherwise saving the
        // failure status re-attempts the same bad inserts and throws again.
        _db.ChangeTracker.Clear();
        var fresh = await _db.SyncStates.FirstOrDefaultAsync(s => s.UserId == state.UserId, ct);
        if (fresh is null) return;
        fresh.Status = SyncStatus.Failed;
        fresh.ConsecutiveFailures++;
        fresh.LastError = ex.Message;
        await _db.SaveChangesAsync(ct);
    }

    private async Task<SyncState> GetOrCreateStateAsync(Guid userId, CancellationToken ct)
    {
        var state = await _db.SyncStates.FirstOrDefaultAsync(s => s.UserId == userId, ct);
        if (state is null)
        {
            state = new SyncState { UserId = userId };
            _db.SyncStates.Add(state);
            await _db.SaveChangesAsync(ct);
        }
        return state;
    }

    private async Task SyncLabelsAsync(Guid userId, CancellationToken ct)
    {
        var remote = await _gmail.ListLabelsAsync(userId, ct);
        foreach (var label in remote)
        {
            var existing = await _db.Labels.FirstOrDefaultAsync(l => l.UserId == userId && l.GmailLabelId == label.GmailLabelId, ct);
            if (existing is null) _db.Labels.Add(label);
            else { existing.Name = label.Name; existing.Type = label.Type; }
        }
        await _db.SaveChangesAsync(ct);
    }

    // Per-sync-run cache of the user's GmailLabelId -> local Label.Id, so email/label linkage
    // costs no extra query per message. Populated lazily; labels are synced before messages.
    private Dictionary<string, Guid>? _labelCache;
    private Guid _labelCacheUserId;

    private async Task<Dictionary<string, Guid>> GetLabelMapAsync(Guid userId, CancellationToken ct)
    {
        if (_labelCache is null || _labelCacheUserId != userId)
        {
            _labelCache = await _db.Labels
                .Where(l => l.UserId == userId)
                .ToDictionaryAsync(l => l.GmailLabelId, l => l.Id, ct);
            _labelCacheUserId = userId;
        }
        return _labelCache;
    }

    /// <summary>Resolves domain/sender/thread, then upserts the email, its labels, and
    /// attachment metadata. Idempotent: re-syncing a message replaces the prior row (and its
    /// labels) rather than duplicating it.</summary>
    private async Task UpsertMessageAsync(Guid userId, GmailMessageDetail d, CancellationToken ct)
    {
        // True upsert: drop any existing copy first so labels/flags re-populate cleanly and
        // an incrementally-changed message can't be duplicated. EmailLabels/Attachments cascade.
        var prior = await _db.Emails.FirstOrDefaultAsync(
            e => e.UserId == userId && e.GmailMessageId == d.GmailMessageId, ct);
        if (prior is not null) _db.Emails.Remove(prior);

        var sender = await ResolveSenderAsync(userId, d.FromAddress, d.FromDisplayName, ct);
        var thread = await ResolveThreadAsync(userId, d.GmailThreadId, d.Subject, d.Snippet, d.SentAtUtc, ct);

        var category = HeuristicClassifier.Classify(d);
        // The heuristic rule set falls back to Personal when nothing more specific
        // matches; if AI is enabled, give it a shot at a better category.
        if (category == EmailCategory.Personal && _ai.IsEnabled)
            category = await _ai.ClassifyFallbackAsync(d.Subject, d.FromAddress, d.Snippet, ct);

        var email = new Email
        {
            UserId = userId,
            GmailMessageId = d.GmailMessageId,
            ThreadId = thread.Id,
            SenderId = sender.Id,
            Subject = Trunc(d.Subject, 1024),
            Snippet = Trunc(d.Snippet, 2048),
            BodyText = d.BodyText, // unlimited (text column)
            SentAtUtc = d.SentAtUtc,
            ReceivedAtUtc = d.SentAtUtc,
            SizeEstimateBytes = d.SizeEstimateBytes,
            IsUnread = d.IsUnread,
            IsInInbox = d.LabelIds.Contains("INBOX"),
            IsStarred = d.LabelIds.Contains("STARRED"),
            IsImportant = d.LabelIds.Contains("IMPORTANT"),
            IsTrashed = d.LabelIds.Contains("TRASH"),
            HasAttachments = d.HasAttachments,
            HasListUnsubscribe = d.HasListUnsubscribe,
            ListUnsubscribeRaw = Trunc(d.ListUnsubscribeRaw, 2048),
            SupportsOneClickUnsubscribe = d.SupportsOneClickUnsubscribe,
            Category = category
        };
        _db.Emails.Add(email);

        // Link the email to its Gmail labels (system + user) so label-based folders — Sent,
        // Spam, and "Unlabelled" (no user label) — resolve correctly. Previously no EmailLabel
        // rows were ever created, so every label folder was empty and every email looked unlabelled.
        var labelMap = await GetLabelMapAsync(userId, ct);
        foreach (var gmailLabelId in d.LabelIds.Distinct())
            if (labelMap.TryGetValue(gmailLabelId, out var localLabelId))
                _db.Set<EmailLabel>().Add(new EmailLabel { EmailId = email.Id, LabelId = localLabelId });

        foreach (var (fileName, mime, size, attId) in d.Attachments)
        {
            _db.Attachments.Add(new Attachment
            {
                UserId = userId,
                EmailId = email.Id,
                FileName = Trunc(fileName, 512) ?? string.Empty,
                MimeType = Trunc(mime, 255),
                SizeBytes = size,
                GmailAttachmentId = attId
            });
        }

        // Maintain sender rollups for fast grouping.
        sender.EmailCount++;
        if (d.IsUnread) sender.UnreadCount++;
        sender.TotalSizeBytes += d.SizeEstimateBytes;
        sender.LastReceivedUtc = d.SentAtUtc;
        if (d.HasListUnsubscribe) sender.HasUnsubscribe = true;

        thread.MessageCount++;
        thread.LastMessageUtc = d.SentAtUtc;
    }

    /// <summary>
    /// Fetches message bodies concurrently (bounded by MaxParallelism) to speed
    /// up the sync. Results only — the caller persists them sequentially.
    /// </summary>
    private async Task<List<GmailMessageDetail>> FetchDetailsParallelAsync(Guid userId, List<string> ids, CancellationToken ct)
    {
        if (ids.Count == 0) return new List<GmailMessageDetail>();
        using var sem = new SemaphoreSlim(Math.Max(1, _options.MaxParallelism));
        var tasks = ids.Select(async id =>
        {
            await sem.WaitAsync(ct);
            try { return await _gmail.GetMessageAsync(userId, id, ct); }
            finally { sem.Release(); }
        });
        var results = await Task.WhenAll(tasks);
        return results.ToList();
    }

    private async Task<Sender> ResolveSenderAsync(Guid userId, string address, string? displayName, CancellationToken ct)
    {
        // Check the in-memory tracker first so senders/domains added earlier in
        // this batch (not yet saved) are reused instead of duplicated.
        var sender = _db.Senders.Local.FirstOrDefault(s => s.UserId == userId && s.Address == address)
            ?? await _db.Senders.FirstOrDefaultAsync(s => s.UserId == userId && s.Address == address, ct);
        if (sender is not null) return sender;

        address = Trunc(address, 320)!;
        var rawDomain = address.Contains('@') ? address[(address.IndexOf('@') + 1)..] : "unknown";
        var domainName = Trunc(rawDomain, 255)!;
        var domain = _db.Domains.Local.FirstOrDefault(x => x.UserId == userId && x.Name == domainName)
            ?? await _db.Domains.FirstOrDefaultAsync(x => x.UserId == userId && x.Name == domainName, ct);
        if (domain is null)
        {
            domain = new MailDomain { UserId = userId, Name = domainName };
            _db.Domains.Add(domain);
        }
        domain.EmailCount++;

        sender = new Sender { UserId = userId, Address = address, DisplayName = Trunc(displayName, 255), Domain = domain, DomainId = domain.Id };
        _db.Senders.Add(sender);
        return sender;
    }

    /// <summary>Truncates a string to a column's max length so no email can overflow it.</summary>
    private static string? Trunc(string? value, int max)
        => value is null ? null : value.Length <= max ? value : value[..max];

    private async Task<MailThread> ResolveThreadAsync(Guid userId, string gmailThreadId, string? subject, string? snippet, DateTimeOffset sentAt, CancellationToken ct)
    {
        var thread = _db.Threads.Local.FirstOrDefault(t => t.UserId == userId && t.GmailThreadId == gmailThreadId)
            ?? await _db.Threads.FirstOrDefaultAsync(t => t.UserId == userId && t.GmailThreadId == gmailThreadId, ct);
        if (thread is not null) return thread;
        thread = new MailThread
        {
            UserId = userId,
            GmailThreadId = gmailThreadId,
            Subject = Trunc(subject, 1024),
            Snippet = Trunc(snippet, 2048),
            FirstMessageUtc = sentAt
        };
        _db.Threads.Add(thread);
        return thread;
    }
}