jobtrackingapp/JobTrackerApi/Services/JobImport/JobImportService.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using JobTrackerApi.Services.JobImport.Translation;

namespace JobTrackerApi.Services.JobImport;

public sealed class JobImportService
{
    private readonly IHttpClientFactory _httpClientFactory;
    private readonly UniversalJobParser _universal;
    private readonly IEnumerable<IJobSitePlugin> _plugins;
    private readonly ITranslationService _translation;

    public JobImportService(
        IHttpClientFactory httpClientFactory,
        UniversalJobParser universal,
        IEnumerable<IJobSitePlugin> plugins,
        ITranslationService translation)
    {
        _httpClientFactory = httpClientFactory;
        _universal = universal;
        _plugins = plugins;
        _translation = translation;
    }

    public async Task<JobImportResult> PreviewAsync(string url, CancellationToken cancellationToken)
    {
        if (!TryValidateUrl(url, out var normalized, out var error))
        {
            return new JobImportResult
            {
                SourceUrl = url ?? "",
                Success = false,
                Parser = "none",
                Error = error
            };
        }

        var html = await FetchHtmlAsync(normalized, cancellationToken);
        if (html is null)
        {
            return new JobImportResult
            {
                SourceUrl = normalized,
                Success = false,
                Parser = "fetch",
                Error = "Failed to fetch HTML."
            };
        }

        var parsed = _universal.Parse(html, normalized);
        if (!parsed.Success)
        {
            foreach (var plugin in _plugins.Where(p => p.CanHandle(normalized)))
            {
                try
                {
                    var p = plugin.Parse(html, normalized);
                    if (p.Success)
                    {
                        parsed = p;
                        break;
                    }
                    parsed = p; // keep last failure for debugging
                }
                catch (Exception ex)
                {
                    parsed = new JobImportResult
                    {
                        SourceUrl = normalized,
                        Success = false,
                        Parser = plugin.GetType().Name,
                        Error = ex.Message
                    };
                }
            }
        }

        if (!parsed.Success) return parsed with { SourceUrl = normalized };

        var lang = LanguageDetector.Detect(parsed.Description);
        var tags = SkillTagger.Detect(parsed.Description);
        string? translated = null;
        if (string.Equals(lang, "no", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrWhiteSpace(parsed.Description))
        {
            translated = await _translation.TranslateToEnglishAsync(parsed.Description!, "no", cancellationToken);
        }

        return parsed with
        {
            SourceUrl = normalized,
            Language = lang,
            Tags = tags,
            TranslatedDescription = translated
        };
    }

    private async Task<string?> FetchHtmlAsync(string url, CancellationToken cancellationToken)
    {
        using var client = _httpClientFactory.CreateClient("jobimport");
        using var req = new HttpRequestMessage(HttpMethod.Get, url);
        req.Headers.TryAddWithoutValidation("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) JobTracker/1.0");
        req.Headers.TryAddWithoutValidation("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
        req.Headers.TryAddWithoutValidation("Accept-Language", "en-US,en;q=0.8,no;q=0.6,nb;q=0.6");

        using var res = await client.SendAsync(req, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
        if ((int)res.StatusCode >= 300 && (int)res.StatusCode < 400) return null; // avoid redirect chains to non-html.
        if (!res.IsSuccessStatusCode) return null;

        var ct = res.Content.Headers.ContentType?.MediaType ?? "";
        if (ct.Length > 0 && !ct.Contains("html", StringComparison.OrdinalIgnoreCase) && !ct.Contains("xml", StringComparison.OrdinalIgnoreCase))
        {
            // Still read: many sites omit content-type. Best-effort.
        }

        // Cap to avoid huge downloads.
        var bytes = await res.Content.ReadAsByteArrayAsync(cancellationToken);
        if (bytes.Length > 4_000_000) return null;
        return System.Text.Encoding.UTF8.GetString(bytes);
    }

    private static bool TryValidateUrl(string? url, out string normalized, out string error)
    {
        normalized = "";
        error = "";
        if (string.IsNullOrWhiteSpace(url))
        {
            error = "URL is required.";
            return false;
        }

        if (!Uri.TryCreate(url.Trim(), UriKind.Absolute, out var uri))
        {
            error = "Invalid URL.";
            return false;
        }

        if (uri.Scheme is not ("http" or "https"))
        {
            error = "Only http/https URLs are supported.";
            return false;
        }

        if (uri.IsLoopback || string.Equals(uri.Host, "localhost", StringComparison.OrdinalIgnoreCase))
        {
            error = "Local URLs are not allowed.";
            return false;
        }

        // Block literal private IPs.
        if (IPAddress.TryParse(uri.Host, out var ip))
        {
            if (IsPrivateIp(ip))
            {
                error = "Private IP URLs are not allowed.";
                return false;
            }
        }

        normalized = uri.ToString();
        return true;
    }

    private static bool IsPrivateIp(IPAddress ip)
    {
        if (ip.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork)
        {
            var b = ip.GetAddressBytes();
            return b[0] == 10 ||
                   (b[0] == 172 && b[1] >= 16 && b[1] <= 31) ||
                   (b[0] == 192 && b[1] == 168) ||
                   (b[0] == 169 && b[1] == 254);
        }
        if (ip.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6)
        {
            return ip.IsIPv6LinkLocal || ip.IsIPv6SiteLocal;
        }
        return false;
    }
}