using System; using System.Collections.Generic; using System.Linq; using System.Net; using System.Net.Http; using System.Threading; using System.Threading.Tasks; using JobTrackerApi.Services.JobImport.Translation; namespace JobTrackerApi.Services.JobImport; public sealed class JobImportService { private readonly IHttpClientFactory _httpClientFactory; private readonly UniversalJobParser _universal; private readonly IEnumerable _plugins; private readonly ITranslationService _translation; public JobImportService( IHttpClientFactory httpClientFactory, UniversalJobParser universal, IEnumerable plugins, ITranslationService translation) { _httpClientFactory = httpClientFactory; _universal = universal; _plugins = plugins; _translation = translation; } public async Task PreviewAsync(string url, CancellationToken cancellationToken) { if (!TryValidateUrl(url, out var normalized, out var error)) { return new JobImportResult { SourceUrl = url ?? "", Success = false, Parser = "none", Error = error }; } var html = await FetchHtmlAsync(normalized, cancellationToken); if (html is null) { return new JobImportResult { SourceUrl = normalized, Success = false, Parser = "fetch", Error = "Failed to fetch HTML." }; } var parsed = _universal.Parse(html, normalized); if (!parsed.Success) { foreach (var plugin in _plugins.Where(p => p.CanHandle(normalized))) { try { var p = plugin.Parse(html, normalized); if (p.Success) { parsed = p; break; } parsed = p; // keep last failure for debugging } catch (Exception ex) { parsed = new JobImportResult { SourceUrl = normalized, Success = false, Parser = plugin.GetType().Name, Error = ex.Message }; } } } if (!parsed.Success) return parsed with { SourceUrl = normalized }; var lang = LanguageDetector.Detect(parsed.Description); var tags = SkillTagger.Detect(parsed.Description); string? translated = null; if (string.Equals(lang, "no", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrWhiteSpace(parsed.Description)) { translated = await _translation.TranslateToEnglishAsync(parsed.Description!, "no", cancellationToken); } return parsed with { SourceUrl = normalized, Language = lang, Tags = tags, TranslatedDescription = translated }; } private async Task FetchHtmlAsync(string url, CancellationToken cancellationToken) { using var client = _httpClientFactory.CreateClient("jobimport"); using var req = new HttpRequestMessage(HttpMethod.Get, url); req.Headers.TryAddWithoutValidation("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) JobTracker/1.0"); req.Headers.TryAddWithoutValidation("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); req.Headers.TryAddWithoutValidation("Accept-Language", "en-US,en;q=0.8,no;q=0.6,nb;q=0.6"); using var res = await client.SendAsync(req, HttpCompletionOption.ResponseHeadersRead, cancellationToken); if ((int)res.StatusCode >= 300 && (int)res.StatusCode < 400) return null; // avoid redirect chains to non-html. if (!res.IsSuccessStatusCode) return null; var ct = res.Content.Headers.ContentType?.MediaType ?? ""; if (ct.Length > 0 && !ct.Contains("html", StringComparison.OrdinalIgnoreCase) && !ct.Contains("xml", StringComparison.OrdinalIgnoreCase)) { // Still read: many sites omit content-type. Best-effort. } // Cap to avoid huge downloads. var bytes = await res.Content.ReadAsByteArrayAsync(cancellationToken); if (bytes.Length > 4_000_000) return null; return System.Text.Encoding.UTF8.GetString(bytes); } private static bool TryValidateUrl(string? url, out string normalized, out string error) { normalized = ""; error = ""; if (string.IsNullOrWhiteSpace(url)) { error = "URL is required."; return false; } if (!Uri.TryCreate(url.Trim(), UriKind.Absolute, out var uri)) { error = "Invalid URL."; return false; } if (uri.Scheme is not ("http" or "https")) { error = "Only http/https URLs are supported."; return false; } if (uri.IsLoopback || string.Equals(uri.Host, "localhost", StringComparison.OrdinalIgnoreCase)) { error = "Local URLs are not allowed."; return false; } // Block literal private IPs. if (IPAddress.TryParse(uri.Host, out var ip)) { if (IsPrivateIp(ip)) { error = "Private IP URLs are not allowed."; return false; } } normalized = uri.ToString(); return true; } private static bool IsPrivateIp(IPAddress ip) { if (ip.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork) { var b = ip.GetAddressBytes(); return b[0] == 10 || (b[0] == 172 && b[1] >= 16 && b[1] <= 31) || (b[0] == 192 && b[1] == 168) || (b[0] == 169 && b[1] == 254); } if (ip.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6) { return ip.IsIPv6LinkLocal || ip.IsIPv6SiteLocal; } return false; } }