using System.IO.Compression; using System.Net; using System.Net.Security; using System.Net.Sockets; using System.Security.Authentication; using System.Security.Cryptography; using System.Security.Cryptography.X509Certificates; using System.Text; using JobTrackerApi.Services.JobImport; using JobTrackerApi.Services.JobImport.Translation; using Moq; using Xunit; namespace JobTrackerApi.Tests; public sealed class JobImportHttpTransportTests { private const string PublicIp = "93.184.216.34"; private const string JobHtml = """ """; public static TheoryData DeniedAddresses => new() { "0.0.0.0", "0.1.2.3", "127.0.0.1", "127.99.1.2", "10.1.2.3", "172.16.0.1", "172.31.255.255", "192.168.1.1", "169.254.169.254", "100.64.0.1", "100.127.255.255", "198.18.0.1", "198.19.255.255", "224.0.0.1", "239.255.255.255", "240.0.0.1", "255.255.255.255", "::", "::1", "fe80::1", "febf::1", "fec0::1", "fc00::1", "fdff::1", "ff02::1", "2001:0000:4136:e378:8000:63bf:3fff:fdd2", // Teredo "::ffff:10.1.2.3", "::ffff:a01:203", "::ffff:127.0.0.1", "::ffff:169.254.169.254", "::ffff:192.168.1.1", "::ffff:0.0.0.0", "::ffff:224.0.0.1", "::10.1.2.3", "64:ff9b::a01:203", "64:ff9b:1::a01:203", "2002:0a01:0203::1", "2606:4700:4700::1111%3" }; [Theory] [MemberData(nameof(DeniedAddresses))] public async Task Handler_rejects_denied_dns_addresses_before_any_socket_attempt(string address) { var resolver = Resolver(address); var attempts = 0; using var handler = JobImportHttpTransport.CreateHandler(resolver.Object, (_, _) => { attempts++; throw new InvalidOperationException("Denied destination reached the socket boundary."); }); using var client = new HttpClient(handler); await Assert.ThrowsAsync(() => client.GetAsync("http://jobs.example.test/job")); Assert.Equal(0, attempts); resolver.Verify(x => x.ResolveAsync("jobs.example.test", It.IsAny()), Times.Once); resolver.VerifyNoOtherCalls(); } [Theory] [InlineData("http://127.0.0.1/")] [InlineData("http://2130706433/")] [InlineData("http://0x7f000001/")] [InlineData("http://0177.0.0.1/")] [InlineData("http://[::1]/")] [InlineData("http://[::ffff:a01:203]/")] [InlineData("http://[::ffff:169.254.169.254]/")] [InlineData("http://[fd00::1]/")] public async Task Handler_rejects_literal_without_dns_or_dial(string url) { var resolver = new Mock(MockBehavior.Strict); var attempts = 0; using var handler = JobImportHttpTransport.CreateHandler(resolver.Object, (_, _) => { attempts++; throw new InvalidOperationException("Unexpected dial."); }); using var client = new HttpClient(handler); await Assert.ThrowsAsync(() => client.GetAsync(url)); Assert.Equal(0, attempts); resolver.VerifyNoOtherCalls(); } [Fact] public async Task Production_dial_path_cannot_reach_a_loopback_listener() { using var listener = new TcpListener(IPAddress.Loopback, 0); listener.Start(); var port = ((IPEndPoint)listener.LocalEndpoint).Port; using var handler = JobImportHttpTransport.CreateHandler(Resolver("127.0.0.1").Object); using var client = new HttpClient(handler); await Assert.ThrowsAsync(() => client.GetAsync($"http://jobs.example.test:{port}/")); Assert.False(listener.Pending()); } [Theory] [InlineData(false)] [InlineData(true)] public async Task Empty_or_mixed_dns_answers_fail_closed(bool mixed) { var resolver = mixed ? Resolver(PublicIp, "::ffff:10.1.2.3") : Resolver(); var attempts = 0; using var handler = JobImportHttpTransport.CreateHandler(resolver.Object, (_, _) => { attempts++; throw new SocketException(); }); using var client = new HttpClient(handler); await Assert.ThrowsAsync(() => client.GetAsync("http://jobs.example.test/")); Assert.Equal(0, attempts); } [Fact] public async Task Dns_failure_fails_closed_without_fallback_resolution() { var resolver = new Mock(MockBehavior.Strict); resolver.Setup(x => x.ResolveAsync(It.IsAny(), It.IsAny())) .ThrowsAsync(new SocketException((int)SocketError.HostNotFound)); var attempts = 0; using var handler = JobImportHttpTransport.CreateHandler(resolver.Object, (_, _) => { attempts++; throw new SocketException(); }); using var client = new HttpClient(handler); await Assert.ThrowsAsync(() => client.GetAsync("http://jobs.example.test/")); Assert.Equal(0, attempts); resolver.Verify(x => x.ResolveAsync(It.IsAny(), It.IsAny()), Times.Once); } [Theory] [InlineData("10.1.2.3")] [InlineData("::ffff:169.254.169.254")] [InlineData("fd00::1")] public async Task Preview_rechecks_changed_dns_at_connection_and_never_dials_private_peer(string nextAddress) { var resolver = new Mock(MockBehavior.Strict); resolver.SetupSequence(x => x.ResolveAsync("jobs.example.test", It.IsAny())) .ReturnsAsync([IPAddress.Parse(PublicIp)]) .ReturnsAsync([IPAddress.Parse(nextAddress)]); var attempts = 0; using var handler = JobImportHttpTransport.CreateHandler(resolver.Object, (_, _) => { attempts++; throw new SocketException(); }); using var client = new HttpClient(handler); await Assert.ThrowsAsync(() => Service(client, resolver.Object) .PreviewAsync("http://jobs.example.test/job", CancellationToken.None)); Assert.Equal(0, attempts); resolver.Verify(x => x.ResolveAsync("jobs.example.test", It.IsAny()), Times.Exactly(2)); } [Theory] [InlineData("jobs.example.test", PublicIp)] [InlineData("jobs.example.test", "2606:4700:4700::1111")] [InlineData("jobs.example.test", "::ffff:93.184.216.34")] [InlineData("93.184.216.34", PublicIp)] [InlineData("[::ffff:93.184.216.34]", PublicIp)] public async Task Public_http_imports_keep_the_original_host_and_parse(string host, string resolvedAddress) { await using var server = new LocalServer((stream, _) => Respond(stream, JobHtml)); var resolver = Resolver(resolvedAddress); using var handler = JobImportHttpTransport.CreateHandler(resolver.Object, server.DialAsync); using var client = new HttpClient(handler); var result = await Service(client, resolver.Object) .PreviewAsync($"http://{host}:{server.Port}/job", CancellationToken.None); Assert.True(result.Success, result.Error); Assert.Equal("Engineer", result.Title); Assert.Contains($"Host: {new Uri($"http://{host}:{server.Port}/").Host}:{server.Port}", server.Request); Assert.Equal(IPAddress.Parse(resolvedAddress).IsIPv4MappedToIPv6 ? IPAddress.Parse(resolvedAddress).MapToIPv4() : IPAddress.Parse(resolvedAddress), Assert.Single(server.Attempts).Address); Assert.False(handler.UseProxy); Assert.False(handler.AllowAutoRedirect); Assert.Null(handler.SslOptions.RemoteCertificateValidationCallback); Assert.Equal(TimeSpan.FromSeconds(100), client.Timeout); } [Fact] public async Task Connection_uses_only_its_checked_answer_even_if_dns_would_change_again() { await using var server = new LocalServer((stream, _) => Respond(stream, JobHtml)); var resolver = new Mock(MockBehavior.Strict); resolver.SetupSequence(x => x.ResolveAsync("jobs.example.test", It.IsAny())) .ReturnsAsync([IPAddress.Parse("1.1.1.1")]) .ReturnsAsync([IPAddress.Parse(PublicIp)]) .ReturnsAsync([IPAddress.Loopback]); using var handler = JobImportHttpTransport.CreateHandler(resolver.Object, server.DialAsync); using var client = new HttpClient(handler); var result = await Service(client, resolver.Object).PreviewAsync($"http://jobs.example.test:{server.Port}/job", default); Assert.True(result.Success, result.Error); Assert.Equal(IPAddress.Parse(PublicIp), Assert.Single(server.Attempts).Address); resolver.Verify(x => x.ResolveAsync("jobs.example.test", It.IsAny()), Times.Exactly(2)); } [Fact] public async Task Failed_public_address_falls_back_only_within_the_checked_answer() { await using var server = new LocalServer((stream, _) => Respond(stream, JobHtml)); var resolver = Resolver("2606:4700:4700::1111", PublicIp); var attempts = new List(); using var handler = JobImportHttpTransport.CreateHandler(resolver.Object, (endpoint, ct) => { attempts.Add(endpoint); if (attempts.Count == 1) throw new SocketException((int)SocketError.NetworkUnreachable); return server.DialAsync(endpoint, ct); }); using var client = new HttpClient(handler); using var response = await client.GetAsync($"http://jobs.example.test:{server.Port}/job"); Assert.Equal(HttpStatusCode.OK, response.StatusCode); Assert.Equal(2, attempts.Count); Assert.Equal(IPAddress.Parse(PublicIp), attempts[1].Address); resolver.Verify(x => x.ResolveAsync(It.IsAny(), It.IsAny()), Times.Once); } [Fact] public async Task A_new_connection_on_the_same_handler_revalidates_dns() { await using var server = new LocalServer((stream, _) => Respond(stream, JobHtml)); var resolver = new Mock(MockBehavior.Strict); resolver.SetupSequence(x => x.ResolveAsync("jobs.example.test", It.IsAny())) .ReturnsAsync([IPAddress.Parse(PublicIp)]) .ReturnsAsync([IPAddress.Parse("::ffff:10.1.2.3")]); using var handler = JobImportHttpTransport.CreateHandler(resolver.Object, server.DialAsync); using var client = new HttpClient(handler); var url = $"http://jobs.example.test:{server.Port}/job"; Assert.Contains("Engineer", await client.GetStringAsync(url)); // Server closes its connection. await Assert.ThrowsAsync(() => client.GetStringAsync(url)); Assert.Single(server.Attempts); resolver.Verify(x => x.ResolveAsync("jobs.example.test", It.IsAny()), Times.Exactly(2)); } [Fact] public async Task Proxy_configuration_cannot_create_an_unchecked_origin_connection() { using var proxyTrap = new TcpListener(IPAddress.Loopback, 0); proxyTrap.Start(); await using var server = new LocalServer((stream, _) => Respond(stream, JobHtml)); var resolver = Resolver(PublicIp); using var handler = JobImportHttpTransport.CreateHandler(resolver.Object, server.DialAsync); handler.Proxy = new WebProxy($"http://127.0.0.1:{((IPEndPoint)proxyTrap.LocalEndpoint).Port}"); using var client = new HttpClient(handler); Assert.Contains("Engineer", await client.GetStringAsync($"http://jobs.example.test:{server.Port}/job")); Assert.False(proxyTrap.Pending()); Assert.Equal(IPAddress.Parse(PublicIp), Assert.Single(server.Attempts).Address); resolver.Verify(x => x.ResolveAsync("jobs.example.test", It.IsAny()), Times.Once); } [Theory] [InlineData(301)] [InlineData(302)] [InlineData(303)] [InlineData(307)] [InlineData(308)] public async Task Redirect_to_private_destination_is_returned_as_fetch_failure_without_following(int status) { await using var server = new LocalServer((stream, _) => Write(stream, $"HTTP/1.1 {status} Redirect\r\nLocation: http://[::ffff:169.254.169.254]/secret\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")); var resolver = Resolver(PublicIp); using var handler = JobImportHttpTransport.CreateHandler(resolver.Object, server.DialAsync); using var client = new HttpClient(handler); var result = await Service(client, resolver.Object).PreviewAsync($"http://jobs.example.test:{server.Port}/job", default); Assert.False(result.Success); Assert.Equal("fetch", result.Parser); Assert.Single(server.Attempts); resolver.Verify(x => x.ResolveAsync("jobs.example.test", It.IsAny()), Times.Exactly(2)); resolver.VerifyNoOtherCalls(); } [Theory] [InlineData("jobs.example.test", true, true)] [InlineData("other.example.test", true, false)] [InlineData("jobs.example.test", false, false)] public async Task Https_keeps_sni_and_requires_trusted_certificate_for_original_hostname(string certificateHost, bool trustRoot, bool succeeds) { using var certificates = new TestCertificates(certificateHost); string? sni = null; await using var server = new LocalServer((stream, _) => Respond(stream, JobHtml), async (stream, ct) => { var tls = new SslStream(stream); try { await tls.AuthenticateAsServerAsync(new SslServerAuthenticationOptions { ServerCertificateSelectionCallback = (_, host) => { sni = host; return certificates.Server; }, EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13 }, ct); return tls; } catch { await tls.DisposeAsync(); throw; } }); var resolver = Resolver(PublicIp); using var handler = JobImportHttpTransport.CreateHandler(resolver.Object, server.DialAsync); if (trustRoot) { // Trust only this fixture CA without changing OS trust or bypassing hostname validation. handler.SslOptions.CertificateChainPolicy = new X509ChainPolicy { TrustMode = X509ChainTrustMode.CustomRootTrust, RevocationMode = X509RevocationMode.NoCheck }; handler.SslOptions.CertificateChainPolicy.CustomTrustStore.Add(certificates.Root); } using var client = new HttpClient(handler) { Timeout = TimeSpan.FromSeconds(10) }; var request = () => Service(client, resolver.Object).PreviewAsync($"https://jobs.example.test:{server.Port}/job", default); if (succeeds) { var result = await request(); Assert.True(result.Success, result.Error); Assert.Contains($"Host: jobs.example.test:{server.Port}", server.Request); } else { var error = await Assert.ThrowsAsync(request); Assert.IsType(error.InnerException); // TLS rejection can surface as EOF after the server handshake on Linux. // Both no read and a zero-byte read mean no HTTP data reached this origin. Assert.True(string.IsNullOrEmpty(server.Request)); } Assert.Equal("jobs.example.test", sni); Assert.Null(handler.SslOptions.RemoteCertificateValidationCallback); Assert.Equal(IPAddress.Parse(PublicIp), Assert.Single(server.Attempts).Address); } [Theory] [InlineData("dns")] [InlineData("connect")] [InlineData("headers")] public async Task Http_client_timeout_cancels_resolution_connect_and_headers(string phase) { await using var server = new LocalServer((_, ct) => Task.Delay(Timeout.Infinite, ct)); var resolver = Resolver(PublicIp); if (phase == "dns") resolver.Setup(x => x.ResolveAsync(It.IsAny(), It.IsAny())) .Returns(async (string _, CancellationToken ct) => { await Task.Delay(Timeout.Infinite, ct); return Array.Empty(); }); using var handler = JobImportHttpTransport.CreateHandler(resolver.Object, async (endpoint, ct) => { if (phase == "connect") await Task.Delay(Timeout.Infinite, ct); return await server.DialAsync(endpoint, ct); }); using var client = new HttpClient(handler) { Timeout = TimeSpan.FromMilliseconds(300) }; var error = await Assert.ThrowsAnyAsync(() => client.GetAsync($"http://jobs.example.test:{server.Port}/job").WaitAsync(TimeSpan.FromSeconds(5))); Assert.IsType(error.InnerException); } [Fact] public async Task Caller_cancellation_interrupts_a_stalled_streamed_body() { var bodyStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); await using var server = new LocalServer(async (stream, ct) => { await Write(stream, "HTTP/1.1 200 OK\r\nContent-Length: 100\r\nConnection: close\r\n\r\n"); bodyStarted.SetResult(); await Task.Delay(Timeout.Infinite, ct); }); var resolver = Resolver(PublicIp); using var handler = JobImportHttpTransport.CreateHandler(resolver.Object, server.DialAsync); using var client = new HttpClient(handler); using var cancellation = new CancellationTokenSource(); var preview = Service(client, resolver.Object).PreviewAsync($"http://jobs.example.test:{server.Port}/job", cancellation.Token); await bodyStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); cancellation.Cancel(); await Assert.ThrowsAnyAsync(() => preview.WaitAsync(TimeSpan.FromSeconds(5))); } [Theory] [InlineData(false)] [InlineData(true)] public async Task Size_limit_rejects_oversized_declared_and_decompressed_responses(bool compressed) { var payload = Encoding.UTF8.GetBytes(new string('x', 4_000_001)); if (compressed) { using var buffer = new MemoryStream(); using (var gzip = new GZipStream(buffer, CompressionLevel.Fastest, leaveOpen: true)) gzip.Write(payload); payload = buffer.ToArray(); } await using var server = new LocalServer(async (stream, _) => { await Write(stream, $"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {payload.Length}\r\n" + (compressed ? "Content-Encoding: gzip\r\n" : "") + "Connection: close\r\n\r\n"); if (compressed) await stream.WriteAsync(payload); }); var resolver = Resolver(PublicIp); using var handler = JobImportHttpTransport.CreateHandler(resolver.Object, server.DialAsync); using var client = new HttpClient(handler); var result = await Service(client, resolver.Object).PreviewAsync($"http://jobs.example.test:{server.Port}/job", default); Assert.False(result.Success); Assert.Equal("fetch", result.Parser); } private static Mock Resolver(params string[] addresses) { var resolver = new Mock(MockBehavior.Strict); resolver.Setup(x => x.ResolveAsync(It.IsAny(), It.IsAny())) .ReturnsAsync(addresses.Select(IPAddress.Parse).ToArray()); return resolver; } private static JobImportService Service(HttpClient client, IHostAddressResolver resolver) { var factory = new Mock(); factory.Setup(x => x.CreateClient("jobimport")).Returns(client); return new JobImportService(factory.Object, new UniversalJobParser(), [], new NoOpTranslationService(), resolver); } private static Task Respond(Stream stream, string body) => Write(stream, $"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {Encoding.UTF8.GetByteCount(body)}\r\nConnection: close\r\n\r\n{body}"); private static Task Write(Stream stream, string text) => stream.WriteAsync(Encoding.UTF8.GetBytes(text)).AsTask(); // Only this test dialer substitutes a loopback peer AFTER the production callback approves // an IP. No test performs public DNS/network I/O or makes private IPs allowed in production. private sealed class LocalServer : IAsyncDisposable { private readonly TcpListener _listener = new(IPAddress.Loopback, 0); private readonly CancellationTokenSource _stop = new(TimeSpan.FromSeconds(10)); private readonly Task _serve; internal int Port => ((IPEndPoint)_listener.LocalEndpoint).Port; internal string? Request { get; private set; } internal List Attempts { get; } = []; internal LocalServer(Func respond, Func>? secure = null) { _listener.Start(); _serve = ServeAsync(respond, secure); } internal ValueTask DialAsync(IPEndPoint approved, CancellationToken ct) { Assert.False(JobImportHttpTransport.IsBlockedAddress(approved.Address)); Assert.Equal(Port, approved.Port); Attempts.Add(approved); return JobImportHttpTransport.ConnectSocketAsync(new IPEndPoint(IPAddress.Loopback, Port), ct); } private async Task ServeAsync(Func respond, Func>? secure) { using var socket = await _listener.AcceptTcpClientAsync(_stop.Token); await using var stream = secure is null ? socket.GetStream() : await secure(socket.GetStream(), _stop.Token); using var reader = new StreamReader(stream, Encoding.ASCII, leaveOpen: true); var headers = new StringBuilder(); while (await reader.ReadLineAsync(_stop.Token) is { Length: > 0 } line) headers.AppendLine(line); Request = headers.ToString(); await respond(stream, _stop.Token); } public async ValueTask DisposeAsync() { await _stop.CancelAsync(); _listener.Stop(); try { await _serve; } catch (OperationCanceledException) { } catch (IOException) { } // Client may reject TLS or stop reading at its byte limit. catch (AuthenticationException) { } finally { _stop.Dispose(); } } } private sealed class TestCertificates : IDisposable { internal X509Certificate2 Root { get; } internal X509Certificate2 Server { get; } internal TestCertificates(string hostname) { using var rootKey = RSA.Create(2048); var root = new CertificateRequest("CN=B01 isolated test CA", rootKey, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); root.CertificateExtensions.Add(new X509BasicConstraintsExtension(true, false, 0, true)); root.CertificateExtensions.Add(new X509KeyUsageExtension(X509KeyUsageFlags.KeyCertSign, true)); Root = root.CreateSelfSigned(DateTimeOffset.UtcNow.AddDays(-1), DateTimeOffset.UtcNow.AddDays(1)); using var serverKey = RSA.Create(2048); var server = new CertificateRequest($"CN={hostname}", serverKey, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); var san = new SubjectAlternativeNameBuilder(); san.AddDnsName(hostname); server.CertificateExtensions.Add(san.Build()); server.CertificateExtensions.Add(new X509EnhancedKeyUsageExtension(new OidCollection { new("1.3.6.1.5.5.7.3.1") }, true)); using var signed = server.Create(Root, DateTimeOffset.UtcNow.AddHours(-1), DateTimeOffset.UtcNow.AddHours(1), RandomNumberGenerator.GetBytes(16)); using var withKey = signed.CopyWithPrivateKey(serverKey); // Schannel needs a persisted private key for a local TLS server on Windows. Server = X509CertificateLoader.LoadPkcs12(withKey.Export(X509ContentType.Pfx), null); } public void Dispose() { Server.Dispose(); Root.Dispose(); } } }