Files
jobtrackingapp/JobTrackerApi.Tests/JobImportHttpTransportTests.cs
T
cesnimda 0954876295 fix(security): enforce audited account, network and resource boundaries
Implement B01-B06 with regression coverage and retained validation evidence. Preserve the production ingress preflight; F04 runtime verification remains open.
2026-09-09 18:21:47 +02:00

514 lines
24 KiB
C#

using System.IO.Compression;
using System.Net;
using System.Net.Security;
using System.Net.Sockets;
using System.Security.Authentication;
using System.Security.Cryptography;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using JobTrackerApi.Services.JobImport;
using JobTrackerApi.Services.JobImport.Translation;
using Moq;
using Xunit;
namespace JobTrackerApi.Tests;
public sealed class JobImportHttpTransportTests
{
private const string PublicIp = "93.184.216.34";
private const string JobHtml = """
<html><script type="application/ld+json">
{"@type":"JobPosting","title":"Engineer","description":"Build useful software.","hiringOrganization":{"name":"Example"}}
</script></html>
""";
public static TheoryData<string> DeniedAddresses => new()
{
"0.0.0.0", "0.1.2.3", "127.0.0.1", "127.99.1.2", "10.1.2.3",
"172.16.0.1", "172.31.255.255", "192.168.1.1", "169.254.169.254",
"100.64.0.1", "100.127.255.255", "198.18.0.1", "198.19.255.255",
"224.0.0.1", "239.255.255.255", "240.0.0.1", "255.255.255.255",
"::", "::1", "fe80::1", "febf::1", "fec0::1", "fc00::1", "fdff::1", "ff02::1",
"2001:0000:4136:e378:8000:63bf:3fff:fdd2", // Teredo
"::ffff:10.1.2.3", "::ffff:a01:203", "::ffff:127.0.0.1", "::ffff:169.254.169.254",
"::ffff:192.168.1.1", "::ffff:0.0.0.0", "::ffff:224.0.0.1",
"::10.1.2.3", "64:ff9b::a01:203", "64:ff9b:1::a01:203", "2002:0a01:0203::1",
"2606:4700:4700::1111%3"
};
[Theory]
[MemberData(nameof(DeniedAddresses))]
public async Task Handler_rejects_denied_dns_addresses_before_any_socket_attempt(string address)
{
var resolver = Resolver(address);
var attempts = 0;
using var handler = JobImportHttpTransport.CreateHandler(resolver.Object, (_, _) =>
{
attempts++;
throw new InvalidOperationException("Denied destination reached the socket boundary.");
});
using var client = new HttpClient(handler);
await Assert.ThrowsAsync<HttpRequestException>(() => client.GetAsync("http://jobs.example.test/job"));
Assert.Equal(0, attempts);
resolver.Verify(x => x.ResolveAsync("jobs.example.test", It.IsAny<CancellationToken>()), Times.Once);
resolver.VerifyNoOtherCalls();
}
[Theory]
[InlineData("http://127.0.0.1/")]
[InlineData("http://2130706433/")]
[InlineData("http://0x7f000001/")]
[InlineData("http://0177.0.0.1/")]
[InlineData("http://[::1]/")]
[InlineData("http://[::ffff:a01:203]/")]
[InlineData("http://[::ffff:169.254.169.254]/")]
[InlineData("http://[fd00::1]/")]
public async Task Handler_rejects_literal_without_dns_or_dial(string url)
{
var resolver = new Mock<IHostAddressResolver>(MockBehavior.Strict);
var attempts = 0;
using var handler = JobImportHttpTransport.CreateHandler(resolver.Object, (_, _) =>
{
attempts++;
throw new InvalidOperationException("Unexpected dial.");
});
using var client = new HttpClient(handler);
await Assert.ThrowsAsync<HttpRequestException>(() => client.GetAsync(url));
Assert.Equal(0, attempts);
resolver.VerifyNoOtherCalls();
}
[Fact]
public async Task Production_dial_path_cannot_reach_a_loopback_listener()
{
using var listener = new TcpListener(IPAddress.Loopback, 0);
listener.Start();
var port = ((IPEndPoint)listener.LocalEndpoint).Port;
using var handler = JobImportHttpTransport.CreateHandler(Resolver("127.0.0.1").Object);
using var client = new HttpClient(handler);
await Assert.ThrowsAsync<HttpRequestException>(() => client.GetAsync($"http://jobs.example.test:{port}/"));
Assert.False(listener.Pending());
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Empty_or_mixed_dns_answers_fail_closed(bool mixed)
{
var resolver = mixed ? Resolver(PublicIp, "::ffff:10.1.2.3") : Resolver();
var attempts = 0;
using var handler = JobImportHttpTransport.CreateHandler(resolver.Object, (_, _) =>
{
attempts++;
throw new SocketException();
});
using var client = new HttpClient(handler);
await Assert.ThrowsAsync<HttpRequestException>(() => client.GetAsync("http://jobs.example.test/"));
Assert.Equal(0, attempts);
}
[Fact]
public async Task Dns_failure_fails_closed_without_fallback_resolution()
{
var resolver = new Mock<IHostAddressResolver>(MockBehavior.Strict);
resolver.Setup(x => x.ResolveAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()))
.ThrowsAsync(new SocketException((int)SocketError.HostNotFound));
var attempts = 0;
using var handler = JobImportHttpTransport.CreateHandler(resolver.Object, (_, _) =>
{
attempts++;
throw new SocketException();
});
using var client = new HttpClient(handler);
await Assert.ThrowsAsync<HttpRequestException>(() => client.GetAsync("http://jobs.example.test/"));
Assert.Equal(0, attempts);
resolver.Verify(x => x.ResolveAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()), Times.Once);
}
[Theory]
[InlineData("10.1.2.3")]
[InlineData("::ffff:169.254.169.254")]
[InlineData("fd00::1")]
public async Task Preview_rechecks_changed_dns_at_connection_and_never_dials_private_peer(string nextAddress)
{
var resolver = new Mock<IHostAddressResolver>(MockBehavior.Strict);
resolver.SetupSequence(x => x.ResolveAsync("jobs.example.test", It.IsAny<CancellationToken>()))
.ReturnsAsync([IPAddress.Parse(PublicIp)])
.ReturnsAsync([IPAddress.Parse(nextAddress)]);
var attempts = 0;
using var handler = JobImportHttpTransport.CreateHandler(resolver.Object, (_, _) =>
{
attempts++;
throw new SocketException();
});
using var client = new HttpClient(handler);
await Assert.ThrowsAsync<HttpRequestException>(() => Service(client, resolver.Object)
.PreviewAsync("http://jobs.example.test/job", CancellationToken.None));
Assert.Equal(0, attempts);
resolver.Verify(x => x.ResolveAsync("jobs.example.test", It.IsAny<CancellationToken>()), Times.Exactly(2));
}
[Theory]
[InlineData("jobs.example.test", PublicIp)]
[InlineData("jobs.example.test", "2606:4700:4700::1111")]
[InlineData("jobs.example.test", "::ffff:93.184.216.34")]
[InlineData("93.184.216.34", PublicIp)]
[InlineData("[::ffff:93.184.216.34]", PublicIp)]
public async Task Public_http_imports_keep_the_original_host_and_parse(string host, string resolvedAddress)
{
await using var server = new LocalServer((stream, _) => Respond(stream, JobHtml));
var resolver = Resolver(resolvedAddress);
using var handler = JobImportHttpTransport.CreateHandler(resolver.Object, server.DialAsync);
using var client = new HttpClient(handler);
var result = await Service(client, resolver.Object)
.PreviewAsync($"http://{host}:{server.Port}/job", CancellationToken.None);
Assert.True(result.Success, result.Error);
Assert.Equal("Engineer", result.Title);
Assert.Contains($"Host: {new Uri($"http://{host}:{server.Port}/").Host}:{server.Port}", server.Request);
Assert.Equal(IPAddress.Parse(resolvedAddress).IsIPv4MappedToIPv6
? IPAddress.Parse(resolvedAddress).MapToIPv4() : IPAddress.Parse(resolvedAddress), Assert.Single(server.Attempts).Address);
Assert.False(handler.UseProxy);
Assert.False(handler.AllowAutoRedirect);
Assert.Null(handler.SslOptions.RemoteCertificateValidationCallback);
Assert.Equal(TimeSpan.FromSeconds(100), client.Timeout);
}
[Fact]
public async Task Connection_uses_only_its_checked_answer_even_if_dns_would_change_again()
{
await using var server = new LocalServer((stream, _) => Respond(stream, JobHtml));
var resolver = new Mock<IHostAddressResolver>(MockBehavior.Strict);
resolver.SetupSequence(x => x.ResolveAsync("jobs.example.test", It.IsAny<CancellationToken>()))
.ReturnsAsync([IPAddress.Parse("1.1.1.1")])
.ReturnsAsync([IPAddress.Parse(PublicIp)])
.ReturnsAsync([IPAddress.Loopback]);
using var handler = JobImportHttpTransport.CreateHandler(resolver.Object, server.DialAsync);
using var client = new HttpClient(handler);
var result = await Service(client, resolver.Object).PreviewAsync($"http://jobs.example.test:{server.Port}/job", default);
Assert.True(result.Success, result.Error);
Assert.Equal(IPAddress.Parse(PublicIp), Assert.Single(server.Attempts).Address);
resolver.Verify(x => x.ResolveAsync("jobs.example.test", It.IsAny<CancellationToken>()), Times.Exactly(2));
}
[Fact]
public async Task Failed_public_address_falls_back_only_within_the_checked_answer()
{
await using var server = new LocalServer((stream, _) => Respond(stream, JobHtml));
var resolver = Resolver("2606:4700:4700::1111", PublicIp);
var attempts = new List<IPEndPoint>();
using var handler = JobImportHttpTransport.CreateHandler(resolver.Object, (endpoint, ct) =>
{
attempts.Add(endpoint);
if (attempts.Count == 1) throw new SocketException((int)SocketError.NetworkUnreachable);
return server.DialAsync(endpoint, ct);
});
using var client = new HttpClient(handler);
using var response = await client.GetAsync($"http://jobs.example.test:{server.Port}/job");
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
Assert.Equal(2, attempts.Count);
Assert.Equal(IPAddress.Parse(PublicIp), attempts[1].Address);
resolver.Verify(x => x.ResolveAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()), Times.Once);
}
[Fact]
public async Task A_new_connection_on_the_same_handler_revalidates_dns()
{
await using var server = new LocalServer((stream, _) => Respond(stream, JobHtml));
var resolver = new Mock<IHostAddressResolver>(MockBehavior.Strict);
resolver.SetupSequence(x => x.ResolveAsync("jobs.example.test", It.IsAny<CancellationToken>()))
.ReturnsAsync([IPAddress.Parse(PublicIp)])
.ReturnsAsync([IPAddress.Parse("::ffff:10.1.2.3")]);
using var handler = JobImportHttpTransport.CreateHandler(resolver.Object, server.DialAsync);
using var client = new HttpClient(handler);
var url = $"http://jobs.example.test:{server.Port}/job";
Assert.Contains("Engineer", await client.GetStringAsync(url)); // Server closes its connection.
await Assert.ThrowsAsync<HttpRequestException>(() => client.GetStringAsync(url));
Assert.Single(server.Attempts);
resolver.Verify(x => x.ResolveAsync("jobs.example.test", It.IsAny<CancellationToken>()), Times.Exactly(2));
}
[Fact]
public async Task Proxy_configuration_cannot_create_an_unchecked_origin_connection()
{
using var proxyTrap = new TcpListener(IPAddress.Loopback, 0);
proxyTrap.Start();
await using var server = new LocalServer((stream, _) => Respond(stream, JobHtml));
var resolver = Resolver(PublicIp);
using var handler = JobImportHttpTransport.CreateHandler(resolver.Object, server.DialAsync);
handler.Proxy = new WebProxy($"http://127.0.0.1:{((IPEndPoint)proxyTrap.LocalEndpoint).Port}");
using var client = new HttpClient(handler);
Assert.Contains("Engineer", await client.GetStringAsync($"http://jobs.example.test:{server.Port}/job"));
Assert.False(proxyTrap.Pending());
Assert.Equal(IPAddress.Parse(PublicIp), Assert.Single(server.Attempts).Address);
resolver.Verify(x => x.ResolveAsync("jobs.example.test", It.IsAny<CancellationToken>()), Times.Once);
}
[Theory]
[InlineData(301)]
[InlineData(302)]
[InlineData(303)]
[InlineData(307)]
[InlineData(308)]
public async Task Redirect_to_private_destination_is_returned_as_fetch_failure_without_following(int status)
{
await using var server = new LocalServer((stream, _) => Write(stream,
$"HTTP/1.1 {status} Redirect\r\nLocation: http://[::ffff:169.254.169.254]/secret\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"));
var resolver = Resolver(PublicIp);
using var handler = JobImportHttpTransport.CreateHandler(resolver.Object, server.DialAsync);
using var client = new HttpClient(handler);
var result = await Service(client, resolver.Object).PreviewAsync($"http://jobs.example.test:{server.Port}/job", default);
Assert.False(result.Success);
Assert.Equal("fetch", result.Parser);
Assert.Single(server.Attempts);
resolver.Verify(x => x.ResolveAsync("jobs.example.test", It.IsAny<CancellationToken>()), Times.Exactly(2));
resolver.VerifyNoOtherCalls();
}
[Theory]
[InlineData("jobs.example.test", true, true)]
[InlineData("other.example.test", true, false)]
[InlineData("jobs.example.test", false, false)]
public async Task Https_keeps_sni_and_requires_trusted_certificate_for_original_hostname(string certificateHost, bool trustRoot, bool succeeds)
{
using var certificates = new TestCertificates(certificateHost);
string? sni = null;
await using var server = new LocalServer((stream, _) => Respond(stream, JobHtml), async (stream, ct) =>
{
var tls = new SslStream(stream);
try
{
await tls.AuthenticateAsServerAsync(new SslServerAuthenticationOptions
{
ServerCertificateSelectionCallback = (_, host) => { sni = host; return certificates.Server; },
EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13
}, ct);
return tls;
}
catch { await tls.DisposeAsync(); throw; }
});
var resolver = Resolver(PublicIp);
using var handler = JobImportHttpTransport.CreateHandler(resolver.Object, server.DialAsync);
if (trustRoot)
{
// Trust only this fixture CA without changing OS trust or bypassing hostname validation.
handler.SslOptions.CertificateChainPolicy = new X509ChainPolicy
{
TrustMode = X509ChainTrustMode.CustomRootTrust,
RevocationMode = X509RevocationMode.NoCheck
};
handler.SslOptions.CertificateChainPolicy.CustomTrustStore.Add(certificates.Root);
}
using var client = new HttpClient(handler) { Timeout = TimeSpan.FromSeconds(10) };
var request = () => Service(client, resolver.Object).PreviewAsync($"https://jobs.example.test:{server.Port}/job", default);
if (succeeds)
{
var result = await request();
Assert.True(result.Success, result.Error);
Assert.Contains($"Host: jobs.example.test:{server.Port}", server.Request);
}
else
{
var error = await Assert.ThrowsAsync<HttpRequestException>(request);
Assert.IsType<AuthenticationException>(error.InnerException);
Assert.Null(server.Request); // No HTTP data sent before TLS authenticates the origin.
}
Assert.Equal("jobs.example.test", sni);
Assert.Null(handler.SslOptions.RemoteCertificateValidationCallback);
Assert.Equal(IPAddress.Parse(PublicIp), Assert.Single(server.Attempts).Address);
}
[Theory]
[InlineData("dns")]
[InlineData("connect")]
[InlineData("headers")]
public async Task Http_client_timeout_cancels_resolution_connect_and_headers(string phase)
{
await using var server = new LocalServer((_, ct) => Task.Delay(Timeout.Infinite, ct));
var resolver = Resolver(PublicIp);
if (phase == "dns")
resolver.Setup(x => x.ResolveAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()))
.Returns(async (string _, CancellationToken ct) => { await Task.Delay(Timeout.Infinite, ct); return Array.Empty<IPAddress>(); });
using var handler = JobImportHttpTransport.CreateHandler(resolver.Object, async (endpoint, ct) =>
{
if (phase == "connect") await Task.Delay(Timeout.Infinite, ct);
return await server.DialAsync(endpoint, ct);
});
using var client = new HttpClient(handler) { Timeout = TimeSpan.FromMilliseconds(300) };
var error = await Assert.ThrowsAnyAsync<OperationCanceledException>(() =>
client.GetAsync($"http://jobs.example.test:{server.Port}/job").WaitAsync(TimeSpan.FromSeconds(5)));
Assert.IsType<TimeoutException>(error.InnerException);
}
[Fact]
public async Task Caller_cancellation_interrupts_a_stalled_streamed_body()
{
var bodyStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
await using var server = new LocalServer(async (stream, ct) =>
{
await Write(stream, "HTTP/1.1 200 OK\r\nContent-Length: 100\r\nConnection: close\r\n\r\n");
bodyStarted.SetResult();
await Task.Delay(Timeout.Infinite, ct);
});
var resolver = Resolver(PublicIp);
using var handler = JobImportHttpTransport.CreateHandler(resolver.Object, server.DialAsync);
using var client = new HttpClient(handler);
using var cancellation = new CancellationTokenSource();
var preview = Service(client, resolver.Object).PreviewAsync($"http://jobs.example.test:{server.Port}/job", cancellation.Token);
await bodyStarted.Task.WaitAsync(TimeSpan.FromSeconds(5));
cancellation.Cancel();
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => preview.WaitAsync(TimeSpan.FromSeconds(5)));
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Size_limit_rejects_oversized_declared_and_decompressed_responses(bool compressed)
{
var payload = Encoding.UTF8.GetBytes(new string('x', 4_000_001));
if (compressed)
{
using var buffer = new MemoryStream();
using (var gzip = new GZipStream(buffer, CompressionLevel.Fastest, leaveOpen: true)) gzip.Write(payload);
payload = buffer.ToArray();
}
await using var server = new LocalServer(async (stream, _) =>
{
await Write(stream, $"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {payload.Length}\r\n" +
(compressed ? "Content-Encoding: gzip\r\n" : "") + "Connection: close\r\n\r\n");
if (compressed) await stream.WriteAsync(payload);
});
var resolver = Resolver(PublicIp);
using var handler = JobImportHttpTransport.CreateHandler(resolver.Object, server.DialAsync);
using var client = new HttpClient(handler);
var result = await Service(client, resolver.Object).PreviewAsync($"http://jobs.example.test:{server.Port}/job", default);
Assert.False(result.Success);
Assert.Equal("fetch", result.Parser);
}
private static Mock<IHostAddressResolver> Resolver(params string[] addresses)
{
var resolver = new Mock<IHostAddressResolver>(MockBehavior.Strict);
resolver.Setup(x => x.ResolveAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(addresses.Select(IPAddress.Parse).ToArray());
return resolver;
}
private static JobImportService Service(HttpClient client, IHostAddressResolver resolver)
{
var factory = new Mock<IHttpClientFactory>();
factory.Setup(x => x.CreateClient("jobimport")).Returns(client);
return new JobImportService(factory.Object, new UniversalJobParser(), [], new NoOpTranslationService(), resolver);
}
private static Task Respond(Stream stream, string body) => Write(stream,
$"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {Encoding.UTF8.GetByteCount(body)}\r\nConnection: close\r\n\r\n{body}");
private static Task Write(Stream stream, string text) => stream.WriteAsync(Encoding.UTF8.GetBytes(text)).AsTask();
// Only this test dialer substitutes a loopback peer AFTER the production callback approves
// an IP. No test performs public DNS/network I/O or makes private IPs allowed in production.
private sealed class LocalServer : IAsyncDisposable
{
private readonly TcpListener _listener = new(IPAddress.Loopback, 0);
private readonly CancellationTokenSource _stop = new(TimeSpan.FromSeconds(10));
private readonly Task _serve;
internal int Port => ((IPEndPoint)_listener.LocalEndpoint).Port;
internal string? Request { get; private set; }
internal List<IPEndPoint> Attempts { get; } = [];
internal LocalServer(Func<Stream, CancellationToken, Task> respond, Func<Stream, CancellationToken, Task<Stream>>? secure = null)
{
_listener.Start();
_serve = ServeAsync(respond, secure);
}
internal ValueTask<Stream> DialAsync(IPEndPoint approved, CancellationToken ct)
{
Assert.False(JobImportHttpTransport.IsBlockedAddress(approved.Address));
Assert.Equal(Port, approved.Port);
Attempts.Add(approved);
return JobImportHttpTransport.ConnectSocketAsync(new IPEndPoint(IPAddress.Loopback, Port), ct);
}
private async Task ServeAsync(Func<Stream, CancellationToken, Task> respond, Func<Stream, CancellationToken, Task<Stream>>? secure)
{
using var socket = await _listener.AcceptTcpClientAsync(_stop.Token);
await using var stream = secure is null ? socket.GetStream() : await secure(socket.GetStream(), _stop.Token);
using var reader = new StreamReader(stream, Encoding.ASCII, leaveOpen: true);
var headers = new StringBuilder();
while (await reader.ReadLineAsync(_stop.Token) is { Length: > 0 } line) headers.AppendLine(line);
Request = headers.ToString();
await respond(stream, _stop.Token);
}
public async ValueTask DisposeAsync()
{
await _stop.CancelAsync();
_listener.Stop();
try { await _serve; }
catch (OperationCanceledException) { }
catch (IOException) { } // Client may reject TLS or stop reading at its byte limit.
catch (AuthenticationException) { }
finally { _stop.Dispose(); }
}
}
private sealed class TestCertificates : IDisposable
{
internal X509Certificate2 Root { get; }
internal X509Certificate2 Server { get; }
internal TestCertificates(string hostname)
{
using var rootKey = RSA.Create(2048);
var root = new CertificateRequest("CN=B01 isolated test CA", rootKey, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1);
root.CertificateExtensions.Add(new X509BasicConstraintsExtension(true, false, 0, true));
root.CertificateExtensions.Add(new X509KeyUsageExtension(X509KeyUsageFlags.KeyCertSign, true));
Root = root.CreateSelfSigned(DateTimeOffset.UtcNow.AddDays(-1), DateTimeOffset.UtcNow.AddDays(1));
using var serverKey = RSA.Create(2048);
var server = new CertificateRequest($"CN={hostname}", serverKey, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1);
var san = new SubjectAlternativeNameBuilder();
san.AddDnsName(hostname);
server.CertificateExtensions.Add(san.Build());
server.CertificateExtensions.Add(new X509EnhancedKeyUsageExtension(new OidCollection { new("1.3.6.1.5.5.7.3.1") }, true));
using var signed = server.Create(Root, DateTimeOffset.UtcNow.AddHours(-1), DateTimeOffset.UtcNow.AddHours(1), RandomNumberGenerator.GetBytes(16));
using var withKey = signed.CopyWithPrivateKey(serverKey);
// Schannel needs a persisted private key for a local TLS server on Windows.
Server = X509CertificateLoader.LoadPkcs12(withKey.Export(X509ContentType.Pfx), null);
}
public void Dispose() { Server.Dispose(); Root.Dispose(); }
}
}