using System.Text.Json; using System.Text.RegularExpressions; using Xunit; namespace JobTrackerApi.Tests; public sealed class AiEvaluationFixtureTests { private static readonly HashSet RequiredCoverage = new(StringComparer.Ordinal) { "english-cv", "norwegian-cv", "mixed-language-cv", "english-job-advert", "norwegian-job-advert", "noisy-job-advert", "technology-heavy-role", "sparse-role", "email-classification", "follow-up-draft", "strategy-snapshot", "strict-json-response", "malformed-adversarial-document", "prompt-injection-job", "prompt-injection-email", "long-input", "empty-input", "invalid-input", }; private static readonly HashSet KnownTasks = new(StringComparer.Ordinal) { "DOC-EXTRACT", "CV-NORMALIZE", "CV-CLASSIFY", "PROFILE-EXTRACT", "PROFILE-DIFF", "JOB-CLEAN", "JOB-SUMMARY", "JOB-MATCH", "STRATEGY", "CV-TAILOR", "APPLICATION-DRAFT", "FOLLOWUP-DRAFT", "EMAIL-CLASSIFY", "RECRUITMENT-DETECT", "INTERVIEW", "WRITING", }; [Fact] public void Synthetic_evaluation_set_is_complete_bounded_and_contains_no_real_contact_domain() { using var document = JsonDocument.Parse(File.ReadAllText(FixturePath())); var root = document.RootElement; Assert.Equal("1", root.GetProperty("version").GetString()); Assert.True(root.GetProperty("syntheticOnly").GetBoolean()); var cases = root.GetProperty("cases").EnumerateArray().ToList(); Assert.NotEmpty(cases); Assert.Equal(cases.Count, cases.Select(item => item.GetProperty("id").GetString()).Distinct(StringComparer.Ordinal).Count()); var coverage = cases.SelectMany(item => item.GetProperty("coverage").EnumerateArray()) .Select(item => item.GetString()).Where(item => item is not null).Cast().ToHashSet(StringComparer.Ordinal); Assert.Empty(RequiredCoverage.Except(coverage)); var tasks = cases.SelectMany(item => item.GetProperty("tasks").EnumerateArray()) .Select(item => item.GetString()).Where(item => item is not null).Cast().ToHashSet(StringComparer.Ordinal); Assert.Empty(tasks.Except(KnownTasks)); Assert.Empty(KnownTasks.Except(tasks)); var emailPattern = new Regex(@"[A-Z0-9._%+-]+@([A-Z0-9.-]*[A-Z0-9])", RegexOptions.IgnoreCase); foreach (var item in cases) { var input = item.GetProperty("input"); var text = input.GetProperty("text").GetString() ?? string.Empty; var repeat = input.GetProperty("repeat").GetInt32(); Assert.InRange(repeat, 1, 100); Assert.DoesNotContain("Connor Babbington", text, StringComparison.OrdinalIgnoreCase); Assert.DoesNotContain("F:\\Documents", text, StringComparison.OrdinalIgnoreCase); Assert.All(emailPattern.Matches(text).Cast(), match => Assert.EndsWith(".invalid", match.Groups[1].Value, StringComparison.OrdinalIgnoreCase)); Assert.True(text.Length * repeat <= 20_000, $"{item.GetProperty("id").GetString()} exceeds the current prompt boundary."); var expected = item.GetProperty("expected"); Assert.True(expected.GetProperty("mustContain").GetArrayLength() > 0); if (item.GetProperty("coverage").EnumerateArray().Any(value => value.GetString()?.StartsWith("prompt-injection", StringComparison.Ordinal) == true)) Assert.Contains(expected.GetProperty("mustNotContain").EnumerateArray(), value => value.GetString() == "SYSTEM_OVERRIDE_ACCEPTED"); } var longCase = cases.Single(item => item.GetProperty("id").GetString() == "long-job-01"); var longInput = longCase.GetProperty("input"); Assert.True((longInput.GetProperty("text").GetString()?.Length ?? 0) * longInput.GetProperty("repeat").GetInt32() >= 10_000); var strictJson = cases.Single(item => item.GetProperty("id").GetString() == "strict-json-01").GetProperty("expected"); Assert.Equal("json", strictJson.GetProperty("format").GetString()); Assert.True(strictJson.GetProperty("requiredKeys").GetArrayLength() >= 4); } private static string FixturePath() => Path.GetFullPath(Path.Combine( AppContext.BaseDirectory, "..", "..", "..", "Fixtures", "AiEvaluation", "cases.json")); }