cesnimda
76 lines
4.3 KiB
C#
76 lines
4.3 KiB
C#
using System.Text.Json;
|
|
using System.Text.RegularExpressions;
|
|
using Xunit;
|
|
|
|
namespace JobTrackerApi.Tests;
|
|
|
|
public sealed class AiEvaluationFixtureTests
|
|
{
|
|
private static readonly HashSet<string> RequiredCoverage = new(StringComparer.Ordinal)
|
|
{
|
|
"english-cv", "norwegian-cv", "mixed-language-cv", "english-job-advert",
|
|
"norwegian-job-advert", "noisy-job-advert", "technology-heavy-role", "sparse-role",
|
|
"email-classification", "follow-up-draft", "strategy-snapshot", "strict-json-response",
|
|
"malformed-adversarial-document", "prompt-injection-job", "prompt-injection-email",
|
|
"long-input", "empty-input", "invalid-input",
|
|
};
|
|
|
|
private static readonly HashSet<string> KnownTasks = new(StringComparer.Ordinal)
|
|
{
|
|
"DOC-EXTRACT", "CV-NORMALIZE", "CV-CLASSIFY", "PROFILE-EXTRACT", "PROFILE-DIFF",
|
|
"JOB-CLEAN", "JOB-SUMMARY", "JOB-MATCH", "STRATEGY", "CV-TAILOR",
|
|
"APPLICATION-DRAFT", "FOLLOWUP-DRAFT", "EMAIL-CLASSIFY", "RECRUITMENT-DETECT",
|
|
"INTERVIEW", "WRITING",
|
|
};
|
|
|
|
[Fact]
|
|
public void Synthetic_evaluation_set_is_complete_bounded_and_contains_no_real_contact_domain()
|
|
{
|
|
using var document = JsonDocument.Parse(File.ReadAllText(FixturePath()));
|
|
var root = document.RootElement;
|
|
Assert.Equal("1", root.GetProperty("version").GetString());
|
|
Assert.True(root.GetProperty("syntheticOnly").GetBoolean());
|
|
var cases = root.GetProperty("cases").EnumerateArray().ToList();
|
|
Assert.NotEmpty(cases);
|
|
Assert.Equal(cases.Count, cases.Select(item => item.GetProperty("id").GetString()).Distinct(StringComparer.Ordinal).Count());
|
|
|
|
var coverage = cases.SelectMany(item => item.GetProperty("coverage").EnumerateArray())
|
|
.Select(item => item.GetString()).Where(item => item is not null).Cast<string>().ToHashSet(StringComparer.Ordinal);
|
|
Assert.Empty(RequiredCoverage.Except(coverage));
|
|
|
|
var tasks = cases.SelectMany(item => item.GetProperty("tasks").EnumerateArray())
|
|
.Select(item => item.GetString()).Where(item => item is not null).Cast<string>().ToHashSet(StringComparer.Ordinal);
|
|
Assert.Empty(tasks.Except(KnownTasks));
|
|
Assert.Empty(KnownTasks.Except(tasks));
|
|
|
|
var emailPattern = new Regex(@"[A-Z0-9._%+-]+@([A-Z0-9.-]*[A-Z0-9])", RegexOptions.IgnoreCase);
|
|
foreach (var item in cases)
|
|
{
|
|
var input = item.GetProperty("input");
|
|
var text = input.GetProperty("text").GetString() ?? string.Empty;
|
|
var repeat = input.GetProperty("repeat").GetInt32();
|
|
Assert.InRange(repeat, 1, 100);
|
|
Assert.DoesNotContain("Connor Babbington", text, StringComparison.OrdinalIgnoreCase);
|
|
Assert.DoesNotContain("F:\\Documents", text, StringComparison.OrdinalIgnoreCase);
|
|
Assert.All(emailPattern.Matches(text).Cast<Match>(), match => Assert.EndsWith(".invalid", match.Groups[1].Value, StringComparison.OrdinalIgnoreCase));
|
|
Assert.True(text.Length * repeat <= 20_000, $"{item.GetProperty("id").GetString()} exceeds the current prompt boundary.");
|
|
|
|
var expected = item.GetProperty("expected");
|
|
Assert.True(expected.GetProperty("mustContain").GetArrayLength() > 0);
|
|
if (item.GetProperty("coverage").EnumerateArray().Any(value => value.GetString()?.StartsWith("prompt-injection", StringComparison.Ordinal) == true))
|
|
Assert.Contains(expected.GetProperty("mustNotContain").EnumerateArray(), value => value.GetString() == "SYSTEM_OVERRIDE_ACCEPTED");
|
|
}
|
|
|
|
var longCase = cases.Single(item => item.GetProperty("id").GetString() == "long-job-01");
|
|
var longInput = longCase.GetProperty("input");
|
|
Assert.True((longInput.GetProperty("text").GetString()?.Length ?? 0) * longInput.GetProperty("repeat").GetInt32() >= 10_000);
|
|
|
|
var strictJson = cases.Single(item => item.GetProperty("id").GetString() == "strict-json-01").GetProperty("expected");
|
|
Assert.Equal("json", strictJson.GetProperty("format").GetString());
|
|
Assert.True(strictJson.GetProperty("requiredKeys").GetArrayLength() >= 4);
|
|
}
|
|
|
|
private static string FixturePath() => Path.GetFullPath(Path.Combine(
|
|
AppContext.BaseDirectory, "..", "..", "..", "Fixtures", "AiEvaluation", "cases.json"));
|
|
}
|