diff --git a/JobTrackerApi.Tests/JobCvMatchServiceTests.cs b/JobTrackerApi.Tests/JobCvMatchServiceTests.cs index 38748d9..85666fa 100644 --- a/JobTrackerApi.Tests/JobCvMatchServiceTests.cs +++ b/JobTrackerApi.Tests/JobCvMatchServiceTests.cs @@ -10,6 +10,84 @@ public sealed class JobCvMatchServiceTests private static Dictionary Sections(params (string Name, string Text)[] items) => items.ToDictionary(i => i.Name, i => i.Text, StringComparer.OrdinalIgnoreCase); + public static IEnumerable QualityFixtures() + { + yield return new object[] + { + "Norwegian", + "Senior systemutvikler", + "Vi ser etter deg som har erfaring med C# og ASP.NET Core. Du vil designe skalerbare distribuerte systemer. Gode samarbeidsevner er en fordel.", + new[] { "C#", "ASP.NET Core", "designe skalerbare distribuerte systemer" }, + new[] { "med", "til", "for", "som", "erfaring", "ser" }, + }; + yield return new object[] + { + "English", + "Cloud platform engineer", + "We need a candidate with experience in AWS and Terraform. You will lead incident response and operate distributed systems.", + new[] { "AWS", "Terraform", "incident response", "operate distributed systems" }, + new[] { "the", "with", "experience", "candidate", "will" }, + }; + yield return new object[] + { + "Mixed Norwegian and English", + "DevOps-utvikler", + "Du vil jobbe med Node.js og CI/CD. Work closely with Azure DevOps and cross-functional product teams.", + new[] { "Node.js", "CI/CD", "Azure DevOps", "Collaboration" }, + new[] { "og", "med", "with", "teams" }, + }; + yield return new object[] + { + "Short", + "Data developer", + "Python and SQL.", + new[] { "Python", "SQL" }, + new[] { "and" }, + }; + yield return new object[] + { + "Noisy HTML", + "Frontend developer", + "
Build accessible web applications with React and TypeScript.
Cookie settings Privacy Terms
", + new[] { "React", "TypeScript", "Build accessible web applications" }, + new[] { "home", "jobs", "login", "cookie", "settings", "privacy", "terms", "trackingcookie" }, + }; + yield return new object[] + { + "Technology heavy", + "Platform developer", + "C++, C#, .NET, ASP.NET Core, Node.js, CI/CD, Azure DevOps and Kubernetes.", + new[] { "C++", "C#", ".NET", "ASP.NET Core", "Node.js", "CI/CD", "Azure DevOps", "Kubernetes" }, + new[] { "and" }, + }; + yield return new object[] + { + "Repeated recruitment filler", + "Software engineer", + "Exciting opportunity for a passionate candidate. Great opportunity, strong experience required. We offer an exciting dynamic environment. Apply now. Build services using domain-driven design and Docker.", + new[] { "domain-driven design", "Docker" }, + new[] { "exciting", "opportunity", "passionate", "candidate", "experience", "environment", "apply" }, + }; + } + + [Theory] + [MemberData(nameof(QualityFixtures))] + public void Quality_fixtures_keep_useful_terms_and_suppress_noise( + string name, + string title, + string description, + string[] expected, + string[] excluded) + { + var result = _service.Evaluate(title, description, Sections(("Skills", "synthetic profile text"))); + var terms = result.MatchedKeywords.Concat(result.MissingKeywords).ToList(); + + foreach (var term in expected) + Assert.True(terms.Contains(term, StringComparer.OrdinalIgnoreCase), $"{name}: expected '{term}' in [{string.Join(", ", terms)}]"); + foreach (var term in excluded) + Assert.False(terms.Contains(term, StringComparer.OrdinalIgnoreCase), $"{name}: did not expect '{term}' in [{string.Join(", ", terms)}]"); + } + [Fact] public void Strong_overlap_scores_high_and_lists_matched_keywords() { diff --git a/JobTrackerApi/Services/JobCvMatchService.cs b/JobTrackerApi/Services/JobCvMatchService.cs index 83ec2b6..8025fd6 100644 --- a/JobTrackerApi/Services/JobCvMatchService.cs +++ b/JobTrackerApi/Services/JobCvMatchService.cs @@ -1,4 +1,5 @@ using System.Globalization; +using System.Net; using System.Text; using System.Text.RegularExpressions; using JobTrackerApi.Services.JobImport; @@ -40,11 +41,15 @@ namespace JobTrackerApi.Services private const int TitleBonus = 2; private const int MaxKeywords = 28; - private static readonly Regex TokenPattern = new(@"[a-z0-9][a-z0-9+.#-]*", RegexOptions.Compiled); + private static readonly Regex TokenPattern = new(@"[\p{L}\p{N}][\p{L}\p{N}+.#/-]*", RegexOptions.Compiled); + private static readonly Regex HtmlBlockPattern = new(@"<(script|style|nav|header|footer)[^>]*>.*?", RegexOptions.IgnoreCase | RegexOptions.Singleline | RegexOptions.Compiled); + private static readonly Regex HtmlTagPattern = new(@"<[^>]+>", RegexOptions.Compiled); + private static readonly Regex SegmentPattern = new(@"[\r\n,;:!?\u2022]+|(?<=[.!?])\s+", RegexOptions.Compiled); private static readonly HashSet StopWords = new(StringComparer.OrdinalIgnoreCase) { - "the", "and", "for", "with", "you", "your", "our", "are", "will", "have", "has", "that", + "the", "a", "an", "and", "or", "of", "to", "in", "on", "as", "is", "be", "if", "it", + "we", "us", "for", "with", "you", "your", "our", "are", "will", "have", "has", "that", "this", "from", "not", "but", "all", "can", "who", "how", "why", "what", "when", "who", "job", "role", "work", "working", "team", "teams", "company", "years", "year", "experience", "experienced", "skills", "skill", "ability", "able", "strong", "good", "great", "excellent", @@ -64,6 +69,22 @@ namespace JobTrackerApi.Services "senior", "junior", "lead", "principal", "mid", "staff", "engineer", "engineers", "engineering", "manager", "specialist", "analyst", "consultant", "administrator", "coordinator", "associate", "intern", "officer", "director", "professional", + // Norwegian function words and generic recruitment language. These are deliberately + // language-wide categories rather than the handful of examples that exposed the bug. + "og", "i", "det", "at", "en", "et", "den", "til", "er", "som", "på", "de", "med", + "av", "ikke", "der", "så", "var", "seg", "men", "har", "om", "vi", "ha", "hadde", + "hun", "han", "nå", "da", "ved", "fra", "du", "ut", "sin", "dem", "oss", "opp", + "man", "kan", "hans", "hvor", "eller", "hva", "skal", "selv", "her", "alle", "vil", + "bli", "ble", "blitt", "kunne", "inn", "når", "være", "noen", "noe", "ville", "dere", + "deres", "kun", "etter", "ned", "skulle", "denne", "disse", "for", "deg", "sine", "sitt", + "mot", "uten", "hvordan", "ingen", "din", "ditt", "blir", "samme", "hvilken", "hvilke", + "erfaring", "erfaringer", "kvalifikasjoner", "arbeidsoppgaver", "stilling", "stillingen", + "søker", "ser", "ønsker", "mulighet", "spennende", "arbeidsmiljø", "selskap", "bedrift", "kandidat", + "relevant", "fordel", "gode", "dyktig", "sammen", + // Common source-page chrome and consent text must never become tailoring advice. + "cookie", "cookies", "privacy", "terms", "conditions", "menu", "home", "login", "contact", + "website", "settings", "navigation", "jobs", "apply", "application", "share", "save", "accept", + "reject", "consent", "exciting", "passionate", "dynamic", "innovative", "motivated", }; public JobCvMatchResult Evaluate(string jobTitle, string jobText, IReadOnlyDictionary cvSections) @@ -134,7 +155,8 @@ namespace JobTrackerApi.Services private static List BuildKeywords(string jobTitle, string jobText, HashSet titleTokens) { - var combined = $"{jobTitle}\n{jobText}"; + var cleanedJobText = CleanSourceText(jobText); + var combined = $"{jobTitle}\n{cleanedJobText}"; var byKey = new Dictionary(StringComparer.OrdinalIgnoreCase); // 1) Curated skill tags: high-signal, canonical spelling. @@ -144,11 +166,22 @@ namespace JobTrackerApi.Services byKey[tag] = new MatchKeyword(tag, CuratedTagWeight + (inTitle ? TitleBonus : 0), inTitle, false, IsCuratedTag: true); } - // 2) Salient posting terms: frequency-ranked content words from the description. - var frequencies = new Dictionary(StringComparer.OrdinalIgnoreCase); - foreach (var token in Tokenize(jobText)) + // 2) Important multi-word terms. Stop words break phrases, so "erfaring med ASP.NET + // Core" keeps the technology but never emits "erfaring" as advice. + var phraseTokens = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var phrase in ExtractPhrases(cleanedJobText).Take(8)) { - if (token.Length is < 3 or > 64 || StopWords.Contains(token) || IsNumeric(token)) continue; + if (byKey.ContainsKey(phrase)) continue; + var inTitle = TitleContains(jobTitle, phrase); + byKey[phrase] = new MatchKeyword(phrase, 2 + (inTitle ? TitleBonus : 0), inTitle, false); + foreach (var token in Tokenize(phrase)) phraseTokens.Add(token); + } + + // 3) Salient posting terms: frequency-ranked content words from the cleaned description. + var frequencies = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var token in Tokenize(cleanedJobText)) + { + if (token.Length is < 3 or > 64 || StopWords.Contains(token) || phraseTokens.Contains(token) || IsNumeric(token)) continue; frequencies[token] = frequencies.TryGetValue(token, out var c) ? c + 1 : 1; } @@ -174,6 +207,55 @@ namespace JobTrackerApi.Services .ToList(); } + private static IEnumerable ExtractPhrases(string text) + { + var candidates = new Dictionary(StringComparer.OrdinalIgnoreCase); + var order = 0; + + foreach (var segment in SegmentPattern.Split(text)) + { + var run = new List(); + foreach (Match match in TokenPattern.Matches(segment)) + { + var display = TrimToken(match.Value); + var normalized = display.ToLowerInvariant(); + if (display.Length == 0 || StopWords.Contains(normalized) || IsNumeric(normalized)) + { + AddRun(run); + run.Clear(); + } + else + { + run.Add(display); + } + } + AddRun(run); + } + + return candidates.Values + .OrderByDescending(candidate => candidate.Count) + .ThenByDescending(candidate => Tokenize(candidate.Display).Count()) + .ThenBy(candidate => candidate.First) + .Select(candidate => candidate.Display); + + void AddRun(List run) + { + if (run.Count < 2) return; + AddCandidate(run.Count <= 4 ? run : run.Take(4).ToList()); + if (run.Count > 4) AddCandidate(run.TakeLast(4).ToList()); + } + + void AddCandidate(List selected) + { + var display = string.Join(" ", selected); + var key = display.ToLowerInvariant(); + if (candidates.TryGetValue(key, out var existing)) + candidates[key] = (existing.Display, existing.Count + 1, existing.First); + else + candidates[key] = (display, 1, order++); + } + } + private static bool TitleContains(string title, string phrase) => Normalize(title).Contains(Normalize(phrase), StringComparison.Ordinal); @@ -199,10 +281,23 @@ namespace JobTrackerApi.Services if (string.IsNullOrWhiteSpace(text)) yield break; foreach (Match m in TokenPattern.Matches(text.ToLowerInvariant())) { - yield return m.Value.Trim('-', '.', '+', '#'); + var token = TrimToken(m.Value); + if (token.Length > 0) yield return token; } } + private static string TrimToken(string token) => token.Trim('-', '.', '/'); + + private static string CleanSourceText(string text) + { + if (string.IsNullOrWhiteSpace(text)) return string.Empty; + var withoutBlocks = HtmlBlockPattern.Replace(text, " "); + var withoutTags = HtmlTagPattern.Replace(withoutBlocks, "\n"); + var decoded = WebUtility.HtmlDecode(withoutTags).Replace("\r\n", "\n", StringComparison.Ordinal).Replace('\r', '\n'); + var withoutHorizontalRuns = Regex.Replace(decoded, @"[^\S\r\n]+", " "); + return Regex.Replace(withoutHorizontalRuns, @"\n{2,}", "\n").Trim(); + } + private static bool IsNumeric(string token) => token.All(c => char.IsDigit(c) || c is '.' or '-' or '+'); diff --git a/JobTrackerApi/Services/JobImport/SkillTagger.cs b/JobTrackerApi/Services/JobImport/SkillTagger.cs index ae5b981..504940d 100644 --- a/JobTrackerApi/Services/JobImport/SkillTagger.cs +++ b/JobTrackerApi/Services/JobImport/SkillTagger.cs @@ -12,18 +12,28 @@ public static class SkillTagger // Symbol skills need punctuation-tolerant boundaries: \b fails next to '#'/'.' // (both non-word chars), which previously left "C#," and ".NET," undetected. ("C#", new Regex(@"(? { jest.clearAllMocks(); }); -test('match score panel shows the score, matched and missing keywords', async () => { +test('match score panel shows the score and honest important-term labels', async () => { renderDialog(); expect(await screen.findByText('82%')).toBeInTheDocument(); expect(await screen.findByText(/strong match/i)).toBeInTheDocument(); - expect(await screen.findByText('4/6 keywords')).toBeInTheDocument(); + expect(await screen.findByText('4/6 important terms')).toBeInTheDocument(); + expect(await screen.findByText('Important terms already in your CV')).toBeInTheDocument(); + expect(await screen.findByText('Important terms from the job to review')).toBeInTheDocument(); + expect(screen.queryByText(/matched keywords/i)).not.toBeInTheDocument(); // Matched keyword chips expect(await screen.findByText('C#')).toBeInTheDocument();