jobtrackingapp/JobTrackerApi/Models/HumanLanguageCatalog.cs

using System.Globalization;
using System.Text;
using System.Text.RegularExpressions;

namespace JobTrackerApi.Models;

public static class HumanLanguageCatalog
{
    private static readonly Dictionary<string, string> LanguageLookup = BuildLanguageLookup();

    private static readonly Regex WordRegex = new(@"\p{L}+", RegexOptions.Compiled);

    private static readonly Regex LevelRegex = new(
        @"\b(native(?:\s+speaker)?|fluent|advanced|intermediate|beginner|basic|conversational|elementary|professional\s+working\s+proficiency|working\s+proficiency|limited\s+working\s+proficiency|full\s+professional\s+proficiency|a1|a2|b1|b2|c1|c2|a1\s*/\s*a2|a2\s*/\s*b1|b1\s*/\s*b2|b2\s*/\s*c1|c1\s*/\s*c2)\b",
        RegexOptions.IgnoreCase | RegexOptions.Compiled);

    public static string? NormalizeLanguageName(string? raw)
    {
        var matches = ExtractLanguageNames(raw);
        return matches.Count == 1 ? matches[0] : null;
    }

    public static IReadOnlyList<string> ExtractLanguageNames(string? raw)
    {
        if (string.IsNullOrWhiteSpace(raw)) return Array.Empty<string>();

        var words = WordRegex.Matches(raw)
            .Select(match => match.Value)
            .Where(value => !string.IsNullOrWhiteSpace(value))
            .ToList();

        if (words.Count == 0) return Array.Empty<string>();

        var matches = new List<(int Start, int Size, string Canonical)>();
        for (var size = Math.Min(4, words.Count); size >= 1; size--)
        {
            for (var start = 0; start <= words.Count - size; start++)
            {
                var phrase = string.Join(" ", words.Skip(start).Take(size));
                if (!LanguageLookup.TryGetValue(NormalizeKey(phrase), out var canonical)) continue;
                if (matches.Any(existing => RangesOverlap(existing.Start, existing.Size, start, size))) continue;
                matches.Add((start, size, canonical));
            }
        }

        return matches
            .OrderBy(match => match.Start)
            .Select(match => match.Canonical)
            .Distinct(StringComparer.OrdinalIgnoreCase)
            .ToList();
    }

    public static bool HasRecognizedLevel(string? raw)
    {
        return ExtractLevel(raw) is not null;
    }

    public static string? ExtractLevel(string? raw)
    {
        if (string.IsNullOrWhiteSpace(raw)) return null;

        var match = LevelRegex.Match(raw);
        if (!match.Success) return null;

        var value = match.Groups[1].Value.Trim();
        var compact = Regex.Replace(value, @"\s+", " ");
        return compact.ToLowerInvariant() switch
        {
            "native speaker" => "Native",
            "native" => "Native",
            "fluent" => "Fluent",
            "advanced" => "Advanced",
            "intermediate" => "Intermediate",
            "beginner" => "Beginner",
            "basic" => "Basic",
            "conversational" => "Conversational",
            "elementary" => "Elementary",
            "professional working proficiency" => "Professional working proficiency",
            "working proficiency" => "Working proficiency",
            "limited working proficiency" => "Limited working proficiency",
            "full professional proficiency" => "Full professional proficiency",
            _ when Regex.IsMatch(compact, @"^[ABC][12](?:\s*/\s*[ABC][12])?$", RegexOptions.IgnoreCase) => compact.ToUpperInvariant().Replace(" ", string.Empty),
            _ => compact,
        };
    }

    private static bool RangesOverlap(int startA, int sizeA, int startB, int sizeB)
    {
        var endA = startA + sizeA;
        var endB = startB + sizeB;
        return startA < endB && startB < endA;
    }

    private static Dictionary<string, string> BuildLanguageLookup()
    {
        var map = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);

        void Add(string? alias, string? canonical)
        {
            var normalizedAlias = NormalizeKey(alias);
            var normalizedCanonical = NormalizeDisplayName(canonical);
            if (string.IsNullOrWhiteSpace(normalizedAlias) || string.IsNullOrWhiteSpace(normalizedCanonical)) return;
            map.TryAdd(normalizedAlias, normalizedCanonical);
        }

        // Force an alias to a canonical, overriding whatever culture enumeration inserted for that
        // key. TryAdd is not enough here: on a host whose ICU data carries a "Norwegian Nynorsk"
        // culture, the enumeration below claims the key "nynorsk" -> "Norwegian Nynorsk" first, and a
        // later TryAdd("nynorsk", "Norwegian") silently loses. That made "nynorsk" resolve to
        // "Norwegian Nynorsk" on the CI runner but "Norwegian" locally — the same host-ICU dependence
        // this seeding exists to remove. Overrides must win regardless of insertion order.
        void Override(string alias, string canonical)
        {
            var normalizedAlias = NormalizeKey(alias);
            var normalizedCanonical = NormalizeDisplayName(canonical);
            if (string.IsNullOrWhiteSpace(normalizedAlias) || string.IsNullOrWhiteSpace(normalizedCanonical)) return;
            map[normalizedAlias] = normalizedCanonical;
        }

        // Seeded FIRST, and deliberately not derived from the host.
        //
        // This table used to come only from CultureInfo.GetCultures, which returns whatever
        // culture data the machine happens to carry: 806 entries on a normal Linux or Windows
        // box, exactly 1 under globalization-invariant mode, and an English-only subset on a
        // container with trimmed ICU data. So whether "Norwegian" was recognised as a human
        // language depended on the deployment environment, not on the CV. Under invariant mode
        // every language was silently dropped and a CV import lost its Languages section with
        // no error at all.
        //
        // These are the languages a CV realistically lists. Culture enumeration still runs
        // below and still adds breadth for free, but nothing here depends on it.
        //
        // Nothing in this list may collide with a technical skill — "Go", "Java", "Swift",
        // "Rust" and "Basic" are deliberately absent. "Basic" is also a proficiency level.
        string[] seed =
        [
            "English", "Norwegian", "Swedish", "Danish", "Finnish", "Icelandic",
            "German", "Dutch", "French", "Spanish", "Portuguese", "Italian",
            "Polish", "Czech", "Slovak", "Slovenian", "Croatian", "Serbian", "Bosnian",
            "Bulgarian", "Romanian", "Hungarian", "Greek", "Albanian", "Macedonian",
            "Russian", "Ukrainian", "Belarusian", "Lithuanian", "Latvian", "Estonian",
            "Turkish", "Arabic", "Hebrew", "Persian", "Kurdish", "Pashto", "Urdu",
            "Hindi", "Bengali", "Punjabi", "Gujarati", "Marathi", "Tamil", "Telugu",
            "Malayalam", "Kannada", "Sinhala", "Nepali",
            "Chinese", "Japanese", "Korean", "Vietnamese", "Thai", "Lao", "Khmer",
            "Burmese", "Malay", "Indonesian", "Filipino", "Tagalog", "Javanese",
            "Swahili", "Amharic", "Somali", "Hausa", "Yoruba", "Igbo", "Zulu", "Afrikaans",
            "Catalan", "Basque", "Galician", "Welsh", "Irish", "Scottish Gaelic", "Maltese",
            "Latin", "Esperanto", "Armenian", "Georgian", "Azerbaijani", "Kazakh", "Uzbek",
        ];

        foreach (var language in seed) Add(language, language);

        foreach (var culture in CultureInfo.GetCultures(CultureTypes.NeutralCultures | CultureTypes.SpecificCultures))
        {
            var english = CleanCultureLanguageName(culture.EnglishName);
            var native = CleanCultureLanguageName(culture.NativeName);
            Add(english, english);
            Add(native, english);
        }

        // These collapse regional/script variants and common exonyms to the umbrella language a CV
        // means. They must beat culture enumeration (see Override), because ICU carries "Norwegian
        // Bokmål"/"Norwegian Nynorsk" and "Chinese (Simplified/Traditional)" as their own cultures.
        Override("norsk", "Norwegian");
        Override("bokmal", "Norwegian");
        Override("bokmål", "Norwegian");
        Override("nynorsk", "Norwegian");
        Override("norwegian bokmal", "Norwegian");
        Override("norwegian bokmål", "Norwegian");
        Override("norwegian nynorsk", "Norwegian");
        Override("mandarin", "Chinese");
        Override("cantonese", "Chinese");
        Override("farsi", "Persian");
        Override("persian", "Persian");

        return map;
    }

    private static string? CleanCultureLanguageName(string? value)
    {
        if (string.IsNullOrWhiteSpace(value)) return null;

        var cleaned = value.Trim();
        var parenIndex = cleaned.IndexOf('(');
        if (parenIndex > 0) cleaned = cleaned[..parenIndex].Trim();
        var commaIndex = cleaned.IndexOf(',');
        if (commaIndex > 0) cleaned = cleaned[..commaIndex].Trim();
        return NormalizeDisplayName(cleaned);
    }

    private static string? NormalizeDisplayName(string? value)
    {
        if (string.IsNullOrWhiteSpace(value)) return null;
        var cleaned = Regex.Replace(value.Trim(), @"\s+", " ");
        return string.Join(" ", cleaned.Split(' ', StringSplitOptions.RemoveEmptyEntries)
            .Select(word => word.Length <= 3 && word.All(char.IsUpper)
                ? word
                : char.ToUpperInvariant(word[0]) + word[1..].ToLowerInvariant()));
    }

    private static string NormalizeKey(string? value)
    {
        if (string.IsNullOrWhiteSpace(value)) return string.Empty;

        var decomposed = value.Trim().Normalize(NormalizationForm.FormD);
        var builder = new StringBuilder(decomposed.Length);
        foreach (var ch in decomposed)
        {
            if (CharUnicodeInfo.GetUnicodeCategory(ch) == UnicodeCategory.NonSpacingMark) continue;
            builder.Append(char.ToLowerInvariant(ch));
        }

        return Regex.Replace(builder.ToString().Normalize(NormalizationForm.FormC), @"[^\p{L}]+", " ").Trim();
    }
}