using System;
using System.Collections.Generic;

public sealed class SymbolTable
{
    private readonly HashSet<string> _allowed;

    public SymbolTable(IEnumerable<string> allowed)
    {
        ArgumentNullException.ThrowIfNull(allowed);
        _allowed = new HashSet<string>(StringComparer.Ordinal);
        foreach (var t in allowed)
        {
            if (t is null) continue;
            _allowed.Add(string.Intern(t)); // pre-canonicalize once, at construction
        }
    }

    // TryGetValue hands back the STORED instance, so the process-wide intern pool is
    // touched once per vocabulary entry in the constructor, not once per token on every
    // parser loop. Anything outside the vocabulary is returned untouched.
    public string Canonicalize(string text)
    {
        ArgumentNullException.ThrowIfNull(text);
        return _allowed.TryGetValue(text, out var canonical) ? canonical : text;
    }

    // ReferenceEquals alone is only valid for tokens that ARE in the vocabulary. Canonicalize
    // returns out-of-vocabulary text unchanged, so two equal strings built at run time would
    // otherwise be reported as different symbols. The ordinal content compare closes that hole,
    // and the reference check in front of it keeps the canonical case O(1).
    public bool AreSameSymbol(string a, string b)
    {
        var ca = Canonicalize(a);
        var cb = Canonicalize(b);
        return ReferenceEquals(ca, cb) || string.Equals(ca, cb, StringComparison.Ordinal);
    }
}

public static class Demo
{
    public static void Main()
    {
        var syms = new SymbolTable(new[] { "IF", "ELSE", "LET", "RETURN" });

        // new string(text) binds to string(ReadOnlySpan<char>) and produces a fresh instance.
        var x1 = syms.Canonicalize(new string("IF"));
        var x2 = syms.Canonicalize("IF");
        var y1 = syms.Canonicalize(new string("WHATEVER")); // not allowed: returned as-is, never interned

        Console.WriteLine(ReferenceEquals(x1, x2));
        Console.WriteLine(syms.AreSameSymbol("IF", "IF"));
        Console.WriteLine(syms.AreSameSymbol("IF", "ELSE"));
        Console.WriteLine(ReferenceEquals(y1, "WHATEVER"));
        var junk = "junk-" + Guid.NewGuid();               // high-cardinality, never a literal
        Console.WriteLine(string.IsInterned(syms.Canonicalize(junk)) is null);

        // Two identical identifiers built at run time, both OUTSIDE the vocabulary.
        var u1 = new string("myVariable");
        var u2 = new string("myVariable");
        Console.WriteLine($"{ReferenceEquals(u1, u2)} / {syms.AreSameSymbol(u1, u2)}");
    }
}

Output (.NET 10, deterministic):

True
True
False
False
True
False / True

Line by line: a runtime-built "IF" and the literal "IF" canonicalize to the same instance; IF is the same symbol as itself and not the same as ELSE; a runtime-built "WHATEVER" comes back untouched, not the pooled literal; and a high-cardinality token is not added to the intern pool, which is the property the whole design exists for.

The last line is the one to read twice. Two identical identifiers built at run time are different references and the same symbol. A reference compare on its own would have said False here, and identifiers are precisely the tokens a parser sees that are not in the keyword vocabulary, so that would be wrong exactly where it matters. AreSameSymbol therefore falls back to an ordinal content compare. The reference check in front of it is the fast path for canonical tokens, not the answer.