Multilingual normalizers> A registry for conservative, language-aware text normalization.

supported_languages

def supported_languages()->tuple[__main__.LanguageSupport, ...]:

Return the language registry in ISO-code order.

source

Normalizer

def Normalizer(
    language:str, **options
):

Normalize text for one language: Normalizer("hi", tts_mode=True)("...").

language accepts ISO codes ("hi"), BCP-47 tags ("hi-IN", "zh-Hans") and English names ("hindi"). Options such as tts_mode=True are validated against the selected language and raise ValueError if it does not accept them, instead of being ignored. Normalizer.for_language("hi") is an equivalent spelling.

source

LanguageSupport

def LanguageSupport(
    code:str, name:str, factory:Callable[..., Callable[[str], str]], capabilities:tuple[str, ...], tier:str,
    marks:Literal['kept', 'stripped']='kept'
)->None:

A supported language and the normalization level available for it.

tier is one of:

"dedicated": a normalizer written and reviewed for this language.
"script": no dedicated class yet, but the language shares a script with a dedicated normalizer that is reused as-is (for example Marathi reuses the Devanagari normalizer, without Hindi’s number/TTS tuning).
"generic": only conservative, script-safe cleanup is applied; no language-specific rules exist.

source

MultilingualTextNormalizer

def MultilingualTextNormalizer(
    lang:str='', tts_mode:bool=False
):

Conservative, script-safe normalization for languages without bespoke rules.

assert len(supported_languages()) == 101

tiers = {"dedicated": 0, "script": 0, "generic": 0}
for entry in supported_languages():
    tiers[entry.tier] += 1
assert tiers == {"dedicated": 15, "script": 4, "generic": 82}

# BCP-47 region/script subtags are accepted; only the primary language subtag
# drives selection.
assert Normalizer("french")("Vingt et un euros.") == "21 euros"
assert Normalizer("es-ES")("El total es 1.234,50.") == "el total es 1234.50"
assert Normalizer("Russian")("Privet, mir!") == "privet mir"
assert Normalizer("hi-IN")("नमस्ते, दुनिया!") == Normalizer("hi")("नमस्ते, दुनिया!")
assert Normalizer("en-IN")("Colour and favour.") == Normalizer("en")("Colour and favour.")

# ar/zh/ru must use their dedicated normalizers (from international.py), not
# the generic fallback: dedicated Chinese/Arabic normalization differs from
# the generic tier by converting written numerals and removing all
# inter-character whitespace, which the generic normalizer does not do.
assert Normalizer("zh")("你好，世界！") == "你好世界"
assert Normalizer("zh")("我有二十一個蘋果。") == "我有21個蘋果"
assert Normalizer("ar")("أَهْلًا، وَسَهْلًا!") == "اهلا وسهلا"

# Odia ("or") is a dedicated normalizer even though it isn't one of the ~99
# languages Whisper itself documents. "od-IN" and "odia" are accepted as
# aliases for the same "or" code.
assert LANGUAGE_REGISTRY["or"].tier == "dedicated"
assert Normalizer("od-IN")("2024") == Normalizer("or")("2024")
assert Normalizer("odia")("2024") == Normalizer("or")("2024")

# Script tier: Marathi (mr) and Assamese (as) now support tts_mode via
# indic_numtowords. Nepali (ne) and Sanskrit (sa) remain script-only (not
# covered by indic_numtowords).
assert LANGUAGE_REGISTRY["mr"].tier == "script"
assert isinstance(Normalizer("mr").normalizer, DevanagariNormalizer)
assert Normalizer("mr")("क: ख") == Normalizer("hi")("क: ख")  # shared Devanagari visarga rule
# Marathi now supports tts_mode via DevanagariNormalizer (indic_numtowords supports 'mr').
assert Normalizer("mr", tts_mode=True).tts_mode is True

assert LANGUAGE_REGISTRY["as"].tier == "script"
assert isinstance(Normalizer("as").normalizer, BengaliNormalizer)
assert Normalizer("as").normalizer.lang == "as"

# tts_mode is validated against the selected normalizer instead of being
# silently ignored or forwarded to a constructor that doesn't accept it.
assert Normalizer("hi", tts_mode=True).tts_mode is True
assert Normalizer("bn", tts_mode=True).tts_mode is True
# French now supports tts_mode via _LatinTextNormalizer (num2words converts digits to words).
assert Normalizer("fr", tts_mode=True).tts_mode is True

try:
    Normalizer("xx")
except ValueError as error:
    assert "Unsupported language" in str(error)
else:
    raise AssertionError("unknown languages must be rejected")

# Generic-tier smoke test: every generic-tier code must lowercase, strip
# punctuation and symbols (including emoji, which are Unicode Symbol
# category), collapse whitespace, and stay safe on empty/mixed-script input,
# without bespoke per-language fixtures.
generic_codes = [entry.code for entry in supported_languages() if entry.tier == "generic"]
assert len(generic_codes) == 82
for code in generic_codes:
    normalizer = Normalizer(code)
    assert normalizer("") == ""
    assert normalizer("HELLO, World!!") == "hello world"
    assert normalizer("Café 😀 مرحبا") == "café مرحبا"

# Per-language regression fixtures — dedicated and script tier

# Marathi (script tier): Devanagari normalisation + correct number words
mr = Normalizer('mr')
assert mr('क: ख') == Normalizer('hi')('क: ख')  # visarga rule shared
mr_tts = Normalizer('mr', tts_mode=True)
assert mr_tts.tts_mode is True
result_mr = mr_tts('मला 42 रुपये द्या')
assert '42' not in result_mr, f'MR TTS: digit not converted: {result_mr}'
# Marathi uses 'बेचाळीस', not Hindi 'बयालीस'
assert 'बेचाळीस' in result_mr, f'MR: wrong number word: {result_mr}'

# Assamese (script tier): Bengali normaliser + Assamese char remapping + TTS
as_norm = Normalizer('as')
assert as_norm.language == 'as'
as_tts = Normalizer('as', tts_mode=True)
result_as = as_tts('আমি 10 টকা')
assert '10' not in result_as, f'AS TTS: digit not converted: {result_as}'

# Nepali / Sanskrit: tts_mode unsupported
try:
    Normalizer('ne', tts_mode=True)
    raise AssertionError('ne should raise ValueError')
except ValueError as e:
    assert 'tts_mode' in str(e) or 'not supported' in str(e)

try:
    Normalizer('sa', tts_mode=True)
    raise AssertionError('sa should raise ValueError')
except ValueError as e:
    assert 'tts_mode' in str(e) or 'not supported' in str(e)

# Generic tier: smoke coverage for a representative sample
for code, sample, expected in [
    ('de', 'Guten Tag!', 'guten tag'),
    ('tr', 'Merhaba dünya!', 'merhaba dünya'),
    ('ko', '안녕하세요!', '안녕하세요'),
    ('th', 'สวัสดี!', 'สวัสดี'),
]:
    result = Normalizer(code)(sample)
    assert result == expected, f'{code}: got {result!r}, expected {expected!r}'

print('Per-language regression fixtures passed')

# Generic-tier TTS regression tests
import importlib

if importlib.util.find_spec('num2words') is not None:
    de_tts = Normalizer('de', tts_mode=True)
    result = de_tts('42 Euro')
    assert '42' not in result, f'DE TTS: digits not converted: {result}'

    nl_tts = Normalizer('nl', tts_mode=True)
    result = nl_tts('42 euro')
    assert '42' not in result, f'NL TTS: digits not converted: {result}'

    # Verify lang attr is injected
    assert de_tts.language == 'de', 'lang not injected for generic tier'
    print('Generic TTS tests passed')
else:
    print('num2words not installed; skipping generic TTS tests')

Generic TTS tests passed

`Normalizer`: the single public entry point

Normalizer("hi") and Normalizer.for_language("hi") are equivalent. The tests below use only that public interface; expected values are literals.

# Normalizer(language) selects the normalizer for a language and is callable. One case per tier.
CASES = [
    ("hi", "नमस्ते, दुनिया!", "नमस्ते, दुनिया!"),  # dedicated, Indic
    ("en", "Colour and favour.", "color and favor"),  # dedicated, English
    ("fr", "Vingt et un euros.", "21 euros"),  # dedicated, international
    ("zh", "我有二十一個蘋果。", "我有21個蘋果"),  # dedicated, non-Latin
    ("es", "El total es 1.234,50.", "el total es 1234.50"),  # locale numbers
    ("de", "Guten Tag!", "guten tag"),  # generic tier
]
bad = {}
for code, text, expected in CASES:
    out = Normalizer(code)(text)
    if out != expected:
        bad[code] = (out, expected)
assert not bad, bad

# Normalizer.for_language(language) is the same thing, spelled as a constructor method.
bad = {}
for code, text, expected in CASES:
    n = Normalizer.for_language(code)
    if not isinstance(n, Normalizer) or n(text) != expected:
        bad[code] = n(text)
assert not bad, bad

# Metadata: the language is resolved once, from a code, a BCP-47 tag or an English name.
for query, code, name, tier in [
    ("hi", "hi", "Hindi", "dedicated"),
    ("hi-IN", "hi", "Hindi", "dedicated"),
    ("Hindi", "hi", "Hindi", "dedicated"),
    ("zh-Hans", "zh", "Chinese", "dedicated"),
    ("odia", "or", "Odia", "dedicated"),
    ("mr", "mr", "Marathi", "script"),
    ("de", "de", "German", "generic"),
]:
    n = Normalizer(query)
    assert (n.language, n.name, n.tier) == (code, name, tier), (query, n.language, n.name, n.tier)

# Capabilities tell you what a language supports before you ask for it.
assert "tts_mode" in Normalizer("hi").capabilities
assert "tts_mode" in Normalizer("en").capabilities
assert Normalizer("en", tts_mode=True)("It costs $20 for 3 items") == "it costs twenty dollars for three items"
assert Normalizer("en-US", tts_mode=True).tts_mode is True

# Options: tts_mode is off by default, is reported back, and reaches the language normalizer.
assert Normalizer("hi").tts_mode is False
n = Normalizer("hi", tts_mode=True)
assert n.tts_mode is True and n.options == {"tts_mode": True}
assert Normalizer("mr", tts_mode=True)("मला 42 रुपये द्या") == "मला बेचाळीस रुपये द्या"
assert Normalizer("mr")("मला 42 रुपये द्या") == "मला बेचाळीस रुपये द्या"  # Indic digit spelling is not gated on tts_mode

# repr shows how to recreate it, using the resolved language code.
assert repr(Normalizer("hi-IN", tts_mode=True)) == "Normalizer('hi', tts_mode=True)"
assert repr(Normalizer("Hindi")) == "Normalizer('hi')"

# Errors are loud and specific, for both spellings, and never fall back to a generic normalizer.
for make in (Normalizer, Normalizer.for_language):
    try:
        make("xx")
        raise AssertionError("unsupported language should raise")
    except ValueError as e:
        assert "Unsupported language: 'xx'" in str(e) and "Supported codes:" in str(e) and "hi" in str(e)

    try:
        make("en", bogus=True)
        raise AssertionError("unsupported option should raise")
    except ValueError as e:
        assert str(e) == "English normalizer does not support option(s): bogus"

    for code in ("ne", "sa"):  # script-tier languages without number words
        try:
            make(code, tts_mode=True)
            raise AssertionError(f"{code} tts_mode should raise")
        except ValueError as e:
            assert "tts_mode" in str(e) or "not supported" in str(e)

# Normalizer replaces get_normalizer as the one canonical path.
assert "get_normalizer" not in globals()

# The package exports Normalizer (and the language listing), and nothing else for selection.
import whisper_normalizer
from whisper_normalizer import Normalizer as ExportedNormalizer, supported_languages as exported_supported_languages

assert ExportedNormalizer("hi-IN", tts_mode=True).language == "hi"
assert ExportedNormalizer.for_language("fr")("Vingt et un euros.") == "21 euros"
assert len(exported_supported_languages()) == 101
assert not hasattr(whisper_normalizer, "get_normalizer")
assert sorted(whisper_normalizer.__all__) == ["Normalizer", "__version__", "supported_languages"]

Capability contract

The registry’s capability metadata is checked against real behaviour for every language, so the documented table cannot drift from what the normalizers do.

# tts_mode is advertised exactly when it can be constructed, and then it spells digits as words.
bad = {}
for entry in supported_languages():
    try:
        n = Normalizer(entry.code, tts_mode=True)
    except ValueError:
        constructible = False
    else:
        constructible = True
        spoken = n("5")
        if any(ch.isdigit() for ch in spoken):
            bad[entry.code] = f"tts_mode left a digit in {spoken!r}"
    if constructible != ("tts_mode" in entry.capabilities):
        bad[entry.code] = f"advertises tts_mode={'tts_mode' in entry.capabilities} but constructible={constructible}"
assert not bad, bad

# digits_to_words says when digits become words, and matches what default mode really does for all languages:
# "always" (Indic family), "tts_mode" (only with tts_mode=True) or "never" (tts_mode unavailable).
bad = {}
for entry in supported_languages():
    spelled_by_default = not any(ch.isdigit() for ch in Normalizer(entry.code)("5"))
    if (entry.digits_to_words == "always") != spelled_by_default:
        bad[entry.code] = (entry.digits_to_words, "spelled by default" if spelled_by_default else "digits kept")
    # Outside the Indic family, "never" is exactly "tts_mode unavailable".
    if entry.digits_to_words != "always" and (entry.digits_to_words == "never") != ("tts_mode" not in entry.capabilities):
        bad[entry.code] = (entry.digits_to_words, entry.capabilities)
assert not bad, bad
assert {e.code for e in supported_languages() if e.digits_to_words == "always"} == {
    "as", "bn", "gu", "hi", "kn", "ml", "mr", "ne", "or", "pa", "sa", "ta", "te"}
assert Normalizer("en").tts_mode is False and [e for e in supported_languages() if e.code == "en"][0].digits_to_words == "tts_mode"

# marks says whether combining marks survive normalization ("kept") or are removed ("stripped"); it is checked against
# real behaviour for every dedicated and script-tier language, plus the generic tier.
import unicodedata


def count_marks(s):
    return sum(unicodedata.category(ch).startswith("M") for ch in unicodedata.normalize("NFD", s))


INDIC_BASE = {"hi": 0x0900, "mr": 0x0900, "ne": 0x0900, "sa": 0x0900, "bn": 0x0980, "as": 0x0980, "pa": 0x0A00,
              "gu": 0x0A80, "or": 0x0B00, "ta": 0x0B80, "te": 0x0C00, "kn": 0x0C80, "ml": 0x0D00}
MARK_SAMPLES = {code: chr(b + 0x15) + chr(b + 0x3F) for code, b in INDIC_BASE.items()}  # KA + vowel sign I
MARK_SAMPLES.update({"en": "café", "fr": "café", "es": "niño", "ar": "أَهْلًا", "ru": "йод", "de": "café", "th": "ที่"})
NO_MARKS_IN_SCRIPT = {"zh"}  # Chinese has no combining marks to keep or strip

bad = {}
checked_marks = set()
for entry in supported_languages():
    # Every generic-tier language runs the same script-safe cleanup, so one accented sample covers each of them.
    sample = MARK_SAMPLES.get(entry.code) or ("café" if entry.tier == "generic" else None)
    if sample is None:
        assert entry.code in NO_MARKS_IN_SCRIPT, f"no marks sample for {entry.code}"
        continue
    checked_marks.add(entry.code)
    out = Normalizer(entry.code)(sample)
    expected_marks = 0 if entry.marks == "stripped" else count_marks(sample)
    if count_marks(out) != expected_marks:
        bad[entry.code] = (entry.marks, sample, out)
assert not bad, bad
assert {e.code for e in supported_languages() if e.marks == "stripped"} == {"ar", "en"}
# Every language except Chinese (no combining marks) is checked: 100 of 101.
assert len(checked_marks) == 100 and checked_marks == {e.code for e in supported_languages()} - NO_MARKS_IN_SCRIPT

Capability table

The table in README.md and nbs/index.ipynb is generated from the registry by capability_section() below; the test pins both files to it, so it cannot go stale.

import json
from pathlib import Path

from indic_numtowords import supported_langs as _numtowords_langs


def capability_section():
    """Markdown for the supported-languages table: one row per dedicated/script language, plus a generic-tier summary."""
    rows = ["| Language | Code | Normalizer | Tier | Digits → words | Marks | `tts_mode` |", "|---|---|---|---|---|---|---|"]
    entries = sorted((e for e in supported_languages() if e.tier != "generic"), key=lambda e: (e.tier != "dedicated", e.name))
    for e in entries:
        cls = getattr(e.factory, "func", e.factory).__name__
        digits = e.digits_to_words
        if digits == "always" and e.code not in _numtowords_langs:
            digits += " (Hindi number words)"
        tts = "yes" if "tts_mode" in e.capabilities else "no"
        rows.append(f"| {e.name} | `{e.code}` | `{cls}` | {e.tier} | {digits} | {e.marks} | {tts} |")
    generic = [e for e in supported_languages() if e.tier == "generic"]
    with_tts = sum("tts_mode" in e.capabilities for e in generic)
    summary = (
        f"The other {len(generic)} languages use the generic tier: Unicode NFC, brackets and punctuation removed, "
        f"case folded, combining marks kept. Digits become words only with `tts_mode=True`, which {with_tts} of them support."
    )
    return "\n".join(rows) + "\n\n" + summary


root = next(p for p in (Path("."), Path("..")) if (p / "README.md").exists())
section = capability_section()
assert section.count("\n| ") >= 19  # 15 dedicated + 4 script rows below the header
assert "| English | `en` | `EnglishTextNormalizer` | dedicated | tts_mode | stripped | yes |" in section
assert "| Hindi | `hi` | `HindiNormalizer` | dedicated | always | kept | yes |" in section
assert "| Nepali | `ne` | `DevanagariNormalizer` | script | always (Hindi number words) | kept | no |" in section
assert section in (root / "README.md").read_text(encoding="utf-8"), "README.md is out of date with the registry"
index_nb = json.load(open(root / "nbs" / "index.ipynb", encoding="utf-8"))
index_text = "\n".join("".join(c["source"]) for c in index_nb["cells"])
assert section in index_text, "nbs/index.ipynb is out of date with the registry"

# Boolean options must be real bools: "no" is truthy, so a string would silently mean the opposite of what was meant.
for make in (Normalizer, Normalizer.for_language):
    for bad_value in ("no", "false", 1, None):
        try:
            make("hi", tts_mode=bad_value)
            raise AssertionError(f"tts_mode={bad_value!r} should raise")
        except TypeError as e:
            assert str(e) == f"tts_mode must be a bool, got {type(bad_value).__name__}: {bad_value!r}", str(e)
    try:
        make("ta", do_normalize_chandras="yes")
        raise AssertionError("other boolean options are validated too")
    except TypeError as e:
        assert "do_normalize_chandras must be a bool" in str(e)

assert Normalizer("hi", tts_mode=False).tts_mode is False
assert Normalizer("hi", tts_mode=True).tts_mode is True
# Non-boolean options are not affected.
assert Normalizer("hi", nasals_mode="to_anusvaara_relaxed").options == {"nasals_mode": "to_anusvaara_relaxed"}

supported_languages

Normalizer

LanguageSupport

MultilingualTextNormalizer

Normalizer: the single public entry point

Capability contract

Capability table

`Normalizer`: the single public entry point