whisper_normalizer

SpanishTextNormalizer

def SpanishTextNormalizer(
    *args, **kwargs
):

Normalize Spanish text without removing accents, dieresis, or enye.

source

FrenchTextNormalizer

def FrenchTextNormalizer(
    *args, **kwargs
):

Normalize French text without removing accents or ligatures.

source

RussianTextNormalizer

def RussianTextNormalizer(
    *args, **kwargs
):

Normalize Russian case, the optional letter ё, and numeric separators.

source

ChineseTextNormalizer

def ChineseTextNormalizer(
    *args, **kwargs
):

Normalize Chinese punctuation, whitespace, and common written numerals.

source

ArabicTextNormalizer

def ArabicTextNormalizer(
    *args, **kwargs
):

Normalize Arabic orthographic variants, diacritics, digits, and punctuation.

french_normalizer = FrenchTextNormalizer()
assert french_normalizer("J\u2019ai pay\u00e9 vingt et un euros pour l\u2019\u00e9t\u00e9.") == "j ai pay\u00e9 21 euros pour l \u00e9t\u00e9"
assert french_normalizer("Le total est 1\u202f234,50 \u20ac.") == "le total est 1234.50"

spanish_normalizer = SpanishTextNormalizer()
assert spanish_normalizer("\u00bfPag\u00f3 veinti\u00fan euros?") == "pag\u00f3 21 euros"
assert spanish_normalizer("El total es 1.234,50 \u20ac.") == "el total es 1234.50"
assert spanish_normalizer("El ni\u00f1o lleg\u00f3.") == "el ni\u00f1o lleg\u00f3"

arabic_normalizer = ArabicTextNormalizer()
assert arabic_normalizer("أَهْلًا، وَسَهْلًا!") == "اهلا وسهلا"
assert arabic_normalizer("السعر ١٬٢٣٤٫٥٠ ريال.") == "السعر 1234.50 ريال"

chinese_normalizer = ChineseTextNormalizer()
assert chinese_normalizer("我有二十一個蘋果。") == "我有21個蘋果"
assert chinese_normalizer("二〇二四年") == "2024年"
assert chinese_normalizer("你好， 世界！") == "你好世界"

russian_normalizer = RussianTextNormalizer()
assert russian_normalizer("Ёлка — зелёная.") == "елка зеленая"
assert russian_normalizer("Цена: 1 234,50 руб.") == "цена 1234.50 руб"

class TestLatinTextNormalizer(_LatinTextNormalizer):
    language = "fr"


assert TestLatinTextNormalizer()("[noise] L’été—test (aside)\u200b!") == "l été test"
assert _NonLatinTextNormalizer()("[noise] Привет (aside)\u200b, мир!") == "привет мир"

assert arabic_normalizer("مــسؤوليةُ الفتى") == "مسوولية الفتي"
assert arabic_normalizer("شَيْئًا") == "شييا"

assert chinese_normalizer("十") == "10"
assert chinese_normalizer("两百零三万零十") == "2030010"

assert russian_normalizer("Цена 1\u00a0234,50 и 9\u202f876,00.") == "цена 1234.50 и 9876.00"

assert spanish_normalizer("Un niño vive en una casa.") == "un niño vive en una casa"
assert spanish_normalizer("Veinte euros.") == "20 euros"
assert chinese_normalizer("一万亿") == "1000000000000"
assert chinese_normalizer("一万亿零三万") == "1000000030000"