espnet2.text.word_tokenizer.WordTokenizer

source

class espnet2.text.word_tokenizer.WordTokenizer(delimiter: str | None = None, non_linguistic_symbols: Path | str | Iterable[str] | None = None, remove_non_linguistic_symbols: bool = False)

Bases: AbsTokenizer

text2tokens(line: str) → List[str]

tokens2text(tokens: Iterable[str]) → str