352 lines
13 KiB
Python
352 lines
13 KiB
Python
|
"""
|
|||
|
|
Metadata about languages used by our model training code for our
|
|||
|
|
SingleByteCharSetProbers. Could be used for other things in the future.
|
|||
|
|
|
|||
|
|
This code is based on the language metadata from the uchardet project.
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
from string import ascii_letters
|
|||
|
|
|
|||
|
|
# TODO: Add Ukrainian (KOI8-U)
|
|||
|
|
|
|||
|
|
|
|||
|
|
class Language:
|
|||
|
|
"""Metadata about a language useful for training models
|
|||
|
|
|
|||
|
|
:ivar name: The human name for the language, in English.
|
|||
|
|
:type name: str
|
|||
|
|
:ivar iso_code: 2-letter ISO 639-1 if possible, 3-letter ISO code otherwise,
|
|||
|
|
or use another catalog as a last resort.
|
|||
|
|
:type iso_code: str
|
|||
|
|
:ivar use_ascii: Whether or not ASCII letters should be included in trained
|
|||
|
|
models.
|
|||
|
|
:type use_ascii: bool
|
|||
|
|
:ivar charsets: The charsets we want to support and create data for.
|
|||
|
|
:type charsets: list of str
|
|||
|
|
:ivar alphabet: The characters in the language's alphabet. If `use_ascii` is
|
|||
|
|
`True`, you only need to add those not in the ASCII set.
|
|||
|
|
:type alphabet: str
|
|||
|
|
:ivar wiki_start_pages: The Wikipedia pages to start from if we're crawling
|
|||
|
|
Wikipedia for training data.
|
|||
|
|
:type wiki_start_pages: list of str
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
def __init__(
|
|||
|
|
self,
|
|||
|
|
name=None,
|
|||
|
|
iso_code=None,
|
|||
|
|
use_ascii=True,
|
|||
|
|
charsets=None,
|
|||
|
|
alphabet=None,
|
|||
|
|
wiki_start_pages=None,
|
|||
|
|
):
|
|||
|
|
super().__init__()
|
|||
|
|
self.name = name
|
|||
|
|
self.iso_code = iso_code
|
|||
|
|
self.use_ascii = use_ascii
|
|||
|
|
self.charsets = charsets
|
|||
|
|
if self.use_ascii:
|
|||
|
|
if alphabet:
|
|||
|
|
alphabet += ascii_letters
|
|||
|
|
else:
|
|||
|
|
alphabet = ascii_letters
|
|||
|
|
elif not alphabet:
|
|||
|
|
raise ValueError("Must supply alphabet if use_ascii is False")
|
|||
|
|
self.alphabet = "".join(sorted(set(alphabet))) if alphabet else None
|
|||
|
|
self.wiki_start_pages = wiki_start_pages
|
|||
|
|
|
|||
|
|
def __repr__(self):
|
|||
|
|
param_str = ", ".join(
|
|||
|
|
f"{k}={v!r}" for k, v in self.__dict__.items() if not k.startswith("_")
|
|||
|
|
)
|
|||
|
|
return f"{self.__class__.__name__}({param_str})"
|
|||
|
|
|
|||
|
|
|
|||
|
|
LANGUAGES = {
|
|||
|
|
"Arabic": Language(
|
|||
|
|
name="Arabic",
|
|||
|
|
iso_code="ar",
|
|||
|
|
use_ascii=False,
|
|||
|
|
# We only support encodings that use isolated
|
|||
|
|
# forms, because the current recommendation is
|
|||
|
|
# that the rendering system handles presentation
|
|||
|
|
# forms. This means we purposefully skip IBM864.
|
|||
|
|
charsets=["ISO-8859-6", "WINDOWS-1256", "CP720", "CP864"],
|
|||
|
|
alphabet="ءآأؤإئابةتثجحخدذرزسشصضطظعغػؼؽؾؿـفقكلمنهوىيًٌٍَُِّ",
|
|||
|
|
wiki_start_pages=["الصفحة_الرئيسية"],
|
|||
|
|
),
|
|||
|
|
"Belarusian": Language(
|
|||
|
|
name="Belarusian",
|
|||
|
|
iso_code="be",
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=["ISO-8859-5", "WINDOWS-1251", "IBM866", "MacCyrillic"],
|
|||
|
|
alphabet="АБВГДЕЁЖЗІЙКЛМНОПРСТУЎФХЦЧШЫЬЭЮЯабвгдеёжзійклмнопрстуўфхцчшыьэюяʼ",
|
|||
|
|
wiki_start_pages=["Галоўная_старонка"],
|
|||
|
|
),
|
|||
|
|
"Bulgarian": Language(
|
|||
|
|
name="Bulgarian",
|
|||
|
|
iso_code="bg",
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=["ISO-8859-5", "WINDOWS-1251", "IBM855"],
|
|||
|
|
alphabet="АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЬЮЯабвгдежзийклмнопрстуфхцчшщъьюя",
|
|||
|
|
wiki_start_pages=["Начална_страница"],
|
|||
|
|
),
|
|||
|
|
"Czech": Language(
|
|||
|
|
name="Czech",
|
|||
|
|
iso_code="cz",
|
|||
|
|
use_ascii=True,
|
|||
|
|
charsets=["ISO-8859-2", "WINDOWS-1250"],
|
|||
|
|
alphabet="áčďéěíňóřšťúůýžÁČĎÉĚÍŇÓŘŠŤÚŮÝŽ",
|
|||
|
|
wiki_start_pages=["Hlavní_strana"],
|
|||
|
|
),
|
|||
|
|
"Danish": Language(
|
|||
|
|
name="Danish",
|
|||
|
|
iso_code="da",
|
|||
|
|
use_ascii=True,
|
|||
|
|
charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252"],
|
|||
|
|
alphabet="æøåÆØÅ",
|
|||
|
|
wiki_start_pages=["Forside"],
|
|||
|
|
),
|
|||
|
|
"German": Language(
|
|||
|
|
name="German",
|
|||
|
|
iso_code="de",
|
|||
|
|
use_ascii=True,
|
|||
|
|
charsets=["ISO-8859-1", "WINDOWS-1252"],
|
|||
|
|
alphabet="äöüßÄÖÜ",
|
|||
|
|
wiki_start_pages=["Wikipedia:Hauptseite"],
|
|||
|
|
),
|
|||
|
|
"Greek": Language(
|
|||
|
|
name="Greek",
|
|||
|
|
iso_code="el",
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=["ISO-8859-7", "WINDOWS-1253"],
|
|||
|
|
alphabet="αβγδεζηθικλμνξοπρσςτυφχψωάέήίόύώΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΣΤΥΦΧΨΩΆΈΉΊΌΎΏ",
|
|||
|
|
wiki_start_pages=["Πύλη:Κύρια"],
|
|||
|
|
),
|
|||
|
|
"English": Language(
|
|||
|
|
name="English",
|
|||
|
|
iso_code="en",
|
|||
|
|
use_ascii=True,
|
|||
|
|
charsets=["ISO-8859-1", "WINDOWS-1252"],
|
|||
|
|
wiki_start_pages=["Main_Page"],
|
|||
|
|
),
|
|||
|
|
"Esperanto": Language(
|
|||
|
|
name="Esperanto",
|
|||
|
|
iso_code="eo",
|
|||
|
|
# Q, W, X, and Y not used at all
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=["ISO-8859-3"],
|
|||
|
|
alphabet="abcĉdefgĝhĥijĵklmnoprsŝtuŭvzABCĈDEFGĜHĤIJĴKLMNOPRSŜTUŬVZ",
|
|||
|
|
wiki_start_pages=["Vikipedio:Ĉefpaĝo"],
|
|||
|
|
),
|
|||
|
|
"Spanish": Language(
|
|||
|
|
name="Spanish",
|
|||
|
|
iso_code="es",
|
|||
|
|
use_ascii=True,
|
|||
|
|
charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252"],
|
|||
|
|
alphabet="ñáéíóúüÑÁÉÍÓÚÜ",
|
|||
|
|
wiki_start_pages=["Wikipedia:Portada"],
|
|||
|
|
),
|
|||
|
|
"Estonian": Language(
|
|||
|
|
name="Estonian",
|
|||
|
|
iso_code="et",
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=["ISO-8859-4", "ISO-8859-13", "WINDOWS-1257"],
|
|||
|
|
# C, F, Š, Q, W, X, Y, Z, Ž are only for
|
|||
|
|
# loanwords
|
|||
|
|
alphabet="ABDEGHIJKLMNOPRSTUVÕÄÖÜabdeghijklmnoprstuvõäöü",
|
|||
|
|
wiki_start_pages=["Esileht"],
|
|||
|
|
),
|
|||
|
|
"Finnish": Language(
|
|||
|
|
name="Finnish",
|
|||
|
|
iso_code="fi",
|
|||
|
|
use_ascii=True,
|
|||
|
|
charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252"],
|
|||
|
|
alphabet="ÅÄÖŠŽåäöšž",
|
|||
|
|
wiki_start_pages=["Wikipedia:Etusivu"],
|
|||
|
|
),
|
|||
|
|
"French": Language(
|
|||
|
|
name="French",
|
|||
|
|
iso_code="fr",
|
|||
|
|
use_ascii=True,
|
|||
|
|
charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252"],
|
|||
|
|
alphabet="œàâçèéîïùûêŒÀÂÇÈÉÎÏÙÛÊ",
|
|||
|
|
wiki_start_pages=["Wikipédia:Accueil_principal", "Bœuf (animal)"],
|
|||
|
|
),
|
|||
|
|
"Hebrew": Language(
|
|||
|
|
name="Hebrew",
|
|||
|
|
iso_code="he",
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=["ISO-8859-8", "WINDOWS-1255"],
|
|||
|
|
alphabet="אבגדהוזחטיךכלםמןנסעףפץצקרשתװױײ",
|
|||
|
|
wiki_start_pages=["עמוד_ראשי"],
|
|||
|
|
),
|
|||
|
|
"Croatian": Language(
|
|||
|
|
name="Croatian",
|
|||
|
|
iso_code="hr",
|
|||
|
|
# Q, W, X, Y are only used for foreign words.
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=["ISO-8859-2", "WINDOWS-1250"],
|
|||
|
|
alphabet="abcčćdđefghijklmnoprsštuvzžABCČĆDĐEFGHIJKLMNOPRSŠTUVZŽ",
|
|||
|
|
wiki_start_pages=["Glavna_stranica"],
|
|||
|
|
),
|
|||
|
|
"Hungarian": Language(
|
|||
|
|
name="Hungarian",
|
|||
|
|
iso_code="hu",
|
|||
|
|
# Q, W, X, Y are only used for foreign words.
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=["ISO-8859-2", "WINDOWS-1250"],
|
|||
|
|
alphabet="abcdefghijklmnoprstuvzáéíóöőúüűABCDEFGHIJKLMNOPRSTUVZÁÉÍÓÖŐÚÜŰ",
|
|||
|
|
wiki_start_pages=["Kezdőlap"],
|
|||
|
|
),
|
|||
|
|
"Italian": Language(
|
|||
|
|
name="Italian",
|
|||
|
|
iso_code="it",
|
|||
|
|
use_ascii=True,
|
|||
|
|
charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252"],
|
|||
|
|
alphabet="ÀÈÉÌÒÓÙàèéìòóù",
|
|||
|
|
wiki_start_pages=["Pagina_principale"],
|
|||
|
|
),
|
|||
|
|
"Lithuanian": Language(
|
|||
|
|
name="Lithuanian",
|
|||
|
|
iso_code="lt",
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=["ISO-8859-13", "WINDOWS-1257", "ISO-8859-4"],
|
|||
|
|
# Q, W, and X not used at all
|
|||
|
|
alphabet="AĄBCČDEĘĖFGHIĮYJKLMNOPRSŠTUŲŪVZŽaąbcčdeęėfghiįyjklmnoprsštuųūvzž",
|
|||
|
|
wiki_start_pages=["Pagrindinis_puslapis"],
|
|||
|
|
),
|
|||
|
|
"Latvian": Language(
|
|||
|
|
name="Latvian",
|
|||
|
|
iso_code="lv",
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=["ISO-8859-13", "WINDOWS-1257", "ISO-8859-4"],
|
|||
|
|
# Q, W, X, Y are only for loanwords
|
|||
|
|
alphabet="AĀBCČDEĒFGĢHIĪJKĶLĻMNŅOPRSŠTUŪVZŽaābcčdeēfgģhiījkķlļmnņoprsštuūvzž",
|
|||
|
|
wiki_start_pages=["Sākumlapa"],
|
|||
|
|
),
|
|||
|
|
"Macedonian": Language(
|
|||
|
|
name="Macedonian",
|
|||
|
|
iso_code="mk",
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=["ISO-8859-5", "WINDOWS-1251", "MacCyrillic", "IBM855"],
|
|||
|
|
alphabet="АБВГДЃЕЖЗЅИЈКЛЉМНЊОПРСТЌУФХЦЧЏШабвгдѓежзѕијклљмнњопрстќуфхцчџш",
|
|||
|
|
wiki_start_pages=["Главна_страница"],
|
|||
|
|
),
|
|||
|
|
"Dutch": Language(
|
|||
|
|
name="Dutch",
|
|||
|
|
iso_code="nl",
|
|||
|
|
use_ascii=True,
|
|||
|
|
charsets=["ISO-8859-1", "WINDOWS-1252"],
|
|||
|
|
wiki_start_pages=["Hoofdpagina"],
|
|||
|
|
),
|
|||
|
|
"Polish": Language(
|
|||
|
|
name="Polish",
|
|||
|
|
iso_code="pl",
|
|||
|
|
# Q and X are only used for foreign words.
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=["ISO-8859-2", "WINDOWS-1250"],
|
|||
|
|
alphabet="AĄBCĆDEĘFGHIJKLŁMNŃOÓPRSŚTUWYZŹŻaąbcćdeęfghijklłmnńoóprsśtuwyzźż",
|
|||
|
|
wiki_start_pages=["Wikipedia:Strona_główna"],
|
|||
|
|
),
|
|||
|
|
"Portuguese": Language(
|
|||
|
|
name="Portuguese",
|
|||
|
|
iso_code="pt",
|
|||
|
|
use_ascii=True,
|
|||
|
|
charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252"],
|
|||
|
|
alphabet="ÁÂÃÀÇÉÊÍÓÔÕÚáâãàçéêíóôõú",
|
|||
|
|
wiki_start_pages=["Wikipédia:Página_principal"],
|
|||
|
|
),
|
|||
|
|
"Romanian": Language(
|
|||
|
|
name="Romanian",
|
|||
|
|
iso_code="ro",
|
|||
|
|
use_ascii=True,
|
|||
|
|
charsets=["ISO-8859-2", "WINDOWS-1250"],
|
|||
|
|
alphabet="ăâîșțĂÂÎȘȚ",
|
|||
|
|
wiki_start_pages=["Pagina_principală"],
|
|||
|
|
),
|
|||
|
|
"Russian": Language(
|
|||
|
|
name="Russian",
|
|||
|
|
iso_code="ru",
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=[
|
|||
|
|
"ISO-8859-5",
|
|||
|
|
"WINDOWS-1251",
|
|||
|
|
"KOI8-R",
|
|||
|
|
"MacCyrillic",
|
|||
|
|
"IBM866",
|
|||
|
|
"IBM855",
|
|||
|
|
],
|
|||
|
|
alphabet="абвгдеёжзийклмнопрстуфхцчшщъыьэюяАБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ",
|
|||
|
|
wiki_start_pages=["Заглавная_страница"],
|
|||
|
|
),
|
|||
|
|
"Slovak": Language(
|
|||
|
|
name="Slovak",
|
|||
|
|
iso_code="sk",
|
|||
|
|
use_ascii=True,
|
|||
|
|
charsets=["ISO-8859-2", "WINDOWS-1250"],
|
|||
|
|
alphabet="áäčďéíĺľňóôŕšťúýžÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽ",
|
|||
|
|
wiki_start_pages=["Hlavná_stránka"],
|
|||
|
|
),
|
|||
|
|
"Slovene": Language(
|
|||
|
|
name="Slovene",
|
|||
|
|
iso_code="sl",
|
|||
|
|
# Q, W, X, Y are only used for foreign words.
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=["ISO-8859-2", "WINDOWS-1250"],
|
|||
|
|
alphabet="abcčdefghijklmnoprsštuvzžABCČDEFGHIJKLMNOPRSŠTUVZŽ",
|
|||
|
|
wiki_start_pages=["Glavna_stran"],
|
|||
|
|
),
|
|||
|
|
# Serbian can be written in both Latin and Cyrillic, but there's no
|
|||
|
|
# simple way to get the Latin alphabet pages from Wikipedia through
|
|||
|
|
# the API, so for now we just support Cyrillic.
|
|||
|
|
"Serbian": Language(
|
|||
|
|
name="Serbian",
|
|||
|
|
iso_code="sr",
|
|||
|
|
alphabet="АБВГДЂЕЖЗИЈКЛЉМНЊОПРСТЋУФХЦЧЏШабвгдђежзијклљмнњопрстћуфхцчџш",
|
|||
|
|
charsets=["ISO-8859-5", "WINDOWS-1251", "MacCyrillic", "IBM855"],
|
|||
|
|
wiki_start_pages=["Главна_страна"],
|
|||
|
|
),
|
|||
|
|
"Thai": Language(
|
|||
|
|
name="Thai",
|
|||
|
|
iso_code="th",
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=["ISO-8859-11", "TIS-620", "CP874"],
|
|||
|
|
alphabet="กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรฤลฦวศษสหฬอฮฯะัาำิีึืฺุู฿เแโใไๅๆ็่้๊๋์ํ๎๏๐๑๒๓๔๕๖๗๘๙๚๛",
|
|||
|
|
wiki_start_pages=["หน้าหลัก"],
|
|||
|
|
),
|
|||
|
|
"Turkish": Language(
|
|||
|
|
name="Turkish",
|
|||
|
|
iso_code="tr",
|
|||
|
|
# Q, W, and X are not used by Turkish
|
|||
|
|
use_ascii=False,
|
|||
|
|
charsets=["ISO-8859-3", "ISO-8859-9", "WINDOWS-1254"],
|
|||
|
|
alphabet="abcçdefgğhıijklmnoöprsştuüvyzâîûABCÇDEFGĞHIİJKLMNOÖPRSŞTUÜVYZÂÎÛ",
|
|||
|
|
wiki_start_pages=["Ana_Sayfa"],
|
|||
|
|
),
|
|||
|
|
"Vietnamese": Language(
|
|||
|
|
name="Vietnamese",
|
|||
|
|
iso_code="vi",
|
|||
|
|
use_ascii=False,
|
|||
|
|
# Windows-1258 is the only common 8-bit
|
|||
|
|
# Vietnamese encoding supported by Python.
|
|||
|
|
# From Wikipedia:
|
|||
|
|
# For systems that lack support for Unicode,
|
|||
|
|
# dozens of 8-bit Vietnamese code pages are
|
|||
|
|
# available.[1] The most common are VISCII
|
|||
|
|
# (TCVN 5712:1993), VPS, and Windows-1258.[3]
|
|||
|
|
# Where ASCII is required, such as when
|
|||
|
|
# ensuring readability in plain text e-mail,
|
|||
|
|
# Vietnamese letters are often encoded
|
|||
|
|
# according to Vietnamese Quoted-Readable
|
|||
|
|
# (VIQR) or VSCII Mnemonic (VSCII-MNEM),[4]
|
|||
|
|
# though usage of either variable-width
|
|||
|
|
# scheme has declined dramatically following
|
|||
|
|
# the adoption of Unicode on the World Wide
|
|||
|
|
# Web.
|
|||
|
|
charsets=["WINDOWS-1258"],
|
|||
|
|
alphabet="aăâbcdđeêghiklmnoôơpqrstuưvxyAĂÂBCDĐEÊGHIKLMNOÔƠPQRSTUƯVXY",
|
|||
|
|
wiki_start_pages=["Chữ_Quốc_ngữ"],
|
|||
|
|
),
|
|||
|
|
}
|