Merge pull request #661 from xxyzz/fr

[fr] translate some tags and topics in gloss and sound fields
tatuylonen · Jun 4, 2024 · 52de4fa · 52de4fa
2 parents e9e0a99 + 8638ff5
commit 52de4fa
Show file tree

Hide file tree

Showing 5 changed files with 102 additions and 25 deletions.
diff --git a/src/wiktextract/extractor/fr/pronunciation.py b/src/wiktextract/extractor/fr/pronunciation.py
@@ -7,6 +7,7 @@
 from ...wxr_context import WiktextractContext
 from ..share import set_sound_file_url_fields
 from .models import Sound, WordEntry
+from .tags import translate_raw_tags
 
 
 def extract_pronunciation(
@@ -85,12 +86,12 @@ def process_pron_list_item(
                 current_raw_tags.append(clean_node(wxr, None, list_item_child))
             elif list_item_child.kind == NodeKind.LINK:
                 for span_tag in list_item_child.find_html_recursively("span"):
-                    sounds_list.append(
-                        Sound(
-                            ipa=clean_node(wxr, None, span_tag),
-                            raw_tags=current_raw_tags[:],
-                        )
+                    sound = Sound(
+                        ipa=clean_node(wxr, None, span_tag),
+                        raw_tags=current_raw_tags[:],
                     )
+                    translate_raw_tags(sound)
+                    sounds_list.append(sound)
         elif isinstance(list_item_child, str):
             if ":" in list_item_child:
                 after_colon = True
@@ -100,6 +101,7 @@ def process_pron_list_item(
                 if len(pron_text) > 0:
                     sound = Sound(raw_tags=current_raw_tags[:])
                     setattr(sound, pron_key, pron_text)
+                    translate_raw_tags(sound)
                     sounds_list.append(sound)
 
     for nest_list_item in list_item_node.find_child_recursively(
@@ -159,6 +161,7 @@ def process_pron_template(
                 prons.add(pron_text)
                 sound = Sound(raw_tags=raw_tags[:])
                 setattr(sound, use_key, pron_text)
+                translate_raw_tags(sound)
                 sounds_list.append(sound)
     return sounds_list
 
@@ -193,6 +196,7 @@ def process_ecouter_template(
         sound.ipa = ipa
     if len(audio_file) > 0:
         set_sound_file_url_fields(wxr, audio_file, sound)
+    translate_raw_tags(sound)
     return sound
 
 
@@ -227,6 +231,7 @@ def process_pron_rimes_template(
             sound.rhymes = span_text
     if len(raw_tags) > 0:
         sound.raw_tags = raw_tags[:]
+    translate_raw_tags(sound)
     return sound
 
 

diff --git a/src/wiktextract/extractor/fr/tags.py b/src/wiktextract/extractor/fr/tags.py
@@ -150,6 +150,8 @@
 SENSE_TAGS: dict[str, str] = {
     # https://fr.wiktionary.org/wiki/Modèle:figuré
     # https://fr.wiktionary.org/wiki/Catégorie:Modèles_de_relation_entre_les_définitions
+    # Catégorie:Modèles de genre textuel
+    # Catégorie:Modèles de registre
     "sens figuré": "figuratively",
     "enclise": "enclitic",
     "idiotisme": "idiomatic",
@@ -161,10 +163,21 @@
     "argot": "slang",
     "rare": "rare",
     "plus rare": "rare",
-    "familier": "colloquial",
     "par extension": "broadly",
     "en particulier": "especially",
-    "informel": "informal",
+    "littéraire": "literary",  # Modèle:littéraire
+    "poétique": "poetic",  # Modèle:poétique
+    # "didactique": "",  # Modèle:didactique
+    "soutenu": "formal",  # Modèle:soutenu
+    "informel": "informal",  # Modèle:informel
+    "familier": "familiar",  # Modèle:familier
+    "très familier": "very-familiar",  # Modèle:très familier
+    # "populaire": "",  # Modèle:populaire
+    "vulgaire": "vulgar",  # Modèle:vulgaire
+    "langage enfantin": "childish",  # Modèle:enfantin
+    # Catégorie:Modèles de thématique
+    "anglicisme informatique": "Anglicism",
+    "proverbe": "proverb",
 }
 
 # https://en.wikipedia.org/wiki/Voice_(grammar)
@@ -197,6 +210,41 @@
     "analyse": "analytic",
 }
 
+# Template:cmn-pron
+# https://fr.wiktionary.org/wiki/自由
+ZH_PRON_TAGS = {
+    "pinyin": "Pinyin",
+    "efeo": "EFEO",  # https://en.wikipedia.org/wiki/EFEO_Chinese_transcription
+    "wade-giles": "Wade-Giles",
+    "yale": "Yale",
+    "zhuyin": "bopomofo",
+    "mandarin": "Mandarin",
+    "cantonais": "Cantonese",
+    "cantonais (yue)": "Cantonese",
+    "jyutping": "Jyutping",
+    "hakka": "Hakka",
+    "pha̍k-fa-sṳ": "Phak-fa-su",
+    "meixian, guangdong": ["Meixian", "Guangdong"],
+    "jin": "Jin",
+    "mindong": "Eastern-Min",
+    # https://en.wikipedia.org/wiki/Bàng-uâ-cê
+    "bàng-uâ-cê (fuzhou)": ["Bang-ua-ce", "Fuzhou"],
+    "minnan": "Min",
+    "pe̍h-ōe-jī (hokkien : fujian, taïwan)": [
+        "Peh-oe-ji",
+        "Hokkien",
+        "Fujian",
+        "Taiwan",
+    ],
+    "chaozhou, peng'im": ["Chaozhou", "Peng'im"],
+    "wu": "Wu",
+    "shanghai": "Shanghai",
+    "chinois médiéval": "Medieval-Chinese",
+    "chinois archaïque": "Old-Chinese",
+    "baxter-sagart": "Baxter-Sagart",
+    "zhengzhang": "Zhengzhang",
+}
+
 GRAMMATICAL_TAGS: dict[str, Union[str, list[str]]] = {
     **GENDER_TAGS,
     **NUMBER_TAGS,
@@ -214,6 +262,7 @@
     **SENSE_TAGS,
     **VOICE_TAGS,
     **LEXIQUE_TAGS,
+    **ZH_PRON_TAGS,
 }
 
 

diff --git a/src/wiktextract/extractor/fr/topics.py b/src/wiktextract/extractor/fr/topics.py
@@ -1,5 +1,6 @@
 # https://fr.wiktionary.org/wiki/Annexe:Glossaire_grammatical
 # https://fr.wiktionary.org/wiki/Module:lexique/data
+# Catégorie:Modèles de thématique
 
 TOPIC_TAGS: dict[str, str] = {
     "botanique": "botany",
@@ -62,6 +63,7 @@
     "chimie": "chemistry",
     "chirurgie": "surgery",
     "christianisme": "Christianity",
+    "cinéma": "film",
     "cirque": "circus",
     "climatologie": "climatology",
     "coiffure": "hairdressing",
@@ -277,4 +279,23 @@
     "boulangerie": "cooking",
     "bourse": "finance",
     "brasserie": "beverages manufacturing",
+    # Catégorie:Modèles de thématique
+    "phycologie": "phycology",
+    "bière": "beer",
+    "chronologie": "chronology",
+    "dialectologie": "dialectology",
+    "média": "media",
+    "état": "state",
+    "mobilier": "furniture",
+    "science": "science",
+    "boisson": "beverages",
+    "vêtement": "clothing",
+    "électronique": "electricity",
+    "carte à jouer": "card-games",
+    "lgbt": "LGBT",
+    "jeu vidéo, minecraft": "video-games",
+    "réseaux": "networking",
+    "science-fiction, univers de star wars": "science-fiction",
+    "textile": "textiles",
+    "transport": "transport",
 }
diff --git a/tests/test_fr_gloss.py b/tests/test_fr_gloss.py
@@ -453,7 +453,9 @@ def test_variante_kyujitai_de(self):
         self.wxr.wtp.add_page(
             "Modèle:désuet",
             10,
-            '<span class="emploi"><span id="désuet"></span>''(<span class="texte">[[Annexe:Glossaire grammatical#D|Désuet]]</span>)''</span>[[Catégorie:Termes désuets en japonais]]',
+            '<span class="emploi"><span id="désuet"></span>'
+            '(<span class="texte">[[Annexe:Glossaire grammatical#D|Désuet]]</span>)'
+            "</span>[[Catégorie:Termes désuets en japonais]]",
         )
         self.wxr.wtp.add_page(
             "Modèle:lien",

diff --git a/tests/test_fr_pronunciation.py b/tests/test_fr_pronunciation.py
@@ -98,7 +98,7 @@ def test_str_pron(self):
                 sound.model_dump(exclude_defaults=True)
                 for sound in page_data[-1].sounds
             ],
-            [{"raw_tags": ["cantonais", "Yale"], "zh_pron": "nei⁵hou²"}],
+            [{"tags": ["Cantonese", "Yale"], "zh_pron": "nei⁵hou²"}],
         )
 
     def test_no_ipa(self):
@@ -231,21 +231,21 @@ def test_cmn_pron(self):
         self.assertEqual(
             [s.model_dump(exclude_defaults=True) for s in page_data[0].sounds],
             [
-                {"ipa": "\\t͡su̯ɔ˥\\", "raw_tags": ["mandarin"]},
-                {"ipa": "\\t͡su̯ɔ˧˥\\", "raw_tags": ["mandarin"]},
-                {"ipa": "\\t͡su̯ɔ˥˩\\", "raw_tags": ["mandarin"]},
-                {"zh_pron": "zuō", "raw_tags": ["mandarin", "Pinyin"]},
-                {"zh_pron": "zuó", "raw_tags": ["mandarin", "Pinyin"]},
-                {"zh_pron": "zuò", "raw_tags": ["mandarin", "Pinyin"]},
-                {"zh_pron": "tso", "raw_tags": ["mandarin", "EFEO"]},
-                {"zh_pron": "tso¹", "raw_tags": ["mandarin", "Wade-Giles"]},
-                {"zh_pron": "tso²", "raw_tags": ["mandarin", "Wade-Giles"]},
-                {"zh_pron": "tso⁴", "raw_tags": ["mandarin", "Wade-Giles"]},
-                {"zh_pron": "dzwō", "raw_tags": ["mandarin", "Yale"]},
-                {"zh_pron": "dzwó", "raw_tags": ["mandarin", "Yale"]},
-                {"zh_pron": "dzwò", "raw_tags": ["mandarin", "Yale"]},
-                {"zh_pron": "ㄗㄨㄛ", "raw_tags": ["mandarin", "Zhuyin"]},
-                {"zh_pron": "ㄗㄨㄛˊ", "raw_tags": ["mandarin", "Zhuyin"]},
-                {"zh_pron": "ㄗㄨㄛˋ", "raw_tags": ["mandarin", "Zhuyin"]},
+                {"ipa": "\\t͡su̯ɔ˥\\", "tags": ["Mandarin"]},
+                {"ipa": "\\t͡su̯ɔ˧˥\\", "tags": ["Mandarin"]},
+                {"ipa": "\\t͡su̯ɔ˥˩\\", "tags": ["Mandarin"]},
+                {"zh_pron": "zuō", "tags": ["Mandarin", "Pinyin"]},
+                {"zh_pron": "zuó", "tags": ["Mandarin", "Pinyin"]},
+                {"zh_pron": "zuò", "tags": ["Mandarin", "Pinyin"]},
+                {"zh_pron": "tso", "tags": ["Mandarin", "EFEO"]},
+                {"zh_pron": "tso¹", "tags": ["Mandarin", "Wade-Giles"]},
+                {"zh_pron": "tso²", "tags": ["Mandarin", "Wade-Giles"]},
+                {"zh_pron": "tso⁴", "tags": ["Mandarin", "Wade-Giles"]},
+                {"zh_pron": "dzwō", "tags": ["Mandarin", "Yale"]},
+                {"zh_pron": "dzwó", "tags": ["Mandarin", "Yale"]},
+                {"zh_pron": "dzwò", "tags": ["Mandarin", "Yale"]},
+                {"zh_pron": "ㄗㄨㄛ", "tags": ["Mandarin", "bopomofo"]},
+                {"zh_pron": "ㄗㄨㄛˊ", "tags": ["Mandarin", "bopomofo"]},
+                {"zh_pron": "ㄗㄨㄛˋ", "tags": ["Mandarin", "bopomofo"]},
             ],
         )