get-wikipron.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "g2p.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rFNOvGVsitRP"
      },
      "source": [
        "ใช้ wikipron https://pypi.org/project/wikipron/ ในการดึง"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IKyykXEObrEs",
        "outputId": "277cf73a-da05-4bac-aa89-e79bdcd74a63",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "!pip install wikipron"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting wikipron\n",
            "  Downloading https://files.pythonhosted.org/packages/9a/42/bad57092d3402afbe1dc83df4944a79d0180fb4419d8428b013a78f3ff58/wikipron-1.1.0-py3-none-any.whl\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from wikipron) (50.3.2)\n",
            "Collecting segments<3,>=2.1.3\n",
            "  Downloading https://files.pythonhosted.org/packages/5b/a0/0c3fe64787745c39eb3f2f5f5f9ed8d008d9ef22e9d7f9f52f71ea4712f7/segments-2.1.3-py2.py3-none-any.whl\n",
            "Collecting requests-html\n",
            "  Downloading https://files.pythonhosted.org/packages/24/bc/a4380f09bab3a776182578ce6b2771e57259d0d4dbce178205779abdc347/requests_html-0.10.0-py3-none-any.whl\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from wikipron) (2.23.0)\n",
            "Collecting iso639\n",
            "  Downloading https://files.pythonhosted.org/packages/d5/23/6aecf85ed735ff017af073d1da764b8f24822f8ec17798fc83816d166826/iso639-0.1.4.tar.gz\n",
            "Collecting csvw>=1.5.6\n",
            "  Downloading https://files.pythonhosted.org/packages/3f/e6/4faefc40bf1a3e8418cf97b5c3e7619d916692dd4202a5a1e40f36802f78/csvw-1.8.0-py2.py3-none-any.whl\n",
            "Collecting clldutils>=1.7.3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/4e/dd/6afa78497c503e8ad02596b27932df7f3216c0d10b4b7785d942a59b9cfd/clldutils-3.5.4-py2.py3-none-any.whl (189kB)\n",
            "\u001b[K     |████████████████████████████████| 194kB 7.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex in /usr/local/lib/python3.6/dist-packages (from segments<3,>=2.1.3->wikipron) (2019.12.20)\n",
            "Collecting w3lib\n",
            "  Downloading https://files.pythonhosted.org/packages/a3/59/b6b14521090e7f42669cafdb84b0ab89301a42f1f1a82fcf5856661ea3a7/w3lib-1.22.0-py2.py3-none-any.whl\n",
            "Collecting pyquery\n",
            "  Downloading https://files.pythonhosted.org/packages/78/43/95d42e386c61cb639d1a0b94f0c0b9f0b7d6b981ad3c043a836c8b5bc68b/pyquery-1.4.1-py2.py3-none-any.whl\n",
            "Collecting parse\n",
            "  Downloading https://files.pythonhosted.org/packages/b8/49/85f19d9ff908817b864deebf7f68211f9a6fc0b48746d372d970f60d01f5/parse-1.18.0.tar.gz\n",
            "Requirement already satisfied: bs4 in /usr/local/lib/python3.6/dist-packages (from requests-html->wikipron) (0.0.1)\n",
            "Collecting fake-useragent\n",
            "  Downloading https://files.pythonhosted.org/packages/d1/79/af647635d6968e2deb57a208d309f6069d31cb138066d7e821e575112a80/fake-useragent-0.1.11.tar.gz\n",
            "Collecting pyppeteer>=0.0.14\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/5d/4b/3c2aabdd1b91fa52aa9de6cde33b488b0592b4d48efb0ad9efbf71c49f5b/pyppeteer-0.2.2-py3-none-any.whl (145kB)\n",
            "\u001b[K     |████████████████████████████████| 153kB 25.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->wikipron) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->wikipron) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->wikipron) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->wikipron) (2020.6.20)\n",
            "Collecting isodate\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/9b/9f/b36f7774ff5ea8e428fdcfc4bb332c39ee5b9362ddd3d40d9516a55221b2/isodate-0.6.0-py2.py3-none-any.whl (45kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 6.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: attrs>=18.1 in /usr/local/lib/python3.6/dist-packages (from csvw>=1.5.6->segments<3,>=2.1.3->wikipron) (20.2.0)\n",
            "Requirement already satisfied: uritemplate>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from csvw>=1.5.6->segments<3,>=2.1.3->wikipron) (3.0.1)\n",
            "Requirement already satisfied: python-dateutil in /usr/local/lib/python3.6/dist-packages (from csvw>=1.5.6->segments<3,>=2.1.3->wikipron) (2.8.1)\n",
            "Collecting rfc3986\n",
            "  Downloading https://files.pythonhosted.org/packages/78/be/7b8b99fd74ff5684225f50dd0e865393d2265656ef3b4ba9eaaaffe622b8/rfc3986-1.4.0-py2.py3-none-any.whl\n",
            "Requirement already satisfied: tabulate>=0.7.7 in /usr/local/lib/python3.6/dist-packages (from clldutils>=1.7.3->segments<3,>=2.1.3->wikipron) (0.8.7)\n",
            "Collecting colorlog\n",
            "  Downloading https://files.pythonhosted.org/packages/8c/10/0b39be7ff1adb8888fe87c8628c071dec5ac282ac1c2312221f5feb09215/colorlog-4.4.0-py2.py3-none-any.whl\n",
            "Requirement already satisfied: six>=1.4.1 in /usr/local/lib/python3.6/dist-packages (from w3lib->requests-html->wikipron) (1.15.0)\n",
            "Collecting cssselect>0.7.9\n",
            "  Downloading https://files.pythonhosted.org/packages/3b/d4/3b5c17f00cce85b9a1e6f91096e1cc8e8ede2e1be8e96b87ce1ed09e92c5/cssselect-1.1.0-py2.py3-none-any.whl\n",
            "Requirement already satisfied: lxml>=2.1 in /usr/local/lib/python3.6/dist-packages (from pyquery->requests-html->wikipron) (4.2.6)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.6/dist-packages (from bs4->requests-html->wikipron) (4.6.3)\n",
            "Collecting appdirs<2.0.0,>=1.4.3\n",
            "  Downloading https://files.pythonhosted.org/packages/3b/00/2344469e2084fb287c2e0b57b72910309874c3245463acd6cf5e3db69324/appdirs-1.4.4-py2.py3-none-any.whl\n",
            "Collecting tqdm<5.0.0,>=4.42.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/93/3a/96b3dc293aa72443cf9627444c3c221a7ba34bb622e4d8bf1b5d4f2d9d08/tqdm-4.51.0-py2.py3-none-any.whl (70kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 7.0MB/s \n",
            "\u001b[?25hCollecting pyee<8.0.0,>=7.0.1\n",
            "  Downloading https://files.pythonhosted.org/packages/64/f3/90db6276dbc6cb1defa558251acc73c8e436ca8e1e2b38ec75786278de7c/pyee-7.0.4-py2.py3-none-any.whl\n",
            "Collecting websockets<9.0,>=8.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/bb/d9/856af84843912e2853b1b6e898ac8b802989fcf9ecf8e8445a1da263bf3b/websockets-8.1-cp36-cp36m-manylinux2010_x86_64.whl (78kB)\n",
            "\u001b[K     |████████████████████████████████| 81kB 8.1MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: iso639, parse, fake-useragent\n",
            "  Building wheel for iso639 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for iso639: filename=iso639-0.1.4-cp36-none-any.whl size=11176 sha256=9487ebbc6488a149faebb44fee7e11b17423ba90045f6371cd34a93707198544\n",
            "  Stored in directory: /root/.cache/pip/wheels/f4/d8/87/d4ab45e286639d860291010e0f5b377ee92a13ff5f4099e1a4\n",
            "  Building wheel for parse (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for parse: filename=parse-1.18.0-cp36-none-any.whl size=24133 sha256=8f0071c869390c625d1ce3ee12064ead3ded17f62fda5fe2312355d16925da59\n",
            "  Stored in directory: /root/.cache/pip/wheels/2a/53/09/869ca5781ede342254ffac09ca99461b008c3e5f8dd079b0c0\n",
            "  Building wheel for fake-useragent (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for fake-useragent: filename=fake_useragent-0.1.11-cp36-none-any.whl size=13485 sha256=16eda416fa0aea74eb5c676efe84e079239bddf79cad945f00e3ac5f591987f7\n",
            "  Stored in directory: /root/.cache/pip/wheels/5e/63/09/d1dc15179f175357d3f5c00cbffbac37f9e8690d80545143ff\n",
            "Successfully built iso639 parse fake-useragent\n",
            "\u001b[31mERROR: pyppeteer 0.2.2 has requirement urllib3<2.0.0,>=1.25.8, but you'll have urllib3 1.24.3 which is incompatible.\u001b[0m\n",
            "Installing collected packages: isodate, rfc3986, csvw, colorlog, clldutils, segments, w3lib, cssselect, pyquery, parse, fake-useragent, appdirs, tqdm, pyee, websockets, pyppeteer, requests-html, iso639, wikipron\n",
            "  Found existing installation: tqdm 4.41.1\n",
            "    Uninstalling tqdm-4.41.1:\n",
            "      Successfully uninstalled tqdm-4.41.1\n",
            "Successfully installed appdirs-1.4.4 clldutils-3.5.4 colorlog-4.4.0 cssselect-1.1.0 csvw-1.8.0 fake-useragent-0.1.11 iso639-0.1.4 isodate-0.6.0 parse-1.18.0 pyee-7.0.4 pyppeteer-0.2.2 pyquery-1.4.1 requests-html-0.10.0 rfc3986-1.4.0 segments-2.1.3 tqdm-4.51.0 w3lib-1.22.0 websockets-8.1 wikipron-1.1.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fTJGFAxkdM_d",
        "outputId": "6c7a34a6-013f-461e-9f04-579b30e5905a",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "!wikipron -h"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "usage: wikipron [-h] [--phonetic] [--no-stress] [--no-syllable-boundaries]\n",
            "                [--dialect DIALECT] [--casefold] [--cut-off-date CUT_OFF_DATE]\n",
            "                [--no-segment]\n",
            "                key\n",
            "\n",
            "Scrapes grapheme-to-phoneme data from Wiktionary.\n",
            "\n",
            "positional arguments:\n",
            "  key                   Key (i.e., ISO 639 code or name) for the language\n",
            "\n",
            "optional arguments:\n",
            "  -h, --help            show this help message and exit\n",
            "  --phonetic            Retrieve the [phonetic] transcriptions rather than the\n",
            "                        /phonemic/ ones.\n",
            "  --no-stress           Remove stress marks in pronunciations.\n",
            "  --no-syllable-boundaries\n",
            "                        Remove syllable boundary marks in pronunciations.\n",
            "  --dialect DIALECT     Retrieve entries that have this dialect specification.\n",
            "                        If not given, then all dialects are included in the\n",
            "                        output. The dialect name can be found together with\n",
            "                        the IPA transcription in the Wiktionary entries, e.g.,\n",
            "                        \"UK\" or \"US\" in \"(UK, US) IPA: /təˈmɑːtəʊ/\". To\n",
            "                        include more than one dialect, use a pipe \"|\" to\n",
            "                        separate the dialect names, e.g., --dialect=\"General\n",
            "                        American | US\". Note that whether or not --dialect is\n",
            "                        used, all entries that have no dialects specified are\n",
            "                        included in the output.\n",
            "  --casefold            Apply case-folding to the orthography.\n",
            "  --cut-off-date CUT_OFF_DATE\n",
            "                        Retrieve only entries that were added to Wiktionary on\n",
            "                        or before this date (in ISO format, e.g., 2018-10-23).\n",
            "                        If not given, today's date is used. Explicitly setting\n",
            "                        a cut-off date is useful if you want a relatively\n",
            "                        stable dataset no matter when you initiate a scraping\n",
            "                        run.\n",
            "  --no-segment          By default, the IPA pronunciation is segmented by\n",
            "                        whitespace and,to the extent possible, with a\n",
            "                        diacritic (either combining or modifier) immediately\n",
            "                        following the parent symbol. For example, \"kʰæt\" is\n",
            "                        segmented as \"kʰ æ t\", with kʰ conveniently segmented\n",
            "                        as an aspirated k for modeling purposes. To disable\n",
            "                        such IPA segmentation, apply this flag.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R0WyopClbv7t",
        "outputId": "0021dd8d-a9cb-446e-d08d-93e6666dbe43",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "!wikipron tha > tha2.tsv"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO: Language: \"Thai\"\n",
            "INFO: No cut-off date specified\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OC5fm63Gd2iW"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}