SpaCy_Chinese.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/bradbradlin/Machine-Learning-Projects/blob/main/SpaCy_Chinese.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')\n",
        "%cd /content/gdrive/My Drive/Colab Notebooks/spacy"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FlODUHMVhASP",
        "outputId": "64bafd2b-1ea0-4c75-8d3d-6fa0b651ff98"
      },
      "id": "FlODUHMVhASP",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n",
            "/content/gdrive/My Drive/Colab Notebooks/spacy\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip3 install virtualenv\n",
        "!virtualenv theanoEnv\n",
        "!source /content/theanoEnv/bin/activate; pip3 install theano"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NzIQLqOQdt9U",
        "outputId": "19ab4f86-1408-497f-f892-69b1382af20f"
      },
      "id": "NzIQLqOQdt9U",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: virtualenv in /usr/local/lib/python3.7/dist-packages (20.12.0)\n",
            "Requirement already satisfied: distlib<1,>=0.3.1 in /usr/local/lib/python3.7/dist-packages (from virtualenv) (0.3.4)\n",
            "Requirement already satisfied: platformdirs<3,>=2 in /usr/local/lib/python3.7/dist-packages (from virtualenv) (2.4.1)\n",
            "Requirement already satisfied: six<2,>=1.9.0 in /usr/local/lib/python3.7/dist-packages (from virtualenv) (1.15.0)\n",
            "Requirement already satisfied: filelock<4,>=3.2 in /usr/local/lib/python3.7/dist-packages (from virtualenv) (3.4.0)\n",
            "Requirement already satisfied: importlib-metadata>=0.12 in /usr/local/lib/python3.7/dist-packages (from virtualenv) (4.8.2)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=0.12->virtualenv) (3.10.0.2)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=0.12->virtualenv) (3.6.0)\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n",
            "created virtual environment CPython3.7.12.final.0-64 in 7551ms\n",
            "  creator CPython3Posix(dest=/content/gdrive/MyDrive/Colab Notebooks/spacy/theanoEnv, clear=False, no_vcs_ignore=False, global=False)\n",
            "  seeder FromAppData(download=False, pip=bundle, setuptools=bundle, wheel=bundle, via=copy, app_data_dir=/root/.local/share/virtualenv)\n",
            "    added seed packages: pip==21.3.1, setuptools==60.2.0, wheel==0.37.1\n",
            "  activators BashActivator,CShellActivator,FishActivator,NushellActivator,PowerShellActivator,PythonActivator\n",
            "/bin/bash: /content/theanoEnv/bin/activate: No such file or directory\n",
            "Requirement already satisfied: theano in /usr/local/lib/python3.7/dist-packages (1.0.5)\n",
            "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.7/dist-packages (from theano) (1.15.0)\n",
            "Requirement already satisfied: scipy>=0.14 in /usr/local/lib/python3.7/dist-packages (from theano) (1.4.1)\n",
            "Requirement already satisfied: numpy>=1.9.1 in /usr/local/lib/python3.7/dist-packages (from theano) (1.19.5)\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -U pip setuptools wheel\n",
        "!pip3 install -U spacy[transformers,lookups]\n",
        "!python3 -m spacy download zh_core_web_trf"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e0IVkrUehI9z",
        "outputId": "6d80fc2b-9190-4d8d-ce83-223357ba2591"
      },
      "id": "e0IVkrUehI9z",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: pip in /usr/local/lib/python3.7/dist-packages (21.3.1)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (60.2.0)\n",
            "Requirement already satisfied: wheel in /usr/local/lib/python3.7/dist-packages (0.37.1)\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n",
            "Requirement already satisfied: spacy[lookups,transformers] in /usr/local/lib/python3.7/dist-packages (3.2.1)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy[lookups,transformers]) (0.4.1)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.8 in /usr/local/lib/python3.7/dist-packages (from spacy[lookups,transformers]) (3.0.8)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from spacy[lookups,transformers]) (1.0.1)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from spacy[lookups,transformers]) (0.8.2)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy[lookups,transformers]) (2.0.6)\n",
            "Requirement already satisfied: typer<0.5.0,>=0.3.0 in /usr/local/lib/python3.7/dist-packages (from spacy[lookups,transformers]) (0.4.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from spacy[lookups,transformers]) (60.2.0)\n",
            "Requirement already satisfied: thinc<8.1.0,>=8.0.12 in /usr/local/lib/python3.7/dist-packages (from spacy[lookups,transformers]) (8.0.13)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy[lookups,transformers]) (3.0.6)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.7/dist-packages (from spacy[lookups,transformers]) (4.62.3)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.7/dist-packages (from spacy[lookups,transformers]) (1.19.5)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.7/dist-packages (from spacy[lookups,transformers]) (2.0.6)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy[lookups,transformers]) (1.8.2)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.7/dist-packages (from spacy[lookups,transformers]) (2.23.0)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.1 in /usr/local/lib/python3.7/dist-packages (from spacy[lookups,transformers]) (2.4.2)\n",
            "Requirement already satisfied: typing-extensions<4.0.0.0,>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy[lookups,transformers]) (3.10.0.2)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.7/dist-packages (from spacy[lookups,transformers]) (3.3.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from spacy[lookups,transformers]) (21.3)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy[lookups,transformers]) (1.0.6)\n",
            "Requirement already satisfied: pathy>=0.3.5 in /usr/local/lib/python3.7/dist-packages (from spacy[lookups,transformers]) (0.6.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from spacy[lookups,transformers]) (2.11.3)\n",
            "Requirement already satisfied: spacy-lookups-data<1.1.0,>=1.0.3 in /usr/local/lib/python3.7/dist-packages (from spacy[lookups,transformers]) (1.0.3)\n",
            "Requirement already satisfied: spacy-transformers<1.2.0,>=1.1.2 in /usr/local/lib/python3.7/dist-packages (from spacy[lookups,transformers]) (1.1.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from catalogue<2.1.0,>=2.0.6->spacy[lookups,transformers]) (3.6.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->spacy[lookups,transformers]) (3.0.6)\n",
            "Requirement already satisfied: smart-open<6.0.0,>=5.0.0 in /usr/local/lib/python3.7/dist-packages (from pathy>=0.3.5->spacy[lookups,transformers]) (5.2.1)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy[lookups,transformers]) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy[lookups,transformers]) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy[lookups,transformers]) (2021.10.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy[lookups,transformers]) (1.24.3)\n",
            "Requirement already satisfied: torch>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from spacy-transformers<1.2.0,>=1.1.2->spacy[lookups,transformers]) (1.10.0+cu111)\n",
            "Requirement already satisfied: transformers<4.13.0,>=3.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy-transformers<1.2.0,>=1.1.2->spacy[lookups,transformers]) (4.12.5)\n",
            "Requirement already satisfied: spacy-alignments<1.0.0,>=0.7.2 in /usr/local/lib/python3.7/dist-packages (from spacy-transformers<1.2.0,>=1.1.2->spacy[lookups,transformers]) (0.8.4)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.7/dist-packages (from typer<0.5.0,>=0.3.0->spacy[lookups,transformers]) (7.1.2)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->spacy[lookups,transformers]) (2.0.1)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers<4.13.0,>=3.4.0->spacy-transformers<1.2.0,>=1.1.2->spacy[lookups,transformers]) (0.0.46)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers<4.13.0,>=3.4.0->spacy-transformers<1.2.0,>=1.1.2->spacy[lookups,transformers]) (3.4.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers<4.13.0,>=3.4.0->spacy-transformers<1.2.0,>=1.1.2->spacy[lookups,transformers]) (4.8.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers<4.13.0,>=3.4.0->spacy-transformers<1.2.0,>=1.1.2->spacy[lookups,transformers]) (6.0)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in /usr/local/lib/python3.7/dist-packages (from transformers<4.13.0,>=3.4.0->spacy-transformers<1.2.0,>=1.1.2->spacy[lookups,transformers]) (0.2.1)\n",
            "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /usr/local/lib/python3.7/dist-packages (from transformers<4.13.0,>=3.4.0->spacy-transformers<1.2.0,>=1.1.2->spacy[lookups,transformers]) (0.10.3)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers<4.13.0,>=3.4.0->spacy-transformers<1.2.0,>=1.1.2->spacy[lookups,transformers]) (2019.12.20)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers<4.13.0,>=3.4.0->spacy-transformers<1.2.0,>=1.1.2->spacy[lookups,transformers]) (1.1.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers<4.13.0,>=3.4.0->spacy-transformers<1.2.0,>=1.1.2->spacy[lookups,transformers]) (1.15.0)\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n",
            "Collecting zh-core-web-trf==3.2.0\n",
            "  Downloading https://github.com/explosion/spacy-models/releases/download/zh_core_web_trf-3.2.0/zh_core_web_trf-3.2.0-py3-none-any.whl (417.4 MB)\n",
            "     |████████████████████████████████| 417.4 MB 17 kB/s               \n",
            "\u001b[?25hRequirement already satisfied: spacy-pkuseg<0.1.0,>=0.0.27 in /usr/local/lib/python3.7/dist-packages (from zh-core-web-trf==3.2.0) (0.0.28)\n",
            "Requirement already satisfied: spacy-transformers<1.2.0,>=1.1.2 in /usr/local/lib/python3.7/dist-packages (from zh-core-web-trf==3.2.0) (1.1.3)\n",
            "Requirement already satisfied: spacy<3.3.0,>=3.2.0 in /usr/local/lib/python3.7/dist-packages (from zh-core-web-trf==3.2.0) (3.2.1)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.8 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->zh-core-web-trf==3.2.0) (3.0.8)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->zh-core-web-trf==3.2.0) (4.62.3)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->zh-core-web-trf==3.2.0) (2.0.6)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->zh-core-web-trf==3.2.0) (1.0.1)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->zh-core-web-trf==3.2.0) (1.19.5)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->zh-core-web-trf==3.2.0) (0.8.2)\n",
            "Requirement already satisfied: typing-extensions<4.0.0.0,>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->zh-core-web-trf==3.2.0) (3.10.0.2)\n",
            "Requirement already satisfied: pathy>=0.3.5 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->zh-core-web-trf==3.2.0) (0.6.1)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->zh-core-web-trf==3.2.0) (1.0.6)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->zh-core-web-trf==3.2.0) (3.0.6)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->zh-core-web-trf==3.2.0) (1.8.2)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->zh-core-web-trf==3.2.0) (0.4.1)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->zh-core-web-trf==3.2.0) (60.2.0)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->zh-core-web-trf==3.2.0) (3.3.0)\n",
            "Requirement already satisfied: thinc<8.1.0,>=8.0.12 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->zh-core-web-trf==3.2.0) (8.0.13)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->zh-core-web-trf==3.2.0) (21.3)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->zh-core-web-trf==3.2.0) (2.0.6)\n",
            "Requirement already satisfied: typer<0.5.0,>=0.3.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->zh-core-web-trf==3.2.0) (0.4.0)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->zh-core-web-trf==3.2.0) (2.11.3)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.1 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->zh-core-web-trf==3.2.0) (2.4.2)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->zh-core-web-trf==3.2.0) (2.23.0)\n",
            "Requirement already satisfied: cython>=0.25 in /usr/local/lib/python3.7/dist-packages (from spacy-pkuseg<0.1.0,>=0.0.27->zh-core-web-trf==3.2.0) (0.29.24)\n",
            "Requirement already satisfied: transformers<4.13.0,>=3.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy-transformers<1.2.0,>=1.1.2->zh-core-web-trf==3.2.0) (4.12.5)\n",
            "Requirement already satisfied: torch>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from spacy-transformers<1.2.0,>=1.1.2->zh-core-web-trf==3.2.0) (1.10.0+cu111)\n",
            "Requirement already satisfied: spacy-alignments<1.0.0,>=0.7.2 in /usr/local/lib/python3.7/dist-packages (from spacy-transformers<1.2.0,>=1.1.2->zh-core-web-trf==3.2.0) (0.8.4)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from catalogue<2.1.0,>=2.0.6->spacy<3.3.0,>=3.2.0->zh-core-web-trf==3.2.0) (3.6.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->spacy<3.3.0,>=3.2.0->zh-core-web-trf==3.2.0) (3.0.6)\n",
            "Requirement already satisfied: smart-open<6.0.0,>=5.0.0 in /usr/local/lib/python3.7/dist-packages (from pathy>=0.3.5->spacy<3.3.0,>=3.2.0->zh-core-web-trf==3.2.0) (5.2.1)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->zh-core-web-trf==3.2.0) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->zh-core-web-trf==3.2.0) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->zh-core-web-trf==3.2.0) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->zh-core-web-trf==3.2.0) (2021.10.8)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers<4.13.0,>=3.4.0->spacy-transformers<1.2.0,>=1.1.2->zh-core-web-trf==3.2.0) (0.0.46)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers<4.13.0,>=3.4.0->spacy-transformers<1.2.0,>=1.1.2->zh-core-web-trf==3.2.0) (6.0)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in /usr/local/lib/python3.7/dist-packages (from transformers<4.13.0,>=3.4.0->spacy-transformers<1.2.0,>=1.1.2->zh-core-web-trf==3.2.0) (0.2.1)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers<4.13.0,>=3.4.0->spacy-transformers<1.2.0,>=1.1.2->zh-core-web-trf==3.2.0) (4.8.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers<4.13.0,>=3.4.0->spacy-transformers<1.2.0,>=1.1.2->zh-core-web-trf==3.2.0) (3.4.0)\n",
            "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /usr/local/lib/python3.7/dist-packages (from transformers<4.13.0,>=3.4.0->spacy-transformers<1.2.0,>=1.1.2->zh-core-web-trf==3.2.0) (0.10.3)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers<4.13.0,>=3.4.0->spacy-transformers<1.2.0,>=1.1.2->zh-core-web-trf==3.2.0) (2019.12.20)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.7/dist-packages (from typer<0.5.0,>=0.3.0->spacy<3.3.0,>=3.2.0->zh-core-web-trf==3.2.0) (7.1.2)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->spacy<3.3.0,>=3.2.0->zh-core-web-trf==3.2.0) (2.0.1)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers<4.13.0,>=3.4.0->spacy-transformers<1.2.0,>=1.1.2->zh-core-web-trf==3.2.0) (1.1.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers<4.13.0,>=3.4.0->spacy-transformers<1.2.0,>=1.1.2->zh-core-web-trf==3.2.0) (1.15.0)\n",
            "Installing collected packages: zh-core-web-trf\n",
            "Successfully installed zh-core-web-trf-3.2.0\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the package via spacy.load('zh_core_web_trf')\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python -m spacy validate"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D6D1rjJomi4g",
        "outputId": "71d78731-1cd8-4775-b788-961e889b15c1"
      },
      "id": "D6D1rjJomi4g",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\r⠙ Loading compatibility table...\r\u001b[2K\u001b[38;5;2m✔ Loaded compatibility table\u001b[0m\n",
            "\u001b[1m\n",
            "================= Installed pipeline packages (spaCy v3.2.1) =================\u001b[0m\n",
            "\u001b[38;5;4mℹ spaCy installation: /usr/local/lib/python3.7/dist-packages/spacy\u001b[0m\n",
            "\n",
            "NAME              SPACY            VERSION                            \n",
            "zh_core_web_trf   >=3.2.0,<3.3.0   \u001b[38;5;2m3.2.0\u001b[0m   \u001b[38;5;2m✔\u001b[0m\n",
            "zh_core_web_sm    >=3.2.0,<3.3.0   \u001b[38;5;2m3.2.0\u001b[0m   \u001b[38;5;2m✔\u001b[0m\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "99d2148d",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "99d2148d",
        "outputId": "50a59599-ceb1-4eba-ac40-fb82fa74ee58"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'3.2.1'"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ],
      "source": [
        "import spacy\n",
        "from spacy import displacy\n",
        "from spacy.tokens import DocBin\n",
        "from collections import Counter\n",
        "import pandas as pd\n",
        "import json\n",
        "import codecs\n",
        "import ast\n",
        "import re\n",
        "\n",
        "#import en_core_web_trf\n",
        "import zh_core_web_trf\n",
        "#import zh_core_web_lg\n",
        "\n",
        "#spacy.prefer_gpu()\n",
        "\n",
        "pd.options.display.max_rows = 600\n",
        "pd.options.display.max_colwidth = 400\n",
        "\n",
        "spacy.__version__"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "4b950f58",
      "metadata": {
        "id": "4b950f58"
      },
      "outputs": [],
      "source": [
        "nlp_zh = zh_core_web_trf.load()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "5cab26bb",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5cab26bb",
        "outputId": "0c0e8fb4-06f9-49d9-a369-73c0770de19a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        }
      ],
      "source": [
        "text = '「香港再出發大聯盟」涉多個界別人士 包括李嘉誠父子 由全國政協副主席董建華和梁振英擔任總召集人的 「香港再出發大聯盟」成立，共同發起人則有 1545 人，涵蓋多個界別。地產商代表方面，包括長和系資深顧 問李嘉誠、兒子李澤鉅和李澤楷;恒基的李兆基，以及兒子李家誠和李家傑。另外，還包括恒隆地產陳文 博、新世界發展鄭家純、鄭志剛，信和置業黃志祥、信德集團的何超鳳，以及新鴻基地產的郭基煇、合和實 業胡應湘等人。政界方面，多名港區全國人大代表、政協委員、行政會議成員及建制派立法會議員都是共同 發起人。建制派政壇元老吳康民、行會前召集人夏佳理、前人大常委范徐麗泰、立法會前主席曾鈺成都有份。前財政司司長梁錦松、前律政司司長袁國強及剛離任的問責局長劉怡翔、劉江華、楊偉雄和羅智光等人都在名單上。學界方面，多名本地大學校長包括港大張翔、中大段崇智、科大史維、浸大錢大康等都榜上有名，但名單上未見城大郭位;而教大學者、運房局前局長張炳良和港大法律學者陳弘毅亦是共同發起人之一。'\n",
        "\n",
        "document = nlp_zh(re.sub(r'(?<=[^\\W\\d_])\\s+(?=[^\\W\\d_])', '', text))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "6c51f276",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 367
        },
        "id": "6c51f276",
        "outputId": "06f7691c-5991-4f30-848c-1c06bcd05eed"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">「\n",
              "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    香港再出發大聯盟\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
              "</mark>\n",
              "」涉多個界別人士包括\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    李嘉誠\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "父子由全\n",
              "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    國政協\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
              "</mark>\n",
              "副主席\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    董建華\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "和\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    梁振英\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "擔任總召集人的 「\n",
              "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    香港再出發大聯盟\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
              "</mark>\n",
              "」成立，共同發起人則有 \n",
              "<mark class=\"entity\" style=\"background: #e4e7d2; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    1545\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">CARDINAL</span>\n",
              "</mark>\n",
              " 人，涵蓋多個界別。地產商代表方面，包括\n",
              "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    長\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
              "</mark>\n",
              "\n",
              "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    和\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
              "</mark>\n",
              "系資深顧問\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    李嘉誠\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "、兒子\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    李澤鉅\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "和\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    李澤楷\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "\n",
              "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    ;恒基\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
              "</mark>\n",
              "的\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    李兆基\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "，以及兒子\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    李家誠\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "和\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    李家傑\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "。另外，還包括\n",
              "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    恒隆地\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
              "</mark>\n",
              "\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    產陳\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    文博\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "、\n",
              "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    新世界發展\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
              "</mark>\n",
              "\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    鄭\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    家純\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "、\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    鄭志剛\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "，\n",
              "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    信和置業\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
              "</mark>\n",
              "\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    黃志祥\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "、\n",
              "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    信德集團\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
              "</mark>\n",
              "的\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    何超鳳\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "，以及\n",
              "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    新鴻基地\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
              "</mark>\n",
              "產的\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    郭基煇\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "、\n",
              "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    合和實業\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
              "</mark>\n",
              "\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    胡應湘\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "等人。政界方面，多名\n",
              "<mark class=\"entity\" style=\"background: #feca74; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    港區\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">GPE</span>\n",
              "</mark>\n",
              "全國人大代表、\n",
              "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    政協\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
              "</mark>\n",
              "委員、行政會議成員及建制派\n",
              "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    立法\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
              "</mark>\n",
              "會議員都是共同發起人。建制派政壇元\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    老吳\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    康民\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "、行會前召集人\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    夏佳理\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "、前\n",
              "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    人大\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
              "</mark>\n",
              "常委\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    范徐\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    麗泰\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "、\n",
              "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    立法\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
              "</mark>\n",
              "\n",
              "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    會前\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
              "</mark>\n",
              "主席\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    曾\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    鈺\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    成都\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "有份。\n",
              "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    前財政司\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
              "</mark>\n",
              "司長\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    梁錦松\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "、\n",
              "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    前律\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
              "</mark>\n",
              "\n",
              "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    政司\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
              "</mark>\n",
              "司長\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    袁國強\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "及剛離任的問責局長劉\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    怡翔\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "、\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    劉江華\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "、\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    楊偉雄\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "和\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    羅智光\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "等人都在名單上。學界方面，多名本地大學校長包括\n",
              "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    港\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
              "</mark>\n",
              "\n",
              "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    大\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
              "</mark>\n",
              "\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    張翔\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "、\n",
              "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    中大段\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
              "</mark>\n",
              "\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    崇智\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "、\n",
              "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    科大\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
              "</mark>\n",
              "\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    史維\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "、\n",
              "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    浸大錢\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
              "</mark>\n",
              "\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    大康\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "等都榜上有名，但名單上未見\n",
              "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    城大\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
              "</mark>\n",
              "\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    郭位\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              ";而教大學者、\n",
              "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    運房局\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
              "</mark>\n",
              "前局\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    長張\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    炳良\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "和\n",
              "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    港\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
              "</mark>\n",
              "\n",
              "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    大\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
              "</mark>\n",
              "法律學者\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    陳\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    弘毅\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              "亦是共同發起人\n",
              "<mark class=\"entity\" style=\"background: #e4e7d2; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    之一\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">CARDINAL</span>\n",
              "</mark>\n",
              "。</div></span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        }
      ],
      "source": [
        "displacy.render(document, style='ent', jupyter=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "5d392fd6",
      "metadata": {
        "scrolled": true,
        "id": "5d392fd6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2efa2e76-a526-41ea-cb31-1c78792d4970"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('香港再出發大聯盟', 'ORG'),\n",
              " ('李嘉誠', 'PERSON'),\n",
              " ('國政協', 'ORG'),\n",
              " ('董建華', 'PERSON'),\n",
              " ('梁振英', 'PERSON'),\n",
              " ('香港再出發大聯盟', 'ORG'),\n",
              " ('1545', 'CARDINAL'),\n",
              " ('長', 'ORG'),\n",
              " ('和', 'ORG'),\n",
              " ('李嘉誠', 'PERSON'),\n",
              " ('李澤鉅', 'PERSON'),\n",
              " ('李澤楷', 'PERSON'),\n",
              " (';恒基', 'ORG'),\n",
              " ('李兆基', 'PERSON'),\n",
              " ('李家誠', 'PERSON'),\n",
              " ('李家傑', 'PERSON'),\n",
              " ('恒隆地', 'ORG'),\n",
              " ('產陳', 'PERSON'),\n",
              " ('文博', 'PERSON'),\n",
              " ('新世界發展', 'ORG'),\n",
              " ('鄭', 'PERSON'),\n",
              " ('家純', 'PERSON'),\n",
              " ('鄭志剛', 'PERSON'),\n",
              " ('信和置業', 'ORG'),\n",
              " ('黃志祥', 'PERSON'),\n",
              " ('信德集團', 'ORG'),\n",
              " ('何超鳳', 'PERSON'),\n",
              " ('新鴻基地', 'ORG'),\n",
              " ('郭基煇', 'PERSON'),\n",
              " ('合和實業', 'ORG'),\n",
              " ('胡應湘', 'PERSON'),\n",
              " ('港區', 'GPE'),\n",
              " ('政協', 'ORG'),\n",
              " ('立法', 'ORG'),\n",
              " ('老吳', 'PERSON'),\n",
              " ('康民', 'PERSON'),\n",
              " ('夏佳理', 'PERSON'),\n",
              " ('人大', 'ORG'),\n",
              " ('范徐', 'PERSON'),\n",
              " ('麗泰', 'PERSON'),\n",
              " ('立法', 'ORG'),\n",
              " ('會前', 'ORG'),\n",
              " ('曾', 'PERSON'),\n",
              " ('鈺', 'PERSON'),\n",
              " ('成都', 'PERSON'),\n",
              " ('前財政司', 'ORG'),\n",
              " ('梁錦松', 'PERSON'),\n",
              " ('前律', 'ORG'),\n",
              " ('政司', 'ORG'),\n",
              " ('袁國強', 'PERSON'),\n",
              " ('怡翔', 'PERSON'),\n",
              " ('劉江華', 'PERSON'),\n",
              " ('楊偉雄', 'PERSON'),\n",
              " ('羅智光', 'PERSON'),\n",
              " ('港', 'ORG'),\n",
              " ('大', 'ORG'),\n",
              " ('張翔', 'PERSON'),\n",
              " ('中大段', 'ORG'),\n",
              " ('崇智', 'PERSON'),\n",
              " ('科大', 'ORG'),\n",
              " ('史維', 'PERSON'),\n",
              " ('浸大錢', 'ORG'),\n",
              " ('大康', 'PERSON'),\n",
              " ('城大', 'ORG'),\n",
              " ('郭位', 'PERSON'),\n",
              " ('運房局', 'ORG'),\n",
              " ('長張', 'PERSON'),\n",
              " ('炳良', 'PERSON'),\n",
              " ('港', 'ORG'),\n",
              " ('大', 'ORG'),\n",
              " ('陳', 'PERSON'),\n",
              " ('弘毅', 'PERSON'),\n",
              " ('之一', 'CARDINAL')]"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ],
      "source": [
        "# result_ls = [(ent.text, ent.label_) for ent in document.ents]\n",
        "# result_ls "
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def convert(lang: str, input_path):\n",
        "    \n",
        "    with codecs.open(input_path, 'r', 'utf-8') as f:\n",
        "        library = json.load(f)\n",
        "\n",
        "    nlp_zh = spacy.blank(lang)\n",
        "    db = DocBin()\n",
        "    stock_list = []\n",
        "    lang_list = []\n",
        "    txt_list = []\n",
        "    ner_list = []\n",
        "    check = ''\n",
        "    check2 = ''\n",
        "    temp_doc = ''\n",
        "    \n",
        "    for document in library:\n",
        "        if (document['language'] == lang):\n",
        "          temp1 = document['stock_code']\n",
        "          temp2 = document['language']\n",
        "\n",
        "          for txt in document['text']:\n",
        "              \n",
        "              temp3 = txt['text']\n",
        "              \n",
        "              for ner in ast.literal_eval(txt['ner']):\n",
        "                  stock_list.append(temp1)\n",
        "                  lang_list.append(temp2)\n",
        "                  txt_list.append(re.sub(r'(?<=[^\\W\\d_])\\s+(?=[^\\W\\d_])', '', temp3))\n",
        "                  #print(ner)\n",
        "                  ner_list.append(dict(ner.copy()))\n",
        "                  \n",
        "    df1 = pd.DataFrame(list(zip(stock_list, lang_list, txt_list)),\n",
        "                            columns=['Stock Code', 'Language', 'Paragraph'])\n",
        "    df1.drop_duplicates(inplace=True)\n",
        "\n",
        "    df1['Paragraph'] = df1.groupby(['Stock Code','Language'])['Paragraph'].transform(lambda x: ','.join(x))\n",
        "    df1.drop_duplicates(inplace=True, ignore_index=True)\n",
        "\n",
        "    return df1"
      ],
      "metadata": {
        "id": "7MUfUjB2qavI"
      },
      "id": "7MUfUjB2qavI",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df = convert('zh', './data/annual_report_23.json')\n",
        "df.to_excel('df.xlsx')\n",
        "df.head()"
      ],
      "metadata": {
        "id": "pyV7j1ClrJlF",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 615
        },
        "outputId": "2963bf73-0d86-423b-8eb9-1e25fdc125ab"
      },
      "id": "pyV7j1ClrJlF",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-58136f55-336e-4c39-8aed-488329c9a768\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Stock Code</th>\n",
              "      <th>Language</th>\n",
              "      <th>Paragraph</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>00005</td>\n",
              "      <td>zh</td>\n",
              "      <td>杜嘉祺（63歲） 集團主席2017年9月獲委加入董事會 2017年10月起擔任集團主席,才能及經驗：於亞洲及英國金融服務業擁有30多年經驗，對行業及集團經營所在市場瞭如指掌。,職業生涯：曾擔任友邦保險控股有限公司（「友邦」）集團首席執行官兼總裁。加入友邦前曾擔任英國保誠有限公司多個高級管理職務，包括任職集團行政總裁四年。曾在保誠董事會任職十年。,曾擔任英倫銀行非執行董事、高盛集團獨立非執行董事及HBOSplc集團財務董事。,外部職位：－ TheCityUK主席－ DiscoveryLimited非執行主席－ BuildBackBetterCouncil成員－ ChapterZero支援主席,祈耀年（59歲）集團行政總裁2019年8月獲委加入董事會 2020年3月起擔任集團行政總裁,才能及經驗：於英國及亞洲銀行業和金融服務業擁有30多年經驗，於滙豐任職超過28年。,職業生涯：自201...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>01919</td>\n",
              "      <td>zh</td>\n",
              "      <td>中遠海運控股股份有限公司二零二零年年報,許立榮先生1,許立榮先生，63歲，現任中國遠洋海運集團有限公司董事長、黨組書記；本公司執行董事、董事長，東方海外（國際）有限公司（一家本公司之非全資附屬公司及於聯交所上市之公司（股份代號：316））執行董事、董事會主席。許先生於一九七五年三月參加工作。許先生過往曾任上海遠洋運輸有限公司船舶管理一處副處長、總經理助理、副總經理及總經理；上遠貨運公司副經理、經理兼黨委書記；上海航運交易所的總裁及黨委書記；中遠海運集裝箱運輸有限公司的總經理、黨委委員及黨委副書記；本公司副總經理、黨委委員、副書記；中國遠洋運輸（集團）總公司（現稱為中國遠洋運輸有限公司，為直接控股股東）副總裁、工會主席及黨組成員；中國海運（集團）總公司（現稱為中國海運集團有限公司，中國遠洋海運的附屬公司）董事、總經理、黨組成員、董事長、黨組書記；東方海外貨櫃航運有限公司董事會主席、執...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>00016</td>\n",
              "      <td>zh</td>\n",
              "      <td>143新鴻基地產發展有限公司二○一九至二○年年報,郭炳聯主席兼董事總經理(67歲）,郭先生自二○一一年十二月起出任本公司主席。在獲委任為本公司主席前，郭先生一直出任本公司副主席達二十一年。 他亦是本公司董事總經理及執行委員會成員。他在本集團已服務四十二年，持有劍橋大學法律系碩士學位、哈佛大學工商管理碩士學位、香港公開大學榮譽工商管理博士學位及香港中文大學榮譽法學博士學位。郭先生是新意網集團有限公司主席及執行董事，他亦是數碼通電訊集團有限公司主席及非執行董事，以及載通國際控股有限公司和永泰地產有限公司之非執行董事。,社會公職方面，郭先生為中國人民政治協商會議第十三屆全國委員會委員。他亦是香港地產建設商會董事及香港中文大學校董會成員。,郭先生是鄺肖卿女士之兒子，鄺肖卿女士則是鄺準先生之胞姊及按證券及期貨條例第XV部所述本公司之主要股東。 郭先生是郭基泓先生及郭顥澧先生之父親，他亦是郭基...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>02318</td>\n",
              "      <td>zh</td>\n",
              "      <td>二零二零年年報中國平安保險（集團）股份有限公司134,從左至右 ：陳德賢先生蔡方方女士姚波先生謝永林先生馬明哲先生陳心穎女士陳克祥先生黃寶新先生葉素蘭女士,二零二零年年報中國平安保險（集團）股份有限公司 135,二零二零年年報中國平安保險（集團）股份有限公司136,馬明哲先生本公司創始人、創辦人董事長（執行董事）65歲自1988年3月起出任董事,謝永林先生執行董事、總經理、聯席首席執行官52歲於1994年加入本公司自2020年4月起出任董事,工作經歷自成立本公司以來，馬先生主持本公司全面經營管理工作至2020年6月不再擔任首席執行官，現主要負責本公司的戰略、人才、文化及重大事項決策，發揮核心領導作用。歷任本公司總經理、董事、董事長兼首席執行官。,在成立本公司之前，馬先生曾任招商局蛇口工業區社會保險公司副經理。,教育背景及資格中南財經政法大學（原中南財經大學）貨幣銀行學博士學位,於本...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>01929</td>\n",
              "      <td>zh</td>\n",
              "      <td>ChowTaiFookJewelleryGroupLimited周大福珠寶集團有限公司142Governance管治,董事會欣然提呈本公司2021財政年度年報以及本集團經審核綜合財務報表。,本公司為投資控股公司。本集團的主要業務為生產和銷售珠寶產品（包括珠寶鑲嵌、鉑金及K金首飾和黃金首飾及產品），以及分銷不同品牌的鐘錶。主要附屬公司的業務載於綜合財務報表附註41。,本集團的營業額來自中國內地、香港、澳門及其他市場進行的業務活動。本集團的業務分部資料載於綜合財務報表附註4。,本集團於2021財政年度的業績載列於第166頁的綜合損益及其他全面收益表內。,董事會建議派付2021財政年度每股0.24港元的末期股息，合共約2,400百萬港元（2020財政年度：末期股息每股0.12港元，金額約1,200百萬港元）。派付該等股息須待股東在本公司將於2021年7月28日（星期三）舉行的應屆股東週年...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-58136f55-336e-4c39-8aed-488329c9a768')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-58136f55-336e-4c39-8aed-488329c9a768 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-58136f55-336e-4c39-8aed-488329c9a768');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "  Stock Code  ...                                                                                                                                                                                                                                                                                                                                                                                                        Paragraph\n",
              "0      00005  ...  杜嘉祺（63歲） 集團主席2017年9月獲委加入董事會 2017年10月起擔任集團主席,才能及經驗：於亞洲及英國金融服務業擁有30多年經驗，對行業及集團經營所在市場瞭如指掌。,職業生涯：曾擔任友邦保險控股有限公司（「友邦」）集團首席執行官兼總裁。加入友邦前曾擔任英國保誠有限公司多個高級管理職務，包括任職集團行政總裁四年。曾在保誠董事會任職十年。,曾擔任英倫銀行非執行董事、高盛集團獨立非執行董事及HBOSplc集團財務董事。,外部職位：－ TheCityUK主席－ DiscoveryLimited非執行主席－ BuildBackBetterCouncil成員－ ChapterZero支援主席,祈耀年（59歲）集團行政總裁2019年8月獲委加入董事會 2020年3月起擔任集團行政總裁,才能及經驗：於英國及亞洲銀行業和金融服務業擁有30多年經驗，於滙豐任職超過28年。,職業生涯：自201...\n",
              "1      01919  ...  中遠海運控股股份有限公司二零二零年年報,許立榮先生1,許立榮先生，63歲，現任中國遠洋海運集團有限公司董事長、黨組書記；本公司執行董事、董事長，東方海外（國際）有限公司（一家本公司之非全資附屬公司及於聯交所上市之公司（股份代號：316））執行董事、董事會主席。許先生於一九七五年三月參加工作。許先生過往曾任上海遠洋運輸有限公司船舶管理一處副處長、總經理助理、副總經理及總經理；上遠貨運公司副經理、經理兼黨委書記；上海航運交易所的總裁及黨委書記；中遠海運集裝箱運輸有限公司的總經理、黨委委員及黨委副書記；本公司副總經理、黨委委員、副書記；中國遠洋運輸（集團）總公司（現稱為中國遠洋運輸有限公司，為直接控股股東）副總裁、工會主席及黨組成員；中國海運（集團）總公司（現稱為中國海運集團有限公司，中國遠洋海運的附屬公司）董事、總經理、黨組成員、董事長、黨組書記；東方海外貨櫃航運有限公司董事會主席、執...\n",
              "2      00016  ...  143新鴻基地產發展有限公司二○一九至二○年年報,郭炳聯主席兼董事總經理(67歲）,郭先生自二○一一年十二月起出任本公司主席。在獲委任為本公司主席前，郭先生一直出任本公司副主席達二十一年。 他亦是本公司董事總經理及執行委員會成員。他在本集團已服務四十二年，持有劍橋大學法律系碩士學位、哈佛大學工商管理碩士學位、香港公開大學榮譽工商管理博士學位及香港中文大學榮譽法學博士學位。郭先生是新意網集團有限公司主席及執行董事，他亦是數碼通電訊集團有限公司主席及非執行董事，以及載通國際控股有限公司和永泰地產有限公司之非執行董事。,社會公職方面，郭先生為中國人民政治協商會議第十三屆全國委員會委員。他亦是香港地產建設商會董事及香港中文大學校董會成員。,郭先生是鄺肖卿女士之兒子，鄺肖卿女士則是鄺準先生之胞姊及按證券及期貨條例第XV部所述本公司之主要股東。 郭先生是郭基泓先生及郭顥澧先生之父親，他亦是郭基...\n",
              "3      02318  ...  二零二零年年報中國平安保險（集團）股份有限公司134,從左至右 ：陳德賢先生蔡方方女士姚波先生謝永林先生馬明哲先生陳心穎女士陳克祥先生黃寶新先生葉素蘭女士,二零二零年年報中國平安保險（集團）股份有限公司 135,二零二零年年報中國平安保險（集團）股份有限公司136,馬明哲先生本公司創始人、創辦人董事長（執行董事）65歲自1988年3月起出任董事,謝永林先生執行董事、總經理、聯席首席執行官52歲於1994年加入本公司自2020年4月起出任董事,工作經歷自成立本公司以來，馬先生主持本公司全面經營管理工作至2020年6月不再擔任首席執行官，現主要負責本公司的戰略、人才、文化及重大事項決策，發揮核心領導作用。歷任本公司總經理、董事、董事長兼首席執行官。,在成立本公司之前，馬先生曾任招商局蛇口工業區社會保險公司副經理。,教育背景及資格中南財經政法大學（原中南財經大學）貨幣銀行學博士學位,於本...\n",
              "4      01929  ...  ChowTaiFookJewelleryGroupLimited周大福珠寶集團有限公司142Governance管治,董事會欣然提呈本公司2021財政年度年報以及本集團經審核綜合財務報表。,本公司為投資控股公司。本集團的主要業務為生產和銷售珠寶產品（包括珠寶鑲嵌、鉑金及K金首飾和黃金首飾及產品），以及分銷不同品牌的鐘錶。主要附屬公司的業務載於綜合財務報表附註41。,本集團的營業額來自中國內地、香港、澳門及其他市場進行的業務活動。本集團的業務分部資料載於綜合財務報表附註4。,本集團於2021財政年度的業績載列於第166頁的綜合損益及其他全面收益表內。,董事會建議派付2021財政年度每股0.24港元的末期股息，合共約2,400百萬港元（2020財政年度：末期股息每股0.12港元，金額約1,200百萬港元）。派付該等股息須待股東在本公司將於2021年7月28日（星期三）舉行的應屆股東週年...\n",
              "\n",
              "[5 rows x 3 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "c40a059d",
      "metadata": {
        "id": "c40a059d"
      },
      "outputs": [],
      "source": [
        "def convert2(cleansed_df, lang: str, output_path):\n",
        "    \n",
        "    nlp_zh = spacy.blank(lang)\n",
        "    db = DocBin()\n",
        "\n",
        "    for txt in cleansed_df['Paragraph']:\n",
        "      doc = nlp_zh.make_doc(txt.strip())\n",
        "      db.add(doc)\n",
        "     \n",
        "\n",
        "    db.to_disk(output_path)\n",
        "    \n",
        "    return db"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "a0dab44e",
      "metadata": {
        "id": "a0dab44e"
      },
      "outputs": [],
      "source": [
        "doc_bin = convert2(df, 'zh', './data/parsed.spacy')\n",
        "doc_ls = list(doc_bin.get_docs(nlp_zh.vocab))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "dddb8cbc",
      "metadata": {
        "id": "dddb8cbc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f004982e-e4c8-4f19-db43-f4de0995fdd9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        }
      ],
      "source": [
        "ents_ls = []\n",
        "for doc in doc_ls:\n",
        "    document = nlp_zh(doc)\n",
        "    ents_ls.append([(ent.text, ent.label_) for ent in document.ents])"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pickle\n",
        "\n",
        "with open('./data/ents_ls.pkl', 'wb') as f:\n",
        "  pickle.dump(ents_ls, f)\n"
      ],
      "metadata": {
        "id": "QiMd0Rp4p9iM"
      },
      "id": "QiMd0Rp4p9iM",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pickle\n",
        "\n",
        "with open('./data/ents_ls.pkl', 'rb') as f:\n",
        "  ents_ls = pickle.load(f)"
      ],
      "metadata": {
        "id": "_1drPAtRtiKp"
      },
      "id": "_1drPAtRtiKp",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "w4dl4kHGUgJm"
      },
      "id": "w4dl4kHGUgJm",
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.12"
    },
    "colab": {
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}