Nested_entity_bis.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ggEOnTyrcAf6"
      },
      "source": [
        "# Initialisation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZDCS-DUSWUnv",
        "outputId": "01ba0089-a111-4d16-a965-b66ffcf37d0e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting pytorch_lightning\n",
            "  Downloading pytorch_lightning-2.1.0-py3-none-any.whl (774 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m774.6/774.6 kB\u001b[0m \u001b[31m9.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17.2 in /usr/local/lib/python3.10/dist-packages (from pytorch_lightning) (1.23.5)\n",
            "Requirement already satisfied: torch>=1.12.0 in /usr/local/lib/python3.10/dist-packages (from pytorch_lightning) (2.1.0+cu118)\n",
            "Requirement already satisfied: tqdm>=4.57.0 in /usr/local/lib/python3.10/dist-packages (from pytorch_lightning) (4.66.1)\n",
            "Requirement already satisfied: PyYAML>=5.4 in /usr/local/lib/python3.10/dist-packages (from pytorch_lightning) (6.0.1)\n",
            "Requirement already satisfied: fsspec[http]>2021.06.0 in /usr/local/lib/python3.10/dist-packages (from pytorch_lightning) (2023.6.0)\n",
            "Collecting torchmetrics>=0.7.0 (from pytorch_lightning)\n",
            "  Downloading torchmetrics-1.2.0-py3-none-any.whl (805 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m805.2/805.2 kB\u001b[0m \u001b[31m53.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from pytorch_lightning) (23.2)\n",
            "Requirement already satisfied: typing-extensions>=4.0.0 in /usr/local/lib/python3.10/dist-packages (from pytorch_lightning) (4.5.0)\n",
            "Collecting lightning-utilities>=0.8.0 (from pytorch_lightning)\n",
            "  Downloading lightning_utilities-0.9.0-py3-none-any.whl (23 kB)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from fsspec[http]>2021.06.0->pytorch_lightning) (2.31.0)\n",
            "Requirement already satisfied: aiohttp!=4.0.0a0,!=4.0.0a1 in /usr/local/lib/python3.10/dist-packages (from fsspec[http]>2021.06.0->pytorch_lightning) (3.8.6)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.12.0->pytorch_lightning) (3.12.4)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.12.0->pytorch_lightning) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.12.0->pytorch_lightning) (3.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.12.0->pytorch_lightning) (3.1.2)\n",
            "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.12.0->pytorch_lightning) (2.1.0)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch_lightning) (23.1.0)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch_lightning) (3.3.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch_lightning) (6.0.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch_lightning) (4.0.3)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch_lightning) (1.9.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch_lightning) (1.4.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch_lightning) (1.3.1)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.12.0->pytorch_lightning) (2.1.3)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->fsspec[http]>2021.06.0->pytorch_lightning) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->fsspec[http]>2021.06.0->pytorch_lightning) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->fsspec[http]>2021.06.0->pytorch_lightning) (2023.7.22)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.12.0->pytorch_lightning) (1.3.0)\n",
            "Installing collected packages: lightning-utilities, torchmetrics, pytorch_lightning\n",
            "Successfully installed lightning-utilities-0.9.0 pytorch_lightning-2.1.0 torchmetrics-1.2.0\n",
            "Collecting datasets\n",
            "  Downloading datasets-2.14.5-py3-none-any.whl (519 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m519.6/519.6 kB\u001b[0m \u001b[31m7.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from datasets) (1.23.5)\n",
            "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (9.0.0)\n",
            "Collecting dill<0.3.8,>=0.3.0 (from datasets)\n",
            "  Downloading dill-0.3.7-py3-none-any.whl (115 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m9.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (1.5.3)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (4.66.1)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets) (3.4.1)\n",
            "Collecting multiprocess (from datasets)\n",
            "  Downloading multiprocess-0.70.15-py310-none-any.whl (134 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m16.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: fsspec[http]<2023.9.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2023.6.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.8.6)\n",
            "Collecting huggingface-hub<1.0.0,>=0.14.0 (from datasets)\n",
            "  Downloading huggingface_hub-0.18.0-py3-none-any.whl (301 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m302.0/302.0 kB\u001b[0m \u001b[31m21.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets) (23.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (6.0.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.1.0)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (3.3.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0.0,>=0.14.0->datasets) (3.12.4)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0.0,>=0.14.0->datasets) (4.5.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2023.7.22)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2023.3.post1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n",
            "Installing collected packages: dill, multiprocess, huggingface-hub, datasets\n",
            "Successfully installed datasets-2.14.5 dill-0.3.7 huggingface-hub-0.18.0 multiprocess-0.70.15\n",
            "Collecting sentencepiece\n",
            "  Downloading sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m8.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: sentencepiece\n",
            "Successfully installed sentencepiece-0.1.99\n",
            "Collecting transformers\n",
            "  Using cached transformers-4.34.1-py3-none-any.whl (7.7 MB)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.4)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.18.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.23.5)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.6.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n",
            "Collecting tokenizers<0.15,>=0.14 (from transformers)\n",
            "  Downloading tokenizers-0.14.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.8/3.8 MB\u001b[0m \u001b[31m29.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting safetensors>=0.3.1 (from transformers)\n",
            "  Downloading safetensors-0.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m74.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.1)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->transformers) (2023.6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->transformers) (4.5.0)\n",
            "Collecting huggingface-hub<1.0,>=0.16.4 (from transformers)\n",
            "  Downloading huggingface_hub-0.17.3-py3-none-any.whl (295 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m295.0/295.0 kB\u001b[0m \u001b[31m30.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.3.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2023.7.22)\n",
            "Installing collected packages: safetensors, huggingface-hub, tokenizers, transformers\n",
            "  Attempting uninstall: huggingface-hub\n",
            "    Found existing installation: huggingface-hub 0.18.0\n",
            "    Uninstalling huggingface-hub-0.18.0:\n",
            "      Successfully uninstalled huggingface-hub-0.18.0\n",
            "Successfully installed huggingface-hub-0.17.3 safetensors-0.4.0 tokenizers-0.14.1 transformers-4.34.1\n"
          ]
        }
      ],
      "source": [
        "!pip install transformers plotly==5.8.0 pyyaml==5.4.1 datasets pytorch-lightning > /dev/null 2>&1\n",
        "!pip install pytorch_lightning\n",
        "!pip install datasets\n",
        "!pip install sentencepiece\n",
        "!pip install transformers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Dt8FlZBnWeFo"
      },
      "outputs": [],
      "source": [
        "from transformers import pipeline, AutoModelForTokenClassification, AutoTokenizer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hATpQbZxXKLr"
      },
      "outputs": [],
      "source": [
        "# Liens des différents NER pré-programmé\n",
        "jb = \"Jean-Baptiste/camembert-ner-with-dates\"\n",
        "dbmdz = \"dbmdz/bert-base-french-europeana-cased\"\n",
        "dslim = \"dslim/bert-base-NER\"\n",
        "davlan = \"Davlan/bert-base-multilingual-cased-ner-hrl\"\n",
        "test_Adam = \"nlpso/m1_ind_layers_ref_ptrn_cmbert_io_level_1\"\n",
        "test_Adam2=\"nlpso/m1_ind_layers_ref_ptrn_cmbert_io_level_2\"\n",
        "\n",
        "# Initialisation des pipelines\n",
        "def ner_init(lien):\n",
        "  ner = pipeline(\n",
        "      'ner',\n",
        "      model= AutoModelForTokenClassification.from_pretrained(lien),\n",
        "      tokenizer= AutoTokenizer.from_pretrained(lien),\n",
        "      aggregation_strategy= \"simple\" )\n",
        "  return ner"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a73mKirofGZb",
        "outputId": "a38b2741-d768-4bb0-d1be-67eaa4398aaf"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Journl d'un Villageois Médiéval en France\t10 avril 1365\tAujourd'hui, le soleil s'est levé à l'est, baignant notre modeste hameau de St-Pierre-sous-Bois dans une lueur dorée. Les coqs du village ont entamé leur rituel chant matinal. Jean Dupont, laboureur de la famille Dupont, a quitté sa chaumière pour une nouvelle journée de labeur.\t12 avril 1365\tLa foire de Bourgogne approche à grands pas, et les préparatifs vont bon train. Mon ami Jacques LeForgeron, originaire de la lignée des LeForgeron, m'a demandé de l'aider à préparer des épées pour les chevaliers de la région. Le seigneur local, le comte de Lorraine, Antoine de la Roche, a annoncé qu'il tiendra une joute en l'honneur de son fils, Antoine de la Roche.\t15 avril 1365\tJacques LeForgeron et moi avons forgé des épées de qualité, avec une lame tranchante comme un rasoir. Le comte de la Roche a demandé un blason distinctif pour son fils, une épée ornée d'une fleur de lys. Mon fils, Louis Martin, descendant des Martin, m'a rejoint à l'atelier aujourd'hui. Il se passionne pour les manuscrits anciens et rêve de devenir copiste un jour.\t20 avril 1365\tLouis Martin est enthousiaste à l'idée de devenir copiste. Il est fasciné par les enluminures et les parchemins. J'ai promis de lui trouver un maître copiste dans la ville voisine, Nancy de Beauchamps, originaire de la famille Beauchamps, qui pourrait lui enseigner l'art de la calligraphie et de l'enluminure.\t25 avril 1365\tLa fête de la foire a débuté aujourd'hui. Le village est en effervescence, avec des jongleurs, des marchands ambulants et des troubadours venant de toute la France. Les rues sont ornées de bannières aux armes du comte de la Roche, et les odeurs de viandes rôties et de pain frais remplissent l'air.\t30 avril 1365\tLa joute s'est déroulée aujourd'hui, et le jeune chevalier Antoine de la Roche a brillamment remporté le tournoi. Son épée ornée de la fleur de lys était magnifique dans l'arène. Les festivités se sont terminées par un grand banquet au château du comte de la Roche, où les nobles et les villageois se sont réunis pour célébrer.\t5 mai 1365\tLouis Martin a quitté le village pour Nancy de Beauchamps, déterminé à poursuivre son rêve de devenir copiste. C'est un moment aigre-doux, de le voir partir, mais je suis fier de son ambition. La vie médiévale continue, rythmée par les saisons et les traditions immuables de notre petit coin de France.\n"
          ]
        }
      ],
      "source": [
        "# Chercher un texte généré sur ChatGPT sur Bertrandus qui se balade à Paris\n",
        "with open(\"/content/sample_data/Journl d'un Villageois Médiéval en .txt\", 'r', encoding='utf-8') as file:\n",
        "    Bertrandus = file.read()\n",
        "print(Bertrandus)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TKrvwuFUb547"
      },
      "source": [
        "# Workspace"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6vwnW7vkbhs1"
      },
      "outputs": [],
      "source": [
        "ner_jb = ner_init(test_Adam)\n",
        "ner2=ner_init(test_Adam2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "99N1UQ5jb5IV"
      },
      "outputs": [],
      "source": [
        "def sub_entities(ner, text):\n",
        "  # Recupere toutes les noms des entites nommés\n",
        "  entity_array = ner(text)\n",
        "  word_array = [item['word'] for item in entity_array]\n",
        "\n",
        "  # List des mots qui composent d'autres entités et qui ne sont pas déjà représenté\n",
        "  sub_word_array = []\n",
        "\n",
        "  # Pour chaque mot composé\n",
        "  for word in word_array:\n",
        "    if ' ' in word:\n",
        "\n",
        "      # On l'ajoute ses \"sous-mots\" si ils ne sont pas déjà dans word_array\n",
        "      for sub_word in word.split(' '):\n",
        "        if sub_word not in word_array:\n",
        "          sub_word_array.append(sub_word)\n",
        "\n",
        "  # Si la sous entites n'est pas déjà une entité nommée\n",
        "  for sub_word in sub_word_array:\n",
        "    sub_entity = ner(sub_word)\n",
        "\n",
        "    # sub_entity est une list donc extend à la place de append\n",
        "    entity_array.extend(sub_entity)\n",
        "\n",
        "\n",
        "  return entity_array"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import re"
      ],
      "metadata": {
        "id": "6meJx9fsud_g"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BVOLHPAGcM8L"
      },
      "source": [
        "# Test"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cO5CLy8Nds_p",
        "outputId": "9d94e038-3161-4bfa-ed91-0ec03ce40484"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[]\n",
            "[]\n",
            "[]\n",
            "[]\n",
            "[]\n",
            "[{'entity_group': 'LOC', 'score': 0.98820996, 'word': 'Louis Martin', 'start': 0, 'end': 12}]\n",
            "[]\n",
            "[]\n",
            "[]\n",
            "[{'entity_group': 'PER', 'score': 0.9939813, 'word': 'Jacques LeForgeron et moi', 'start': 0, 'end': 25}, {'entity_group': 'ACT', 'score': 0.98191005, 'word': 'forgé des épées de qualité', 'start': 31, 'end': 58}]\n",
            "[{'entity_group': 'DESC', 'score': 0.7915242, 'word': 'avec une lame tranchante comme un rasoir', 'start': 0, 'end': 40}]\n",
            "[{'entity_group': 'PER', 'score': 0.9504716, 'word': 'Le comte de la Roche', 'start': 0, 'end': 20}, {'entity_group': 'DESC', 'score': 0.6053759, 'word': 'demandé un blason distinctif', 'start': 22, 'end': 51}, {'entity_group': 'DESC', 'score': 0.58072966, 'word': 'fils', 'start': 60, 'end': 65}]\n",
            "[{'entity_group': 'PER', 'score': 0.9554817, 'word': 'une épée ornée', 'start': 0, 'end': 14}, {'entity_group': 'ACT', 'score': 0.8961279, 'word': \"d'une fleur de lys\", 'start': 14, 'end': 33}]\n",
            "[{'entity_group': 'PER', 'score': 0.9949763, 'word': 'Mon fils', 'start': 0, 'end': 8}]\n",
            "[{'entity_group': 'PER', 'score': 0.9941989, 'word': 'Louis Martin', 'start': 0, 'end': 12}]\n",
            "[{'entity_group': 'PER', 'score': 0.98502517, 'word': 'descendant des Martin', 'start': 0, 'end': 21}]\n",
            "[]\n",
            "[{'entity_group': 'DESC', 'score': 0.7290352, 'word': 'Il se passionne pour les manuscrits anciens et rêve de devenir copiste un jour', 'start': 0, 'end': 78}]\n"
          ]
        }
      ],
      "source": [
        "text=\"Jacques LeForgeron et moi avons forgé des épées de qualité, avec une lame tranchante comme un rasoir. Le comte de la Roche a demandé un blason distinctif pour son fils, une épée ornée d'une fleur de lys. Mon fils, Louis Martin, descendant des Martin, m'a rejoint à l'atelier aujourd'hui. Il se passionne pour les manuscrits anciens et rêve de devenir copiste un jour.\"\n",
        "\n",
        "entity2=re.split(r'[.,!?]',text)\n",
        "entity2=[sentence.strip() for sentence in entity2 if sentence.strip()]\n",
        "for sentence in entity2:\n",
        "  print(ner2(sentence))\n",
        "entity=re.split(r'[.,!?]',text)\n",
        "entity=[sentence.strip() for sentence in entity if sentence.strip()]\n",
        "for sentence in entity:\n",
        "  print(ner_jb(sentence))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MVAngfCwpS29",
        "outputId": "29b8d0fc-7c33-4510-c161-e865ba701ec4"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'entity_group': 'MISC',\n",
              "  'score': 0.77501416,\n",
              "  'word': \"Journl d'un Villageois Médiéval\",\n",
              "  'start': 0,\n",
              "  'end': 31},\n",
              " {'entity_group': 'LOC',\n",
              "  'score': 0.9258664,\n",
              "  'word': 'France',\n",
              "  'start': 34,\n",
              "  'end': 41},\n",
              " {'entity_group': 'DATE',\n",
              "  'score': 0.9888371,\n",
              "  'word': \"10 avril 1365 Aujourd'hui\",\n",
              "  'start': 42,\n",
              "  'end': 69},\n",
              " {'entity_group': 'DATE',\n",
              "  'score': 0.528513,\n",
              "  'word': 'le',\n",
              "  'start': 70,\n",
              "  'end': 73},\n",
              " {'entity_group': 'LOC',\n",
              "  'score': 0.9932431,\n",
              "  'word': 'St-Pierre-sous-Bois',\n",
              "  'start': 133,\n",
              "  'end': 153},\n",
              " {'entity_group': 'PER',\n",
              "  'score': 0.99018055,\n",
              "  'word': 'Jean Dupont',\n",
              "  'start': 233,\n",
              "  'end': 245},\n",
              " {'entity_group': 'PER',\n",
              "  'score': 0.9653243,\n",
              "  'word': 'Dupont',\n",
              "  'start': 270,\n",
              "  'end': 277},\n",
              " {'entity_group': 'DATE',\n",
              "  'score': 0.9942385,\n",
              "  'word': '12 avril 1365',\n",
              "  'start': 338,\n",
              "  'end': 352},\n",
              " {'entity_group': 'LOC',\n",
              "  'score': 0.59629416,\n",
              "  'word': 'Bourgogne',\n",
              "  'start': 365,\n",
              "  'end': 375},\n",
              " {'entity_group': 'PER',\n",
              "  'score': 0.99522066,\n",
              "  'word': 'Jacques LeForgeron',\n",
              "  'start': 441,\n",
              "  'end': 460},\n",
              " {'entity_group': 'PER',\n",
              "  'score': 0.9889892,\n",
              "  'word': 'LeForgeron',\n",
              "  'start': 489,\n",
              "  'end': 500},\n",
              " {'entity_group': 'PER',\n",
              "  'score': 0.43212146,\n",
              "  'word': 'comte de',\n",
              "  'start': 601,\n",
              "  'end': 610},\n",
              " {'entity_group': 'LOC',\n",
              "  'score': 0.6947874,\n",
              "  'word': 'Lorraine',\n",
              "  'start': 610,\n",
              "  'end': 619},\n",
              " {'entity_group': 'PER',\n",
              "  'score': 0.97760344,\n",
              "  'word': 'Antoine de la Roche',\n",
              "  'start': 620,\n",
              "  'end': 640},\n",
              " {'entity_group': 'PER',\n",
              "  'score': 0.9893917,\n",
              "  'word': 'Antoine de la Roche',\n",
              "  'start': 701,\n",
              "  'end': 721},\n",
              " {'entity_group': 'DATE',\n",
              "  'score': 0.9955787,\n",
              "  'word': '15 avril 1365',\n",
              "  'start': 723,\n",
              "  'end': 737},\n",
              " {'entity_group': 'PER',\n",
              "  'score': 0.99420035,\n",
              "  'word': 'Jacques LeForgeron',\n",
              "  'start': 738,\n",
              "  'end': 757},\n",
              " {'entity_group': 'PER',\n",
              "  'score': 0.8694964,\n",
              "  'word': 'comte de la Roche',\n",
              "  'start': 843,\n",
              "  'end': 861},\n",
              " {'entity_group': 'PER',\n",
              "  'score': 0.99292797,\n",
              "  'word': 'Louis Martin',\n",
              "  'start': 952,\n",
              "  'end': 965},\n",
              " {'entity_group': 'PER',\n",
              "  'score': 0.9798759,\n",
              "  'word': 'Martin',\n",
              "  'start': 981,\n",
              "  'end': 988},\n",
              " {'entity_group': 'DATE',\n",
              "  'score': 0.9898048,\n",
              "  'word': \"aujourd'hui\",\n",
              "  'start': 1013,\n",
              "  'end': 1025},\n",
              " {'entity_group': 'DATE',\n",
              "  'score': 0.9776493,\n",
              "  'word': 'un jour. 20 avril 1365',\n",
              "  'start': 1097,\n",
              "  'end': 1121},\n",
              " {'entity_group': 'PER',\n",
              "  'score': 0.99438095,\n",
              "  'word': 'Louis Martin',\n",
              "  'start': 1122,\n",
              "  'end': 1135},\n",
              " {'entity_group': 'PER',\n",
              "  'score': 0.98985434,\n",
              "  'word': 'Nancy de Beauchamps',\n",
              "  'start': 1303,\n",
              "  'end': 1323},\n",
              " {'entity_group': 'PER',\n",
              "  'score': 0.98192006,\n",
              "  'word': 'Beauchamps',\n",
              "  'start': 1349,\n",
              "  'end': 1360},\n",
              " {'entity_group': 'DATE',\n",
              "  'score': 0.9930569,\n",
              "  'word': '25 avril 1365',\n",
              "  'start': 1434,\n",
              "  'end': 1448},\n",
              " {'entity_group': 'DATE',\n",
              "  'score': 0.88107985,\n",
              "  'word': \"aujourd'hui.\",\n",
              "  'start': 1478,\n",
              "  'end': 1491},\n",
              " {'entity_group': 'LOC',\n",
              "  'score': 0.94813114,\n",
              "  'word': 'la France',\n",
              "  'start': 1603,\n",
              "  'end': 1613},\n",
              " {'entity_group': 'PER',\n",
              "  'score': 0.9008155,\n",
              "  'word': 'comte de la Roche',\n",
              "  'start': 1661,\n",
              "  'end': 1679},\n",
              " {'entity_group': 'DATE',\n",
              "  'score': 0.9936331,\n",
              "  'word': '30 avril 1365',\n",
              "  'start': 1749,\n",
              "  'end': 1763},\n",
              " {'entity_group': 'DATE',\n",
              "  'score': 0.9870848,\n",
              "  'word': \"aujourd'hui\",\n",
              "  'start': 1788,\n",
              "  'end': 1800},\n",
              " {'entity_group': 'DATE',\n",
              "  'score': 0.5857378,\n",
              "  'word': 'et',\n",
              "  'start': 1801,\n",
              "  'end': 1804},\n",
              " {'entity_group': 'PER',\n",
              "  'score': 0.98929465,\n",
              "  'word': 'Antoine de la Roche',\n",
              "  'start': 1823,\n",
              "  'end': 1843},\n",
              " {'entity_group': 'PER',\n",
              "  'score': 0.87077737,\n",
              "  'word': 'comte de la Roche',\n",
              "  'start': 2011,\n",
              "  'end': 2029},\n",
              " {'entity_group': 'DATE',\n",
              "  'score': 0.98296744,\n",
              "  'word': '5 mai 13',\n",
              "  'start': 2093,\n",
              "  'end': 2102},\n",
              " {'entity_group': 'PER',\n",
              "  'score': 0.7970362,\n",
              "  'word': 'Journl',\n",
              "  'start': 0,\n",
              "  'end': 6},\n",
              " {'entity_group': 'LOC',\n",
              "  'score': 0.96030605,\n",
              "  'word': 'Villageois',\n",
              "  'start': 0,\n",
              "  'end': 10},\n",
              " {'entity_group': 'MISC',\n",
              "  'score': 0.9495097,\n",
              "  'word': 'Médiéval',\n",
              "  'start': 0,\n",
              "  'end': 8},\n",
              " {'entity_group': 'DATE',\n",
              "  'score': 0.99254733,\n",
              "  'word': 'avril',\n",
              "  'start': 0,\n",
              "  'end': 5},\n",
              " {'entity_group': 'DATE',\n",
              "  'score': 0.99248964,\n",
              "  'word': '1365',\n",
              "  'start': 0,\n",
              "  'end': 4},\n",
              " {'entity_group': 'DATE',\n",
              "  'score': 0.9886775,\n",
              "  'word': \"Aujourd'hui\",\n",
              "  'start': 0,\n",
              "  'end': 11},\n",
              " {'entity_group': 'PER',\n",
              "  'score': 0.6992477,\n",
              "  'word': 'Jean',\n",
              "  'start': 0,\n",
              "  'end': 4},\n",
              " {'entity_group': 'DATE',\n",
              "  'score': 0.99254733,\n",
              "  'word': 'avril',\n",
              "  'start': 0,\n",
              "  'end': 5},\n",
              " {'entity_group': 'DATE',\n",
              "  'score': 0.99248964,\n",
              "  'word': '1365',\n",
              "  'start': 0,\n",
              "  'end': 4},\n",
              " {'entity_group': 'PER',\n",
              "  'score': 0.9719006,\n",
              "  'word': 'Jacques',\n",
              "  'start': 0,\n",
              "  'end': 7},\n",
              " {'entity_group': 'PER',\n",
              "  'score': 0.5595349,\n",
              "  'word': 'comte',\n",
              "  'start': 0,\n",
              "  'end': 5},\n",
              " {'entity_group': 'PER',\n",
              "  'score': 0.9824003,\n",
              "  'word': 'Antoine',\n",
              "  'start': 0,\n",
              "  'end': 7},\n",
              " {'entity_group': 'LOC',\n",
              "  'score': 0.44760227,\n",
              "  'word': 'Roche',\n",
              "  'start': 0,\n",
              "  'end': 5},\n",
              " {'entity_group': 'PER',\n",
              "  'score': 0.9824003,\n",
              "  'word': 'Antoine',\n",
              "  'start': 0,\n",
              "  'end': 7},\n",
              " {'entity_group': 'LOC',\n",
              "  'score': 0.44760227,\n",
              "  'word': 'Roche',\n",
              "  'start': 0,\n",
              "  'end': 5},\n",
              " {'entity_group': 'DATE',\n",
              "  'score': 0.99254733,\n",
              "  'word': 'avril',\n",
              "  'start': 0,\n",
              "  'end': 5},\n",
              " {'entity_group': 'DATE',\n",
              "  'score': 0.99248964,\n",
              "  'word': '1365',\n",
              "  'start': 0,\n",
              "  'end': 4},\n",
              " {'entity_group': 'PER',\n",
              "  'score': 0.9719006,\n",
              "  'word': 'Jacques',\n",
              "  'start': 0,\n",
              "  'end': 7},\n",
              " {'entity_group': 'PER',\n",
              "  'score': 0.5595349,\n",
              "  'word': 'comte',\n",
              "  'start': 0,\n",
              "  'end': 5},\n",
              " {'entity_group': 'LOC',\n",
              "  'score': 0.44760227,\n",
              "  'word': 'Roche',\n",
              "  'start': 0,\n",
              "  'end': 5},\n",
              " {'entity_group': 'PER',\n",
              "  'score': 0.96074724,\n",
              "  'word': 'Louis',\n",
              "  'start': 0,\n",
              "  'end': 5},\n",
              " {'entity_group': 'DATE',\n",
              "  'score': 0.99254733,\n",
              "  'word': 'avril',\n",
              "  'start': 0,\n",
              "  'end': 5},\n",
              " {'entity_group': 'DATE',\n",
              "  'score': 0.99248964,\n",
              "  'word': '1365',\n",
              "  'start': 0,\n",
              "  'end': 4},\n",
              " {'entity_group': 'PER',\n",
              "  'score': 0.96074724,\n",
              "  'word': 'Louis',\n",
              "  'start': 0,\n",
              "  'end': 5},\n",
              " {'entity_group': 'LOC',\n",
              "  'score': 0.97317356,\n",
              "  'word': 'Nancy',\n",
              "  'start': 0,\n",
              "  'end': 5},\n",
              " {'entity_group': 'DATE',\n",
              "  'score': 0.99254733,\n",
              "  'word': 'avril',\n",
              "  'start': 0,\n",
              "  'end': 5},\n",
              " {'entity_group': 'DATE',\n",
              "  'score': 0.99248964,\n",
              "  'word': '1365',\n",
              "  'start': 0,\n",
              "  'end': 4},\n",
              " {'entity_group': 'PER',\n",
              "  'score': 0.5595349,\n",
              "  'word': 'comte',\n",
              "  'start': 0,\n",
              "  'end': 5},\n",
              " {'entity_group': 'LOC',\n",
              "  'score': 0.44760227,\n",
              "  'word': 'Roche',\n",
              "  'start': 0,\n",
              "  'end': 5},\n",
              " {'entity_group': 'DATE',\n",
              "  'score': 0.99254733,\n",
              "  'word': 'avril',\n",
              "  'start': 0,\n",
              "  'end': 5},\n",
              " {'entity_group': 'DATE',\n",
              "  'score': 0.99248964,\n",
              "  'word': '1365',\n",
              "  'start': 0,\n",
              "  'end': 4},\n",
              " {'entity_group': 'PER',\n",
              "  'score': 0.9824003,\n",
              "  'word': 'Antoine',\n",
              "  'start': 0,\n",
              "  'end': 7},\n",
              " {'entity_group': 'LOC',\n",
              "  'score': 0.44760227,\n",
              "  'word': 'Roche',\n",
              "  'start': 0,\n",
              "  'end': 5},\n",
              " {'entity_group': 'PER',\n",
              "  'score': 0.5595349,\n",
              "  'word': 'comte',\n",
              "  'start': 0,\n",
              "  'end': 5},\n",
              " {'entity_group': 'LOC',\n",
              "  'score': 0.44760227,\n",
              "  'word': 'Roche',\n",
              "  'start': 0,\n",
              "  'end': 5},\n",
              " {'entity_group': 'DATE',\n",
              "  'score': 0.99069464,\n",
              "  'word': 'mai',\n",
              "  'start': 0,\n",
              "  'end': 3}]"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ],
      "source": [
        "sub_entities(ner_jb, Bertrandus)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.10"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}