helsinki_translator.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "private_outputs": true,
      "provenance": [],
      "gpuType": "T4",
      "authorship_tag": "ABX9TyPzs+YxcOtH5XLieZGX2MJU"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "Questo notebook per Colaboratory di Google mostra come utilizzando i modelli di **Helsinki-NLP** per tradurre del testo"
      ],
      "metadata": {
        "id": "Xheftca1xjav"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "u2ZdIfRUnH8u"
      },
      "outputs": [],
      "source": [
        "!pip install transformers sentencepiece sacremoses fasttext\n",
        "\n",
        "import os\n",
        "if not os.path.isfile(\"/content/lid218e.bin\"):\n",
        "  !wget https://dl.fbaipublicfiles.com/nllb/lid/lid218e.bin"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Iniziamo inserendo il testo da tradurre"
      ],
      "metadata": {
        "id": "aWybDQMnE8zg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "testo_src = input(\"Inserisci il testo da tradurre: \")"
      ],
      "metadata": {
        "id": "xU-7pnYsJcuz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Infine inseriamo una o più parole nella lingua di testo per la traduzione"
      ],
      "metadata": {
        "id": "cT2Lanj7Jsbr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "testo_dst = input(\"Inserisci una o più parole nella lingua per la traduziobne, es. name: \")"
      ],
      "metadata": {
        "id": "86kDwT-oNDTZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Cominciamo a rilevare le lingue usate per la traduzione per il nome del modello da usare"
      ],
      "metadata": {
        "id": "uMzAVa__NTyd"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import fasttext\n",
        "\n",
        "plm = \"/content/lid218e.bin\"\n",
        "model = fasttext.load_model(plm)\n",
        "\n",
        "p = model.predict(testo_src, k=-1, threshold=0.5)\n",
        "input_lang = p[0][0].replace('__label__', '')\n",
        "print(\"Lingua sorgente: \" + input_lang)\n",
        "\n",
        "p = model.predict(testo_dst, k=-1, threshold=0.5)\n",
        "output_lang = p[0][0].replace('__label__', '')\n",
        "print(\"Lingua destinaizone: \" + output_lang)\n",
        "\n",
        "src = input_lang[:2]\n",
        "trg = output_lang[:2]\n",
        "model_name = f'Helsinki-NLP/opus-mt-{src}-{trg}'"
      ],
      "metadata": {
        "id": "wcJGYpQFNg3O"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Individua la presenza di una GPU"
      ],
      "metadata": {
        "id": "xzyEf6sHo6ZS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "\n",
        "if not torch.cuda.is_available():\n",
        "  print('WARNING: You may want to change the runtime to GPU for faster training!')\n",
        "  device = 'cpu'\n",
        "else:\n",
        "  device = 'cuda:0'"
      ],
      "metadata": {
        "id": "vXTFLxRoo9Ao"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Infine traduciamo il testo nella lingua desiderata"
      ],
      "metadata": {
        "id": "msuSqzWnP1LF"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Esempio n°1"
      ],
      "metadata": {
        "id": "AQ2me_UTpMhg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import MarianTokenizer, MarianMTModel\n",
        "from typing import List\n",
        "\n",
        "tokenizer = MarianTokenizer.from_pretrained(model_name)\n",
        "model = MarianMTModel.from_pretrained(model_name).to(device)\n",
        "batch = tokenizer([testo_src], return_tensors=\"pt\", padding=True).to(device)\n",
        "translated = model.generate(**batch)\n",
        "\n",
        "for t in translated:\n",
        "    print(tokenizer.decode(t, skip_special_tokens=True))"
      ],
      "metadata": {
        "id": "mWC8kvfSToXC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Esempio n°2"
      ],
      "metadata": {
        "id": "uh8dk8WPpPfa"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import pipeline\n",
        "\n",
        "pipe = pipeline(\"translation\", model=model_name,device=device)\n",
        "print(pipe(testo_src))"
      ],
      "metadata": {
        "id": "gL5BY4yumyR3"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}