tutorials/blogposts/5.TrainingContextSpellChecker.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<H1> Noisy Channel Model Spell Checker - Training </H1>\n",
    "In this notebook we're going to learn how to train the Noisy Channel Model Spell Checker, a.k.a. ContextSpellChecker, as it can leverage context word information to produce corrections for each word."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Italian Language Spell Checking\n",
    "This is a toy Italian Spell Checking Model used here to exemplify how to train a Spell Checker. It may require more work to become a real world model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.7.5'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sparknlp.annotator import *\n",
    "from sparknlp.common import *\n",
    "from sparknlp.base import *\n",
    "import sparknlp\n",
    "from pyspark.sql import SparkSession\n",
    "\n",
    "spark = sparknlp.start(gpu=True)\n",
    "sparknlp.version()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+\n",
      "|               value|\n",
      "+--------------------+\n",
      "|Davide Guglielmin...|\n",
      "|Avete partecipato...|\n",
      "+--------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Let's use the Paisa corpus\n",
    "# https://clarin.eurac.edu/repository/xmlui/bitstream/handle/20.500.12124/3/paisa.raw.utf8.gz\n",
    "from pyspark.sql.functions import *\n",
    "paisaCorpusPath = \"/home/jose/Downloads/paisa/paisa.raw.utf8\"\n",
    "\n",
    "\n",
    "# do some brief DS exploration, and preparation to get clean text\n",
    "df = spark.read.text(paisaCorpusPath)\n",
    "df = df.filter(~col('value').contains('</text')).\\\n",
    "          filter(~col('value').contains('<text')).\\\n",
    "          filter(~col('value').startswith('#'))#.\\\n",
    "          #limit(10000)\n",
    "df.limit(2).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Abaco',\n",
       " 'Abbondanza',\n",
       " 'Abbondanzia',\n",
       " 'Abbondanzio',\n",
       " 'Abbondazio',\n",
       " 'Abbondia',\n",
       " 'Abbondina',\n",
       " 'Abbondio',\n",
       " 'Abdelkrim',\n",
       " 'Abdellah',\n",
       " 'Abdenago',\n",
       " 'Abdon',\n",
       " 'Abdone',\n",
       " 'Abela',\n",
       " 'Abelarda',\n",
       " 'Abelardo',\n",
       " 'Abele',\n",
       " 'Abelina',\n",
       " 'Abelino',\n",
       " 'Aberardo',\n",
       " 'Abilio',\n",
       " 'Abondio',\n",
       " 'Abrama',\n",
       " 'Abramina',\n",
       " 'Abramino',\n",
       " 'Abramo',\n",
       " 'Accorso',\n",
       " 'Accursa',\n",
       " 'Accursia',\n",
       " 'Accursio',\n",
       " 'Accurso',\n",
       " 'Acheropita',\n",
       " 'Achilla',\n",
       " 'Achille',\n",
       " 'Achillea',\n",
       " 'Achilleo',\n",
       " 'Achillina',\n",
       " 'Achiropita',\n",
       " 'Acilia',\n",
       " 'Acilio',\n",
       " 'Acquisto',\n",
       " 'Acrisio',\n",
       " 'Ada',\n",
       " 'Adalberta',\n",
       " 'Adalberto',\n",
       " 'Adalciso',\n",
       " 'Adalgerio',\n",
       " 'Adalgisa',\n",
       " 'Adalgisio',\n",
       " 'Adalgiso',\n",
       " 'Adalia',\n",
       " 'Adalinda',\n",
       " 'Adalindo',\n",
       " 'Adalio',\n",
       " 'Adalisa',\n",
       " 'Adama',\n",
       " 'Adamaria',\n",
       " 'Adamello',\n",
       " 'Adamina',\n",
       " 'Adamino',\n",
       " 'Adamo',\n",
       " 'Adastro',\n",
       " 'Addamiano',\n",
       " 'Addario',\n",
       " 'Addiego',\n",
       " 'Addolorata',\n",
       " 'Addolorato',\n",
       " 'Addonizio',\n",
       " 'Adea',\n",
       " 'Adela',\n",
       " 'Adelaida',\n",
       " 'Adelaide',\n",
       " 'Adelaido',\n",
       " 'Adelasia',\n",
       " 'Adelasio',\n",
       " 'Adelca',\n",
       " 'Adelchi',\n",
       " 'Adelchio',\n",
       " 'Adelcisa',\n",
       " 'Adelco',\n",
       " 'Adele',\n",
       " 'Adelelma',\n",
       " 'Adelelmo',\n",
       " 'Adelfa',\n",
       " 'Adelfina',\n",
       " 'Adelfino',\n",
       " 'Adelfio',\n",
       " 'Adelfo',\n",
       " 'Adelia',\n",
       " 'Adelina',\n",
       " 'Adelinda',\n",
       " 'Adelindo',\n",
       " 'Adelino',\n",
       " 'Adelio',\n",
       " 'Adelisa',\n",
       " 'Adelisia',\n",
       " 'Adelisio',\n",
       " 'Adelita',\n",
       " 'Adelma',\n",
       " 'Adelmaro',\n",
       " 'Adelmina',\n",
       " 'Adelmino',\n",
       " 'Adelmira',\n",
       " 'Adelmiro',\n",
       " 'Adelmo',\n",
       " 'Adelo',\n",
       " 'Adema',\n",
       " 'Ademara',\n",
       " 'Ademaro',\n",
       " 'Ademia',\n",
       " 'Ademina',\n",
       " 'Ademio',\n",
       " 'Ademo',\n",
       " 'Aden',\n",
       " 'Adeo',\n",
       " 'Adeodata',\n",
       " 'Adeodato',\n",
       " 'Aderita',\n",
       " 'Aderito',\n",
       " 'Adermo',\n",
       " 'Adia',\n",
       " 'Adige',\n",
       " 'Adigina',\n",
       " 'Adigrat',\n",
       " 'Adilia',\n",
       " 'Adilio',\n",
       " 'Adimaro',\n",
       " 'Adimiro',\n",
       " 'Adina',\n",
       " 'Adino',\n",
       " 'Adinolfa',\n",
       " 'Adinolfo',\n",
       " 'Adio',\n",
       " 'Adionilla',\n",
       " 'Adiuto',\n",
       " 'Adiutore',\n",
       " 'Admeto',\n",
       " 'Ado',\n",
       " 'Adolfa',\n",
       " 'Adolfina',\n",
       " 'Adolfino',\n",
       " 'Adolfo',\n",
       " 'Adolfo pietro',\n",
       " 'Adona',\n",
       " 'Adone',\n",
       " 'Adonella',\n",
       " 'Adonello',\n",
       " 'Adoranda',\n",
       " 'Adorando',\n",
       " 'Adorata',\n",
       " 'Adorato',\n",
       " 'Adorna',\n",
       " 'Adornino',\n",
       " 'Adorno',\n",
       " 'Adrasta',\n",
       " 'Adrasto',\n",
       " 'Adreano',\n",
       " 'Adria',\n",
       " 'Adrian andrei petre',\n",
       " 'Adriana',\n",
       " 'Adriano',\n",
       " 'Adrienn',\n",
       " 'Adrio',\n",
       " 'Adua',\n",
       " 'Aduana',\n",
       " 'Aduilio',\n",
       " 'Aduino',\n",
       " 'Aduo',\n",
       " 'Affortunata',\n",
       " 'Affortunato',\n",
       " 'Afra',\n",
       " 'Africa',\n",
       " 'Africana',\n",
       " 'Africano',\n",
       " 'Africo',\n",
       " 'Afro',\n",
       " 'Afrodisio',\n",
       " 'Afrodite',\n",
       " 'Agabio',\n",
       " 'Agabito',\n",
       " 'Agamennone',\n",
       " 'Agape',\n",
       " 'Agapio',\n",
       " 'Agapito',\n",
       " 'Agata',\n",
       " 'Agatella',\n",
       " 'Agatina',\n",
       " 'Agatino',\n",
       " 'Agato',\n",
       " 'Agatocle',\n",
       " 'Agatoclia',\n",
       " 'Agatone',\n",
       " 'Agazia',\n",
       " 'Agazio',\n",
       " 'Agea',\n",
       " 'Agenore',\n",
       " 'Ageo',\n",
       " 'Agesilao',\n",
       " 'Aggeo',\n",
       " 'Agide',\n",
       " 'Agilulfo',\n",
       " 'Aglaia',\n",
       " 'Agnella',\n",
       " 'Agnello',\n",
       " 'Agnese',\n",
       " 'Agnesina',\n",
       " 'Agnesio',\n",
       " 'Agnieszka ewa',\n",
       " 'Agnieszka irena',\n",
       " 'Agostina',\n",
       " 'Agostino',\n",
       " 'Agricola',\n",
       " 'Agricolo',\n",
       " 'Agrippina',\n",
       " 'Agrippino',\n",
       " 'Agrisio',\n",
       " 'Aguinaldo',\n",
       " 'Ahmed',\n",
       " 'Aiace',\n",
       " 'Aica',\n",
       " 'Aicardo',\n",
       " 'Aida',\n",
       " 'Aide',\n",
       " 'Aido',\n",
       " 'Aimo',\n",
       " 'Aimone',\n",
       " 'Airaldo',\n",
       " 'Aishe',\n",
       " 'Aiuto',\n",
       " 'Ala',\n",
       " 'Aladina',\n",
       " 'Aladino',\n",
       " 'Alaghiera',\n",
       " 'Alaghiero',\n",
       " 'Alam',\n",
       " 'Alamanno',\n",
       " 'Alamiro',\n",
       " 'Alan',\n",
       " 'Alarico',\n",
       " 'Alba',\n",
       " 'Albana',\n",
       " 'Albania',\n",
       " 'Albanino',\n",
       " 'Albano',\n",
       " 'Albarosa',\n",
       " 'Alberica',\n",
       " 'Alberice',\n",
       " 'Alberico',\n",
       " 'Alberigo',\n",
       " 'Alberino',\n",
       " 'Alberta',\n",
       " 'Albertina',\n",
       " 'Albertino',\n",
       " 'Alberto',\n",
       " 'Albina',\n",
       " 'Albino',\n",
       " 'Albio',\n",
       " 'Albionte',\n",
       " 'Albizio',\n",
       " 'Albizzo',\n",
       " 'Albo',\n",
       " 'Alboina',\n",
       " 'Alboino',\n",
       " 'Alcea',\n",
       " 'Alceo',\n",
       " 'Alcesta',\n",
       " 'Alceste',\n",
       " 'Alcestina',\n",
       " 'Alcesto',\n",
       " 'Alcibiade',\n",
       " 'Alcida',\n",
       " 'Alcide',\n",
       " 'Alcidio',\n",
       " 'Alcido',\n",
       " 'Alcina',\n",
       " 'Alcino',\n",
       " 'Alcisa',\n",
       " 'Alciso',\n",
       " 'Alcmena',\n",
       " 'Alda',\n",
       " 'Aldeandra',\n",
       " 'Aldebrando',\n",
       " 'Aldegarda',\n",
       " 'Aldegardo',\n",
       " 'Aldegonda',\n",
       " 'Aldegondo',\n",
       " 'Aldemara',\n",
       " 'Aldemaro',\n",
       " 'Aldemia',\n",
       " 'Aldemina',\n",
       " 'Aldemio',\n",
       " 'Aldemira',\n",
       " 'Aldemiro',\n",
       " 'Aldemo',\n",
       " 'Alderano',\n",
       " 'Alderica',\n",
       " 'Alderice',\n",
       " 'Alderico',\n",
       " 'Alderigi',\n",
       " 'Alderigo',\n",
       " 'Alderina',\n",
       " 'Alderino',\n",
       " 'Alderio',\n",
       " 'Aldero',\n",
       " 'Aldesina',\n",
       " 'Aldesino',\n",
       " 'Aldesira',\n",
       " 'Aldezza',\n",
       " 'Aldighiero',\n",
       " 'Aldimaro',\n",
       " 'Aldimiro',\n",
       " 'Aldina',\n",
       " 'Aldino',\n",
       " 'Aldisio',\n",
       " 'Aldiviero',\n",
       " 'Aldo',\n",
       " 'Aldobranda',\n",
       " 'Aldobrandino',\n",
       " 'Aldobrando',\n",
       " 'Aldoino',\n",
       " 'Aldomiro',\n",
       " 'Aldorino',\n",
       " 'Aldovina',\n",
       " 'Aldovino',\n",
       " 'Alduccio',\n",
       " 'Alduina',\n",
       " 'Alduino',\n",
       " 'Alealdo',\n",
       " 'Aleandra',\n",
       " 'Aleandro',\n",
       " 'Alearco',\n",
       " 'Alearda',\n",
       " 'Aleardo',\n",
       " 'Alemanno',\n",
       " 'Alemaro',\n",
       " 'Aleramo',\n",
       " 'Alerano',\n",
       " 'Alerino',\n",
       " 'Alesia',\n",
       " 'Alesio',\n",
       " 'Alessandra',\n",
       " 'Alessandria',\n",
       " 'Alessandrina',\n",
       " 'Alessandrino',\n",
       " 'Alessandro',\n",
       " 'Alessi',\n",
       " 'Alessia',\n",
       " 'Alessina',\n",
       " 'Alessio',\n",
       " 'Aletta',\n",
       " 'Alexandra',\n",
       " 'Alexandre',\n",
       " 'Alfa',\n",
       " 'Alfano',\n",
       " 'Alfea',\n",
       " 'Alfeo',\n",
       " 'Alferia',\n",
       " 'Alferino',\n",
       " 'Alferio',\n",
       " 'Alfero',\n",
       " 'Alfia',\n",
       " 'Alfie navar',\n",
       " 'Alfiera',\n",
       " 'Alfiere',\n",
       " 'Alfieri',\n",
       " 'Alfierina',\n",
       " 'Alfierino',\n",
       " 'Alfiero',\n",
       " 'Alfina',\n",
       " 'Alfino',\n",
       " 'Alfio',\n",
       " 'Alfo',\n",
       " 'Alfonsa',\n",
       " 'Alfonsina',\n",
       " 'Alfonsino',\n",
       " 'Alfonso',\n",
       " 'Alfonza',\n",
       " 'Alfonzina',\n",
       " 'Alfonzo',\n",
       " 'Alfreda',\n",
       " 'Alfredina',\n",
       " 'Alfredino',\n",
       " 'Alfredo',\n",
       " 'Alfrida',\n",
       " 'Alfride',\n",
       " 'Algemiro',\n",
       " 'Algeo',\n",
       " 'Algeri',\n",
       " 'Algeria',\n",
       " 'Algerina',\n",
       " 'Algerino',\n",
       " 'Algero',\n",
       " 'Alghero',\n",
       " 'Alghisio',\n",
       " 'Algia',\n",
       " 'Algimiro',\n",
       " 'Algisa',\n",
       " 'Algisio',\n",
       " 'Algiso',\n",
       " 'Aliberto',\n",
       " 'Alibrando',\n",
       " 'Alice',\n",
       " 'Alicia',\n",
       " 'Alicio',\n",
       " 'Alida',\n",
       " 'Alide',\n",
       " 'Alidea',\n",
       " 'Alideo',\n",
       " 'Alido',\n",
       " 'Alidora',\n",
       " 'Alidoro',\n",
       " 'Alighiera',\n",
       " 'Alighieri',\n",
       " 'Alighiero',\n",
       " 'Aligi',\n",
       " 'Aligio',\n",
       " 'Alimo',\n",
       " 'Alina',\n",
       " 'Alinda',\n",
       " 'Alindo',\n",
       " 'Alino',\n",
       " 'Alipia',\n",
       " 'Alipio',\n",
       " 'Aliprando',\n",
       " 'Alisa',\n",
       " 'Alisia',\n",
       " 'Alisio',\n",
       " 'Aliso',\n",
       " 'Aliviero',\n",
       " 'Allegra',\n",
       " 'Allegrina',\n",
       " 'Allegrino',\n",
       " 'Allegro',\n",
       " 'Alma',\n",
       " 'Almarosa',\n",
       " 'Almeria',\n",
       " 'Almerica',\n",
       " 'Almerico',\n",
       " 'Almerigo',\n",
       " 'Almerina',\n",
       " 'Almerinda',\n",
       " 'Almerindo',\n",
       " 'Almerino',\n",
       " 'Almerio',\n",
       " 'Almeris',\n",
       " 'Almiero',\n",
       " 'Almina',\n",
       " 'Almino',\n",
       " 'Almira',\n",
       " 'Almirante',\n",
       " 'Almiro',\n",
       " 'Almo',\n",
       " 'Alodia',\n",
       " 'Aloisa',\n",
       " 'Aloise',\n",
       " 'Aloisio',\n",
       " 'Alonzo',\n",
       " 'Alpa',\n",
       " 'Alpidio',\n",
       " 'Alpiniano',\n",
       " 'Alpino',\n",
       " 'Alpinolo',\n",
       " 'Alpo',\n",
       " 'Altabella',\n",
       " 'Altavilla',\n",
       " 'Altavillo',\n",
       " 'Altea',\n",
       " 'Altemio',\n",
       " 'Alteo',\n",
       " 'Altera',\n",
       " 'Alteria',\n",
       " 'Alterino',\n",
       " 'Alterio',\n",
       " 'Altero',\n",
       " 'Altieri',\n",
       " 'Altiero',\n",
       " 'Altimiro',\n",
       " 'Altobello',\n",
       " 'Altobrando',\n",
       " 'Altomare',\n",
       " 'Altomira',\n",
       " 'Altomiro',\n",
       " 'Altorino',\n",
       " 'Alvara',\n",
       " 'Alvarina',\n",
       " 'Alvarino',\n",
       " 'Alvario',\n",
       " 'Alvaro',\n",
       " 'Alvezio',\n",
       " 'Alvisa',\n",
       " 'Alvise',\n",
       " 'Alvisia',\n",
       " 'Alvisio',\n",
       " 'Alviso',\n",
       " 'Amabile',\n",
       " 'Amabilia',\n",
       " 'Amabilio',\n",
       " 'Amaddio',\n",
       " 'Amadea',\n",
       " 'Amadeo',\n",
       " 'Amadia',\n",
       " 'Amadio',\n",
       " 'Amalfa',\n",
       " 'Amalfia',\n",
       " 'Amalfio',\n",
       " 'Amalfo',\n",
       " 'Amalia',\n",
       " 'Amalio',\n",
       " 'Amanda',\n",
       " 'Amandina',\n",
       " 'Amandino',\n",
       " 'Amando',\n",
       " 'Amante',\n",
       " 'Amantina',\n",
       " 'Amantino',\n",
       " 'Amanto',\n",
       " 'Amanzia',\n",
       " 'Amanzio',\n",
       " 'Amarando',\n",
       " 'Amaranta',\n",
       " 'Amaranto',\n",
       " 'Amarilda',\n",
       " 'Amarilli',\n",
       " 'Amarillide',\n",
       " 'Amario',\n",
       " 'Amata',\n",
       " 'Amatino',\n",
       " 'Amato',\n",
       " 'Amatore',\n",
       " 'Amatrice',\n",
       " 'Amatuccio',\n",
       " 'Amberta',\n",
       " 'Amberto',\n",
       " 'Ambra',\n",
       " 'Ambretta',\n",
       " 'Ambro',\n",
       " 'Ambrogia',\n",
       " 'Ambrogina',\n",
       " 'Ambrogino',\n",
       " 'Ambrogio',\n",
       " 'Ambrosia',\n",
       " 'Ambrosina',\n",
       " 'Ambrosino',\n",
       " 'Ambrosio',\n",
       " 'Amedea',\n",
       " 'Amedeo',\n",
       " 'Ameglia',\n",
       " 'Ameglio',\n",
       " 'Amelia',\n",
       " 'Amelina',\n",
       " 'Amelino',\n",
       " 'Amelio',\n",
       " 'Amelita',\n",
       " 'America',\n",
       " 'Americo',\n",
       " 'Ameriga',\n",
       " 'Amerigo',\n",
       " 'Amerina',\n",
       " 'Amerinda',\n",
       " 'Amerindo',\n",
       " 'Amerino',\n",
       " 'Amerio',\n",
       " 'Amero',\n",
       " 'Amica',\n",
       " 'Amico',\n",
       " 'Amilcara',\n",
       " 'Amilcare',\n",
       " 'Amina',\n",
       " 'Aminda',\n",
       " 'Amino',\n",
       " 'Aminta',\n",
       " 'Aminto',\n",
       " 'Amintore',\n",
       " 'Amleta',\n",
       " 'Amleto',\n",
       " 'Amodeo',\n",
       " 'Amodia',\n",
       " 'Amodio',\n",
       " 'Amonasro',\n",
       " 'Amonastro',\n",
       " 'Amore',\n",
       " 'Amorina',\n",
       " 'Amorino',\n",
       " 'Amorosa',\n",
       " 'Amoroso',\n",
       " 'Amos',\n",
       " 'Ampeglio',\n",
       " 'Ampelia',\n",
       " 'Ampelio',\n",
       " 'Ampellio',\n",
       " 'Amperio',\n",
       " 'Ampilio',\n",
       " 'Amsicora',\n",
       " 'Amulia',\n",
       " 'Amulio',\n",
       " 'Ana paula',\n",
       " 'Ana rita',\n",
       " 'Anabel',\n",
       " 'Anacleta',\n",
       " 'Anacleto',\n",
       " 'Anania',\n",
       " 'Ananio',\n",
       " 'Anastasia',\n",
       " 'Anastasio',\n",
       " 'Anatolia',\n",
       " 'Anatolio',\n",
       " 'Anca',\n",
       " 'Ancella',\n",
       " 'Anchise',\n",
       " 'Ancilla',\n",
       " 'Ancillo',\n",
       " 'Anco',\n",
       " 'Ancomarzio',\n",
       " 'Anda',\n",
       " 'Andalusa',\n",
       " 'Andaluso',\n",
       " 'Andina',\n",
       " 'Andino',\n",
       " 'Ando',\n",
       " 'Andra',\n",
       " 'Andrea',\n",
       " 'Andreana',\n",
       " 'Andreano',\n",
       " 'Andreatta',\n",
       " 'Andreea daniela',\n",
       " 'Andreina',\n",
       " 'Andreino',\n",
       " 'Andreola',\n",
       " 'Andreolo',\n",
       " 'Andres gregorio',\n",
       " 'Andretta',\n",
       " 'Andrettina',\n",
       " 'Andreuccia',\n",
       " 'Andreuccio',\n",
       " 'Andrietta',\n",
       " 'Andrietto',\n",
       " 'Andro',\n",
       " 'Andromaca',\n",
       " 'Andronico',\n",
       " 'Anella',\n",
       " 'Anellina',\n",
       " 'Anellino',\n",
       " 'Anelusco',\n",
       " 'Angela',\n",
       " 'Angelamaria',\n",
       " 'Angelantonio',\n",
       " 'Angelarosa',\n",
       " 'Angeletta',\n",
       " 'Angelica',\n",
       " 'Angelico',\n",
       " 'Angelina',\n",
       " 'Angelino',\n",
       " 'Angelita',\n",
       " 'Angelo',\n",
       " 'Angelomaria',\n",
       " 'Angilberto',\n",
       " 'Angiola',\n",
       " 'Angiolamaria',\n",
       " 'Angioletta',\n",
       " 'Angioletto',\n",
       " 'Angiolina',\n",
       " 'Angiolino',\n",
       " 'Angiolo',\n",
       " 'Aniceta',\n",
       " 'Aniceto',\n",
       " 'Anicetta',\n",
       " 'Aniella',\n",
       " 'Aniello',\n",
       " 'Anita',\n",
       " 'Anito',\n",
       " 'Anna',\n",
       " 'Anna maria',\n",
       " 'Anna rosa',\n",
       " 'Annabella',\n",
       " 'Annachiara',\n",
       " 'Annaflavia',\n",
       " 'Annagrazia',\n",
       " 'Annalaura',\n",
       " 'Annalia',\n",
       " 'Annalisa',\n",
       " 'Annaluisa',\n",
       " 'Annamaria',\n",
       " 'Annarella',\n",
       " 'Annarita',\n",
       " 'Annarosa',\n",
       " 'Anne marie luce',\n",
       " 'Annella',\n",
       " 'Annetta',\n",
       " 'Annette',\n",
       " 'Annettina',\n",
       " 'Annetto',\n",
       " 'Annibala',\n",
       " 'Annibale',\n",
       " 'Annibalina',\n",
       " 'Annico',\n",
       " 'Annina',\n",
       " 'Annino',\n",
       " 'Annio',\n",
       " 'Annita',\n",
       " 'Annito',\n",
       " 'Anno',\n",
       " 'Annuccia',\n",
       " 'Annuccio',\n",
       " 'Annuncia',\n",
       " 'Annunciata',\n",
       " 'Annunciato',\n",
       " 'Annuncio',\n",
       " 'Annunzia',\n",
       " 'Annunziata',\n",
       " 'Annunziatina',\n",
       " 'Annunziato',\n",
       " 'Annunzio',\n",
       " 'Anrico',\n",
       " 'Ansalda',\n",
       " 'Ansaldo',\n",
       " 'Ansano',\n",
       " 'Anselma',\n",
       " 'Anselmina',\n",
       " 'Anselmino',\n",
       " 'Anselmo',\n",
       " 'Ansovino',\n",
       " 'Anspertina',\n",
       " 'Ansperto',\n",
       " 'Ansuina',\n",
       " 'Ansuino',\n",
       " 'Anta',\n",
       " 'Antea',\n",
       " 'Antenora',\n",
       " 'Antenore',\n",
       " 'Anteo',\n",
       " 'Antera',\n",
       " 'Anterino',\n",
       " 'Anterio',\n",
       " 'Antero',\n",
       " 'Antigona',\n",
       " 'Antigone',\n",
       " 'Antigono',\n",
       " 'Antima',\n",
       " 'Antimina',\n",
       " 'Antimino',\n",
       " 'Antimo',\n",
       " 'Antioca',\n",
       " 'Antioco',\n",
       " 'Anto',\n",
       " 'Antonangelo',\n",
       " 'Antonella',\n",
       " 'Antonello',\n",
       " 'Antonetta',\n",
       " 'Antonetto',\n",
       " 'Antonia',\n",
       " 'Antonia ivanova',\n",
       " 'Antoniano',\n",
       " 'Antonica',\n",
       " 'Antonico',\n",
       " 'Antonietta',\n",
       " 'Antonietto',\n",
       " 'Antonilla',\n",
       " 'Antonillo',\n",
       " 'Antonina',\n",
       " 'Antonino',\n",
       " 'Antonio',\n",
       " 'Antonio paolo',\n",
       " 'Antonita',\n",
       " 'Antoniuccio',\n",
       " 'Antonmaria',\n",
       " 'Antuono',\n",
       " 'Anwar',\n",
       " 'Anzia',\n",
       " 'Anzio',\n",
       " 'Aonio',\n",
       " 'Apollinare',\n",
       " 'Apollo',\n",
       " 'Apollonia',\n",
       " 'Apollonio',\n",
       " 'Apostolo',\n",
       " 'Apparizio',\n",
       " 'Appia',\n",
       " 'Appiano',\n",
       " 'Appio',\n",
       " 'Appioclaudio',\n",
       " 'Aprile',\n",
       " 'Aprilia',\n",
       " 'Aprilio',\n",
       " 'Aquila',\n",
       " 'Aquilante',\n",
       " 'Aquilina',\n",
       " 'Aquilino',\n",
       " 'Aquilio',\n",
       " 'Arabella',\n",
       " 'Aralda',\n",
       " 'Araldo',\n",
       " 'Arames',\n",
       " 'Aramis',\n",
       " 'Arbace',\n",
       " 'Arcadio',\n",
       " 'Arcangela',\n",
       " 'Arcangelina',\n",
       " 'Arcangelo',\n",
       " 'Arcangiolo',\n",
       " 'Arceo',\n",
       " 'Archelao',\n",
       " 'Archibaldo',\n",
       " 'Archimede',\n",
       " 'Archita',\n",
       " 'Arcibaldo',\n",
       " 'Arcide',\n",
       " 'Arcidio',\n",
       " 'Arcido',\n",
       " 'Arcisa',\n",
       " 'Arcisio',\n",
       " 'Arciso',\n",
       " 'Arconte',\n",
       " 'Arda',\n",
       " 'Ardemaro',\n",
       " 'Ardengo',\n",
       " 'Ardingo',\n",
       " 'Ardino',\n",
       " 'Ardita',\n",
       " 'Ardito',\n",
       " 'Ardo',\n",
       " 'Ardoino',\n",
       " 'Ardolino',\n",
       " 'Ardovino',\n",
       " 'Arduilio',\n",
       " 'Arduina',\n",
       " 'Arduino',\n",
       " 'Arduo',\n",
       " 'Arealdo',\n",
       " 'Arete',\n",
       " 'Argante',\n",
       " 'Argantina',\n",
       " 'Arge',\n",
       " 'Argea',\n",
       " 'Argemiro',\n",
       " 'Argenta',\n",
       " 'Argentina',\n",
       " 'Argentino',\n",
       " 'Argento',\n",
       " 'Argenzia',\n",
       " 'Argeo',\n",
       " 'Argia',\n",
       " 'Argimiro',\n",
       " 'Argio',\n",
       " 'Argo',\n",
       " 'Aria',\n",
       " 'Arialda',\n",
       " 'Arialdo',\n",
       " 'Ariana',\n",
       " 'Arianna',\n",
       " 'Arianno',\n",
       " 'Ariano',\n",
       " 'Ariberto',\n",
       " 'Ariela',\n",
       " 'Ariele',\n",
       " 'Ariella',\n",
       " 'Ariello',\n",
       " 'Arienzo',\n",
       " 'Arietta',\n",
       " 'Arietto',\n",
       " 'Arimondo',\n",
       " 'Arina',\n",
       " 'Arino',\n",
       " 'Ario',\n",
       " 'Ariola',\n",
       " 'Ariosta',\n",
       " 'Ariosto',\n",
       " 'Aris',\n",
       " 'Aristarco',\n",
       " 'Aristea',\n",
       " 'Aristeo',\n",
       " 'Aristide',\n",
       " 'Aristidina',\n",
       " 'Aristo',\n",
       " 'Aristodema',\n",
       " 'Aristodemo',\n",
       " 'Aristotele',\n",
       " 'Aristotile',\n",
       " 'Arkadiusz jan',\n",
       " 'Arlesiana',\n",
       " 'Arletta',\n",
       " 'Armanda',\n",
       " 'Armandina',\n",
       " 'Armandino',\n",
       " 'Armando',\n",
       " 'Armanno',\n",
       " 'Armano',\n",
       " 'Armela',\n",
       " 'Armelina',\n",
       " 'Armelinda',\n",
       " 'Armelindo',\n",
       " 'Armella',\n",
       " 'Armellina',\n",
       " 'Armellino',\n",
       " 'Armello',\n",
       " 'Armena',\n",
       " 'Armenia',\n",
       " 'Armenio',\n",
       " 'Armeno',\n",
       " 'Armentina',\n",
       " 'Armerina',\n",
       " 'Armerino',\n",
       " 'Armida',\n",
       " 'Armidio',\n",
       " 'Armido',\n",
       " 'Armildo',\n",
       " 'Armilla',\n",
       " 'Armina',\n",
       " 'Arminda',\n",
       " 'Armindo',\n",
       " 'Arminia',\n",
       " 'Arminio',\n",
       " 'Armino',\n",
       " 'Armirio',\n",
       " 'Armiro',\n",
       " 'Armistizio',\n",
       " 'Arnalda',\n",
       " 'Arnaldina',\n",
       " 'Arnaldo',\n",
       " 'Arnolda',\n",
       " 'Arnoldo',\n",
       " 'Arnolfo',\n",
       " 'Arolda',\n",
       " 'Aroldo',\n",
       " 'Aronne',\n",
       " 'Aronte',\n",
       " 'Arpalice',\n",
       " 'Arriga',\n",
       " 'Arrighetto',\n",
       " 'Arrigo',\n",
       " 'Arriguccio',\n",
       " 'Arsenia',\n",
       " 'Arsenio',\n",
       " 'Arseno',\n",
       " 'Arsilia',\n",
       " 'Arsilio',\n",
       " 'Artaserse',\n",
       " 'Artemia',\n",
       " 'Artemide',\n",
       " 'Artemidio',\n",
       " 'Artemino',\n",
       " 'Artemio',\n",
       " 'Artemisia',\n",
       " 'Artemisio',\n",
       " 'Artemo',\n",
       " 'Artenia',\n",
       " 'Artenice',\n",
       " 'Artenio',\n",
       " 'Arterio',\n",
       " 'Artico',\n",
       " 'Artide',\n",
       " 'Artiero',\n",
       " 'Artimina',\n",
       " 'Artimino',\n",
       " 'Artimio',\n",
       " 'Artura',\n",
       " 'Arturina',\n",
       " 'Arturino',\n",
       " 'Arturo',\n",
       " 'Arzelio',\n",
       " 'Arzenio',\n",
       " 'Ascania',\n",
       " 'Ascanio',\n",
       " 'Ascensa',\n",
       " 'Ascensina',\n",
       " 'Ascenso',\n",
       " 'Ascenza',\n",
       " 'Ascenzina',\n",
       " 'Ascenzio',\n",
       " 'Ascenzo',\n",
       " 'Asclepiade',\n",
       " 'Asclepio',\n",
       " 'Asdrubale',\n",
       " 'Asia',\n",
       " 'Asiago',\n",
       " 'Asmara',\n",
       " 'Asmarino',\n",
       " 'Asmaro',\n",
       " 'Asmerino',\n",
       " 'Aspasia',\n",
       " 'Aspasio',\n",
       " 'Aspromonte',\n",
       " 'Assalonne',\n",
       " 'Assenzio',\n",
       " 'Assuera',\n",
       " 'Assuero',\n",
       " 'Assunta',\n",
       " 'Assuntina',\n",
       " 'Assuntino',\n",
       " 'Assunto',\n",
       " 'Aster',\n",
       " 'Asteria',\n",
       " 'Asterino',\n",
       " 'Asterio',\n",
       " 'Astero',\n",
       " 'Astolfo',\n",
       " 'Astore',\n",
       " 'Astorino',\n",
       " 'Astorre',\n",
       " 'Astra',\n",
       " 'Astrea',\n",
       " ...]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get a list of common Italian names\n",
    "import pandas as pd\n",
    "import io\n",
    "import requests\n",
    "url=\"https://gist.githubusercontent.com/pdesterlich/2562329/raw/7c09ac44d769539c61df15d5b3c441eaebb77660/nomi_italiani.txt\"\n",
    "s=requests.get(url).content\n",
    "# remove the first couple of lines (which are comments) & capitalize first letter\n",
    "names = [name[0].upper() + name[1:] for name in s.decode('utf-8').split('\\n')[7:]]\n",
    "# visualize\n",
    "names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assembler = DocumentAssembler()\\\n",
    "    .setInputCol(\"value\")\\\n",
    "    .setOutputCol(\"document\")\\\n",
    "\n",
    "tokenizer = RecursiveTokenizer()\\\n",
    "    .setInputCols(\"document\")\\\n",
    "    .setOutputCol(\"token\")\\\n",
    "    .setInfixes([\"’\"])\\\n",
    "    .setWhitelist([\"L’unica\"])\\\n",
    "    .setPrefixes([\"\\\"\", \"“\", \"(\", \"[\", \"\\n\", \".\", \"l’\", \"dell’\", \"nell’\", \"sull’\", \"all’\", \"d’\", \"un’\"])\\\n",
    "    .setSuffixes([\"\\\"\", \"”\", \".\", \",\", \"?\", \")\", \"]\", \"!\", \";\", \":\"])\n",
    "\n",
    "# we're going to add a special class for names, and use another two\n",
    "# that come predefined with the model: numbers and dates\n",
    "spellChecker = ContextSpellCheckerApproach().\\\n",
    "    setInputCols(\"token\").\\\n",
    "    setOutputCol(\"corrected\").\\\n",
    "    addVocabClass('_NAME_', names).\\\n",
    "    setLanguageModelClasses(1650).\\\n",
    "    setWordMaxDistance(3).\\\n",
    "    setBatchSize(24).\\\n",
    "    setEpochs(8)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Let's try the tokenization here!\n",
    "Before we get into training, we first take a look at how the tokenization works, just to be sure we're happy with it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenization = Pipeline(\n",
    "    stages = [\n",
    "    assembler,\n",
    "    tokenizer\n",
    "  ])\n",
    "lp = LightPipeline(tokenization.fit(df.limit(10000)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = [\"La dona è nata dala costola dell’uoma.\",\n",
    "             \"L’unica maneira per realizare i proprio sogni è svegliarsi.\",\n",
    "             \"Smettila di penssare a cosa potrebbe andare male e inizia a pensare a cosa potrebbe andare bene.\",\n",
    "             \"L'è questione che ognunno c'ha il suo ramo di studi: il contadino studia la vaca e lo studente studia la fica.\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'document': ['La dona è nata dala costola dell’uoma.'],\n",
       "  'token': ['La',\n",
       "   'dona',\n",
       "   'è',\n",
       "   'nata',\n",
       "   'dala',\n",
       "   'costola',\n",
       "   'dell',\n",
       "   '’',\n",
       "   'uoma',\n",
       "   '.']},\n",
       " {'document': ['L’unica maneira per realizare i proprio sogni è svegliarsi.'],\n",
       "  'token': ['L’unica',\n",
       "   'maneira',\n",
       "   'per',\n",
       "   'realizare',\n",
       "   'i',\n",
       "   'proprio',\n",
       "   'sogni',\n",
       "   'è',\n",
       "   'svegliarsi',\n",
       "   '.']},\n",
       " {'document': ['Smettila di penssare a cosa potrebbe andare male e inizia a pensare a cosa potrebbe andare bene.'],\n",
       "  'token': ['Smettila',\n",
       "   'di',\n",
       "   'penssare',\n",
       "   'a',\n",
       "   'cosa',\n",
       "   'potrebbe',\n",
       "   'andare',\n",
       "   'male',\n",
       "   'e',\n",
       "   'inizia',\n",
       "   'a',\n",
       "   'pensare',\n",
       "   'a',\n",
       "   'cosa',\n",
       "   'potrebbe',\n",
       "   'andare',\n",
       "   'bene',\n",
       "   '.']},\n",
       " {'document': [\"L'è questione che ognunno c'ha il suo ramo di studi: il contadino studia la vaca e lo studente studia la fica.\"],\n",
       "  'token': [\"L'è\",\n",
       "   'questione',\n",
       "   'che',\n",
       "   'ognunno',\n",
       "   \"c'ha\",\n",
       "   'il',\n",
       "   'suo',\n",
       "   'ramo',\n",
       "   'di',\n",
       "   'studi',\n",
       "   ':',\n",
       "   'il',\n",
       "   'contadino',\n",
       "   'studia',\n",
       "   'la',\n",
       "   'vaca',\n",
       "   'e',\n",
       "   'lo',\n",
       "   'studente',\n",
       "   'studia',\n",
       "   'la',\n",
       "   'fica',\n",
       "   '.']}]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lp.annotate(sentences)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train the model\n",
    "Here we use the complete pipeline, and we limit the input dataset to 200000, which should be enough to get a better than trivial model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline = Pipeline(\n",
    "    stages = [\n",
    "    assembler,\n",
    "    tokenizer,\n",
    "    spellChecker\n",
    "  ])\n",
    "fitPipeline = pipeline.fit(df.limit(200000))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Persist the model\n",
    "Here we will save the model for later use."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fitPipeline.stages[-1].write().overwrite().save('italian_spell')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Try the model\n",
    "In subsequent uses you can either load it from disk, or access it through pretrained()."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "spellcheck_dl download started this may take some time.\n",
      "Approximate size to download 124.3 MB\n",
      "[OK!]\n"
     ]
    }
   ],
   "source": [
    "loaded = ContextSpellCheckerModel.pretrained('spellcheck_dl', \"it\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "loaded = ContextSpellCheckerModel.load('./italian_spell')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_lightpipeline():\n",
    "    pipeline = Pipeline(\n",
    "        stages = [\n",
    "        assembler,\n",
    "        tokenizer,\n",
    "        loaded\n",
    "      ])\n",
    "    fitPipeline = pipeline.fit(df)\n",
    "    return LightPipeline(fitPipeline)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "lp = get_lightpipeline()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['sonno',\n",
       " 'Gloria',\n",
       " 'ho',\n",
       " 'lasciato',\n",
       " 'la',\n",
       " 'patente',\n",
       " 'sul',\n",
       " 'tavolo',\n",
       " 'accanto',\n",
       " 'alla',\n",
       " 'frutta']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lp.annotate(\"sonno Glorea ho lasciatth la paterte sul tavolo acanto allu fruttu\")['corrected']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's do something a bit more challenging - pay attention to the 'lo' article"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'document': ['sono Glorea ho lasciatto lo patente sul tavolo acanto alla frutta'],\n",
       " 'token': ['sono',\n",
       "  'Glorea',\n",
       "  'ho',\n",
       "  'lasciatto',\n",
       "  'lo',\n",
       "  'patente',\n",
       "  'sul',\n",
       "  'tavolo',\n",
       "  'acanto',\n",
       "  'alla',\n",
       "  'frutta'],\n",
       " 'corrected': ['sono',\n",
       "  'Gloria',\n",
       "  'ha',\n",
       "  'lasciato',\n",
       "  'la',\n",
       "  'patente',\n",
       "  'sul',\n",
       "  'lavoro',\n",
       "  'accanto',\n",
       "  'alla',\n",
       "  'frutta']}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lp.annotate(\"sono Glorea ho lasciatto lo patente sul tavolo acanto alla frutta\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's try to adjust the model to both a) pay more attention to the context with the decreased tradeoff, and generate more corrections with the decrease in error threshold."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(18.0, 10.0)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loaded.getTradeoff(), loaded.getErrorThreshold()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "loaded.setTradeoff(5)\n",
    "loaded.setErrorThreshold(5)\n",
    "# Refresh the pipeline\n",
    "lp = get_lightpipeline()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'document': ['sono Glorea ho lasciatto lo patenre sul tavolo acanto alla frutta'],\n",
       " 'token': ['sono',\n",
       "  'Glorea',\n",
       "  'ho',\n",
       "  'lasciatto',\n",
       "  'lo',\n",
       "  'patenre',\n",
       "  'sul',\n",
       "  'tavolo',\n",
       "  'acanto',\n",
       "  'alla',\n",
       "  'frutta'],\n",
       " 'corrected': ['sono',\n",
       "  'Gloria',\n",
       "  'ho',\n",
       "  'lasciato',\n",
       "  'la',\n",
       "  'patente',\n",
       "  'sul',\n",
       "  'tavoli',\n",
       "  'accanto',\n",
       "  'alla',\n",
       "  'frutta']}"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lp.annotate(\"sono Glorea ho lasciatto lo patenre sul tavolo acanto alla frutta\")"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Raw Cell Format",
  "kernelspec": {
   "display_name": "tf",
   "language": "python",
   "name": "tf"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}