tutorials/streamlit_notebooks/SPELL_CHECKER_EN.ipynb

{"cells":[{"cell_type":"markdown","metadata":{"id":"I4b_2KemgDWf"},"source":["\n","\n","![JohnSnowLabs](https://nlp.johnsnowlabs.com/assets/images/logo.png)\n","\n","[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/SPELL_CHECKER_EN.ipynb)\n","\n","\n"]},{"cell_type":"markdown","metadata":{"id":"TnZG0I4ogNLI"},"source":["# **Spell check your text documents**"]},{"cell_type":"markdown","metadata":{"id":"apjCmRyjgQll"},"source":["## 1. Colab Setup"]},{"cell_type":"markdown","metadata":{"id":"2phEj9SygX4n"},"source":["Install dependencies"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"uAiXj3DOfyZ-"},"outputs":[],"source":["# Install PySpark and Spark NLP\n","! pip install -q pyspark==3.3.0 spark-nlp==4.2.8"]},{"cell_type":"markdown","metadata":{"id":"DE2XyDI7NHtg"},"source":["Import dependencies"]},{"cell_type":"code","execution_count":2,"metadata":{"id":"g9hfxX3df3n3","executionInfo":{"status":"ok","timestamp":1675538932922,"user_tz":-60,"elapsed":1298,"user":{"displayName":"Damla","userId":"03285166568766987047"}}},"outputs":[],"source":["import json\n","import pandas as pd\n","import numpy as np\n","\n","import sparknlp\n","import pyspark.sql.functions as F\n","\n","from pyspark.ml import Pipeline\n","from pyspark.sql import SparkSession\n","from sparknlp.annotator import *\n","from sparknlp.base import *\n","from sparknlp.pretrained import PretrainedPipeline\n","from pyspark.sql.types import StringType, IntegerType\n","\n","from sklearn.model_selection import train_test_split\n","from sklearn.metrics import classification_report, accuracy_score"]},{"cell_type":"markdown","metadata":{"id":"T3Ur62RrgaxX"},"source":["Start Spark Session"]},{"cell_type":"code","execution_count":3,"metadata":{"id":"763GC_wVNN6F","colab":{"base_uri":"https://localhost:8080/","height":244},"executionInfo":{"status":"ok","timestamp":1675538972243,"user_tz":-60,"elapsed":39324,"user":{"displayName":"Damla","userId":"03285166568766987047"}},"outputId":"e64f16a1-566d-4203-94c9-4b29085fc6c0"},"outputs":[{"output_type":"stream","name":"stdout","text":["Spark NLP version 4.2.8\n","Apache Spark version: 3.3.0\n"]},{"output_type":"execute_result","data":{"text/plain":["<pyspark.sql.session.SparkSession at 0x7f28848d2040>"],"text/html":["\n","            <div>\n","                <p><b>SparkSession - in-memory</b></p>\n","                \n","        <div>\n","            <p><b>SparkContext</b></p>\n","\n","            <p><a href=\"http://b1a2c15e9f6d:4040\">Spark UI</a></p>\n","\n","            <dl>\n","              <dt>Version</dt>\n","                <dd><code>v3.3.0</code></dd>\n","              <dt>Master</dt>\n","                <dd><code>local[*]</code></dd>\n","              <dt>AppName</dt>\n","                <dd><code>Spark NLP</code></dd>\n","            </dl>\n","        </div>\n","        \n","            </div>\n","        "]},"metadata":{},"execution_count":3}],"source":["spark = sparknlp.start()\n","\n","print(\"Spark NLP version\", sparknlp.version())\n","print(\"Apache Spark version:\", spark.version)\n","\n","spark"]},{"cell_type":"markdown","metadata":{"id":"AUP6-XeQgeQW"},"source":["## 2. Select the NER model and construct the pipeline"]},{"cell_type":"code","execution_count":4,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"taOqLG1Ogc3D","outputId":"632792d6-3bc3-4dc4-ed0d-578eabadb9b0","executionInfo":{"status":"ok","timestamp":1675539037222,"user_tz":-60,"elapsed":64986,"user":{"displayName":"Damla","userId":"03285166568766987047"}}},"outputs":[{"output_type":"stream","name":"stdout","text":["spellcheck_dl download started this may take some time.\n","Approximate size to download 95.1 MB\n","[OK!]\n"]}],"source":["document_assembler = DocumentAssembler()\\\n","  .setInputCol(\"text\")\\\n","  .setOutputCol(\"document\")\n","\n","tokenizer = RecursiveTokenizer()\\\n","  .setInputCols([\"document\"])\\\n","  .setOutputCol(\"token\")\\\n","  .setPrefixes([\"\\\"\", \"(\", \"[\", \"\\n\"])\\\n","  .setSuffixes([\".\", \",\", \"?\", \")\",\"!\", \"‘s\"])\n","\n","spell_model = ContextSpellCheckerModel\\\n","    .pretrained('spellcheck_dl')\\\n","    .setInputCols(\"token\")\\\n","    .setOutputCol(\"corrected\")\n","\n","finisher = Finisher().setInputCols(\"corrected\")\n","\n","light_pipeline = Pipeline(stages = [document_assembler,\n","                                    tokenizer,\n","                                    spell_model,\n","                                    finisher])\n","## For comparison\n","full_pipeline = Pipeline(stages = [document_assembler,\n","                                   tokenizer,\n","                                   spell_model])\n","\n","empty_ds = spark.createDataFrame([[\"\"]]).toDF(\"text\")\n","pipeline_model = full_pipeline.fit(empty_ds)\n","l_pipeline_model = LightPipeline(light_pipeline.fit(empty_ds))"]},{"cell_type":"markdown","metadata":{"id":"C_DzGuMPibKr"},"source":["## 3. Create example inputs"]},{"cell_type":"code","execution_count":5,"metadata":{"id":"c7yPd884i_XQ","executionInfo":{"status":"ok","timestamp":1675539037223,"user_tz":-60,"elapsed":15,"user":{"displayName":"Damla","userId":"03285166568766987047"}}},"outputs":[],"source":["# Enter examples as strings in this array\n","input_list = [\"Plaese alliow me tao introdduce myhelf, I am a man of waelth und tiaste\"]"]},{"cell_type":"markdown","metadata":{"id":"OvyOmrgHjO5J"},"source":["## 4. Use the pipeline to create outputs"]},{"cell_type":"markdown","metadata":{"id":"ICREynF-jzn8"},"source":["Full Pipeline"]},{"cell_type":"code","execution_count":6,"metadata":{"id":"RLbrPvC3jOSw","executionInfo":{"status":"ok","timestamp":1675539037602,"user_tz":-60,"elapsed":387,"user":{"displayName":"Damla","userId":"03285166568766987047"}}},"outputs":[],"source":["df = spark.createDataFrame(pd.DataFrame({\"text\": input_list}))\n","result = pipeline_model.transform(df)"]},{"cell_type":"markdown","metadata":{"id":"mhlbvs1Dj1ck"},"source":["Light Pipeline"]},{"cell_type":"code","execution_count":7,"metadata":{"id":"YulVrYgAj2ex","executionInfo":{"status":"ok","timestamp":1675539039127,"user_tz":-60,"elapsed":1527,"user":{"displayName":"Damla","userId":"03285166568766987047"}}},"outputs":[],"source":["# Light pipelines expect a single example.\n","light_result = l_pipeline_model.annotate(input_list[0])"]},{"cell_type":"markdown","metadata":{"id":"Uk72hQGqjX3f"},"source":["## 5. Visualize results"]},{"cell_type":"markdown","metadata":{"id":"J9JDK9_EjaF_"},"source":["Visualize comparison as dataframe"]},{"cell_type":"code","source":["result.select(F.explode(F.arrays_zip(result.token.result, \n","                                     result.corrected.result)).alias(\"cols\")) \\\n","      .select(F.expr(\"cols['0']\").alias(\"original\"),\n","              F.expr(\"cols['1']\").alias('corrected')).show(truncate=False)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"uekWfx_tFL3x","executionInfo":{"status":"ok","timestamp":1675539043923,"user_tz":-60,"elapsed":4802,"user":{"displayName":"Damla","userId":"03285166568766987047"}},"outputId":"ff9c36dd-d3b2-4a02-c5f0-16466c103c28"},"execution_count":8,"outputs":[{"output_type":"stream","name":"stdout","text":["+----------+---------+\n","|original  |corrected|\n","+----------+---------+\n","|Plaese    |Please   |\n","|alliow    |allow    |\n","|me        |me       |\n","|tao       |to       |\n","|introdduce|introduce|\n","|myhelf    |myself   |\n","|,         |,        |\n","|I         |I        |\n","|am        |am       |\n","|a         |a        |\n","|man       |man      |\n","|of        |of       |\n","|waelth    |wealth   |\n","|und       |and      |\n","|tiaste    |taste    |\n","+----------+---------+\n","\n"]}]},{"cell_type":"markdown","metadata":{"id":"5yItQfZmhcji"},"source":["Vizualise light pipeline and finished result"]},{"cell_type":"code","execution_count":9,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"9zHSiB0skP83","outputId":"b7a582a7-0f50-4af9-bbce-f3e83d3b8c76","executionInfo":{"status":"ok","timestamp":1675539043924,"user_tz":-60,"elapsed":8,"user":{"displayName":"Damla","userId":"03285166568766987047"}}},"outputs":[{"output_type":"execute_result","data":{"text/plain":["['Please',\n"," 'allow',\n"," 'me',\n"," 'to',\n"," 'introduce',\n"," 'myself',\n"," ',',\n"," 'I',\n"," 'am',\n"," 'a',\n"," 'man',\n"," 'of',\n"," 'wealth',\n"," 'and',\n"," 'taste']"]},"metadata":{},"execution_count":9}],"source":["# this finished result does not need parsing and can directly be used an any other task.\n","light_result['corrected']"]}],"metadata":{"colab":{"provenance":[]},"interpreter":{"hash":"45150093197569bb3a58481dcd32cd1adb45462fa3448719e8ac38ada6166aca"},"kernelspec":{"display_name":"Python 3.6.10 64-bit ('tensorflow2_p36': conda)","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.10"}},"nbformat":4,"nbformat_minor":0}