From 75d398e5184e91e92e05f6add4e538cb4ce4ceb3 Mon Sep 17 00:00:00 2001 From: Maziyar Panahi Date: Sat, 2 Mar 2024 07:02:23 +0100 Subject: [PATCH 1/7] Fix M2M100 not working on the second run [skip test] --- src/main/scala/com/johnsnowlabs/ml/ai/M2M100.scala | 14 ++++++-------- .../nlp/annotators/seq2seq/LLAMA2TestSpec.scala | 4 ++++ .../nlp/annotators/seq2seq/M2M100TestSpec.scala | 12 ++++++++---- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/main/scala/com/johnsnowlabs/ml/ai/M2M100.scala b/src/main/scala/com/johnsnowlabs/ml/ai/M2M100.scala index 1a33224d266c..9477b72f452c 100644 --- a/src/main/scala/com/johnsnowlabs/ml/ai/M2M100.scala +++ b/src/main/scala/com/johnsnowlabs/ml/ai/M2M100.scala @@ -196,12 +196,12 @@ private[johnsnowlabs] class M2M100( applySoftmax = false) // Run the prompt through the decoder and get the past -// val decoderOutputs = -// generateGreedyOnnx( -// decoderInputIds, -// decoderEncoderStateTensors, -// encoderAttentionMaskTensors, -// onnxSession = (decoderSession, decoderEnv)) + // val decoderOutputs = + // generateGreedyOnnx( + // decoderInputIds, + // decoderEncoderStateTensors, + // encoderAttentionMaskTensors, + // onnxSession = (decoderSession, decoderEnv)) // close sessions decoderEncoderStateTensors.fold( @@ -216,8 +216,6 @@ private[johnsnowlabs] class M2M100( }, onnxTensor => onnxTensor.close()) - encoderSession.close() - decoderSession.close() encoderEnv.close() decoderEnv.close() diff --git a/src/test/scala/com/johnsnowlabs/nlp/annotators/seq2seq/LLAMA2TestSpec.scala b/src/test/scala/com/johnsnowlabs/nlp/annotators/seq2seq/LLAMA2TestSpec.scala index aca009ab7ce7..1216707da242 100644 --- a/src/test/scala/com/johnsnowlabs/nlp/annotators/seq2seq/LLAMA2TestSpec.scala +++ b/src/test/scala/com/johnsnowlabs/nlp/annotators/seq2seq/LLAMA2TestSpec.scala @@ -50,6 +50,10 @@ class LLAMA2TestSpec extends AnyFlatSpec { val pipelineModel = pipeline.fit(testData) + pipelineModel + .transform(testData) + .show(truncate = false) + pipelineModel .transform(testData) .show(truncate = false) diff --git a/src/test/scala/com/johnsnowlabs/nlp/annotators/seq2seq/M2M100TestSpec.scala b/src/test/scala/com/johnsnowlabs/nlp/annotators/seq2seq/M2M100TestSpec.scala index 66ce09ff8bb7..f8629e2407bc 100644 --- a/src/test/scala/com/johnsnowlabs/nlp/annotators/seq2seq/M2M100TestSpec.scala +++ b/src/test/scala/com/johnsnowlabs/nlp/annotators/seq2seq/M2M100TestSpec.scala @@ -45,11 +45,15 @@ class M2M100TestSpec extends AnyFlatSpec { .setOutputCol("generation") .setBeamSize(1) - new Pipeline() + val pipeline = new Pipeline() .setStages(Array(documentAssembler, m2m100)) - .fit(testData) - .transform(testData) - .show(truncate = false) + + val pipelineModel = pipeline.fit(testData) + + val result = pipelineModel.transform(testData) + + result.show(truncate = false) + result.show(truncate = false) } From 88774541d63d593cbc2db8be6ad0ed369b6efd93 Mon Sep 17 00:00:00 2001 From: Maziyar Panahi Date: Sat, 2 Mar 2024 07:03:05 +0100 Subject: [PATCH 2/7] Fix ONNX models failing in clusters like Databricks --- .../com/johnsnowlabs/ml/onnx/OnnxSerializeModel.scala | 1 - src/main/scala/com/johnsnowlabs/ml/onnx/OnnxWrapper.scala | 7 +++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/scala/com/johnsnowlabs/ml/onnx/OnnxSerializeModel.scala b/src/main/scala/com/johnsnowlabs/ml/onnx/OnnxSerializeModel.scala index cf5802b39124..273562693001 100644 --- a/src/main/scala/com/johnsnowlabs/ml/onnx/OnnxSerializeModel.scala +++ b/src/main/scala/com/johnsnowlabs/ml/onnx/OnnxSerializeModel.scala @@ -16,7 +16,6 @@ package com.johnsnowlabs.ml.onnx -import ai.onnxruntime.{OrtEnvironment, OrtLoggingLevel} import ai.onnxruntime.OrtSession.SessionOptions import com.johnsnowlabs.util.FileHelper import org.apache.commons.io.FileUtils diff --git a/src/main/scala/com/johnsnowlabs/ml/onnx/OnnxWrapper.scala b/src/main/scala/com/johnsnowlabs/ml/onnx/OnnxWrapper.scala index fb53c35530ec..5478a5228299 100644 --- a/src/main/scala/com/johnsnowlabs/ml/onnx/OnnxWrapper.scala +++ b/src/main/scala/com/johnsnowlabs/ml/onnx/OnnxWrapper.scala @@ -101,6 +101,7 @@ object OnnxWrapper { } } + // TODO: the parts related to onnx_data should be refactored once we support addFile() def read( modelPath: String, zipped: Boolean = true, @@ -152,7 +153,7 @@ object OnnxWrapper { session = _session env = _env } else { - val (_session, _env) = withSafeOnnxModelLoader(modelBytes, sessionOptions, Some(onnxFile)) + val (_session, _env) = withSafeOnnxModelLoader(modelBytes, sessionOptions, None) session = _session env = _env @@ -160,7 +161,9 @@ object OnnxWrapper { // 4. Remove tmp folder FileHelper.delete(tmpFolder) - val onnxWrapper = new OnnxWrapper(modelBytes, Option(onnxFile)) + val onnxWrapper = + if (onnxDataFileExist) new OnnxWrapper(modelBytes, Option(onnxFile)) + else new OnnxWrapper(modelBytes) onnxWrapper.ortSession = session onnxWrapper.ortEnv = env onnxWrapper From 901c88425a8153b1224c9dce103fd46e9ff43a48 Mon Sep 17 00:00:00 2001 From: Danilo Burbano <37355249+danilojsl@users.noreply.github.com> Date: Sun, 3 Mar 2024 16:07:45 -0500 Subject: [PATCH 3/7] [SPARKNLP-994] Adding changes required to load ONNX ZeroShotNerClassification and fix predictions issue (#14186) --- .../2023-02-08-zero_shot_ner_roberta_en.md | 6 +-- .../ml/ai/RoBertaClassification.scala | 4 +- .../ml/ai/ZeroShotNerClassification.scala | 3 +- .../annotators/ner/dl/ZeroShotNerModel.scala | 40 ++++++++++++++----- 4 files changed, 38 insertions(+), 15 deletions(-) diff --git a/docs/_posts/maziyarpanahi/2023-02-08-zero_shot_ner_roberta_en.md b/docs/_posts/maziyarpanahi/2023-02-08-zero_shot_ner_roberta_en.md index 5a07d61ac6ad..ce1e1995b037 100644 --- a/docs/_posts/maziyarpanahi/2023-02-08-zero_shot_ner_roberta_en.md +++ b/docs/_posts/maziyarpanahi/2023-02-08-zero_shot_ner_roberta_en.md @@ -101,15 +101,15 @@ val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "zero_shot_ner")) .setOutputCol("ner_chunk") -val pipeline = new .setStages(Array( +val pipeline = new Pipeline().setStages(Array( documentAssembler, sentenceDetector, tokenizer, zero_shot_ner, ner_converter)) -val data = Seq(Array("Hellen works in London, Paris and Berlin. My name is Clara, I live in New York and Hellen lives in Paris.", - "John is a man who works in London, London and London.")toDS().toDF("text") +val data = Seq("Hellen works in London, Paris and Berlin. My name is Clara, I live in New York and Hellen lives in Paris.", + "John is a man who works in London, London and London.").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` diff --git a/src/main/scala/com/johnsnowlabs/ml/ai/RoBertaClassification.scala b/src/main/scala/com/johnsnowlabs/ml/ai/RoBertaClassification.scala index 4296c8bcf5a5..fda62f52bee6 100644 --- a/src/main/scala/com/johnsnowlabs/ml/ai/RoBertaClassification.scala +++ b/src/main/scala/com/johnsnowlabs/ml/ai/RoBertaClassification.scala @@ -352,11 +352,11 @@ private[johnsnowlabs] class RoBertaClassification( val endDim = endLogits.length / batchLength val endScores: Array[Array[Float]] = - endLogits.grouped(endDim).toArray + endLogits.grouped(endDim).map(scores => calculateSoftmax(scores)).toArray val startDim = startLogits.length / batchLength val startScores: Array[Array[Float]] = - startLogits.grouped(startDim).toArray + startLogits.grouped(startDim).map(scores => calculateSoftmax(scores)).toArray (startScores, endScores) } diff --git a/src/main/scala/com/johnsnowlabs/ml/ai/ZeroShotNerClassification.scala b/src/main/scala/com/johnsnowlabs/ml/ai/ZeroShotNerClassification.scala index f1cc6cb7b9a8..638138223176 100644 --- a/src/main/scala/com/johnsnowlabs/ml/ai/ZeroShotNerClassification.scala +++ b/src/main/scala/com/johnsnowlabs/ml/ai/ZeroShotNerClassification.scala @@ -88,7 +88,7 @@ private[johnsnowlabs] class ZeroShotNerClassification( val allTokenPieces = wordPieceTokenizedQuestion.head.tokens ++ wordPieceTokenizedContext.flatMap(x => x.tokens) - val decodedAnswer = allTokenPieces.slice(startIndex._2 - 2, endIndex._2 - 1) + val decodedAnswer = allTokenPieces.slice(startIndex._2 - 3, endIndex._2 - 2) // Check if the answer span starts at the CLS symbol 0 - if so return empty string val content = if (startIndex._2 > 0) @@ -141,4 +141,5 @@ private[johnsnowlabs] class ZeroShotNerClassification( } } + } diff --git a/src/main/scala/com/johnsnowlabs/nlp/annotators/ner/dl/ZeroShotNerModel.scala b/src/main/scala/com/johnsnowlabs/nlp/annotators/ner/dl/ZeroShotNerModel.scala index 5c26a2615fbb..518781817580 100644 --- a/src/main/scala/com/johnsnowlabs/nlp/annotators/ner/dl/ZeroShotNerModel.scala +++ b/src/main/scala/com/johnsnowlabs/nlp/annotators/ner/dl/ZeroShotNerModel.scala @@ -17,8 +17,10 @@ package com.johnsnowlabs.nlp.annotators.ner.dl import com.johnsnowlabs.ml.ai.{RoBertaClassification, ZeroShotNerClassification} -import com.johnsnowlabs.ml.onnx.OnnxWrapper +import com.johnsnowlabs.ml.onnx.{OnnxWrapper, ReadOnnxModel} import com.johnsnowlabs.ml.tensorflow.{ReadTensorflowModel, TensorflowWrapper} +import com.johnsnowlabs.ml.util.LoadExternalModel.notSupportedEngineError +import com.johnsnowlabs.ml.util.{ONNX, TensorFlow} import com.johnsnowlabs.nlp.AnnotatorType.{DOCUMENT, NAMED_ENTITY, TOKEN} import com.johnsnowlabs.nlp.annotator.RoBertaForQuestionAnswering import com.johnsnowlabs.nlp.pretrained.ResourceDownloader @@ -448,19 +450,34 @@ trait ReadablePretrainedZeroShotNer } } -trait ReadZeroShotNerDLModel extends ReadTensorflowModel { +trait ReadZeroShotNerDLModel extends ReadTensorflowModel with ReadOnnxModel { this: ParamsAndFeaturesReadable[ZeroShotNerModel] => override val tfFile: String = "roberta_classification_tensorflow" + override val onnxFile: String = "roberta_classification_onnx" - def readTensorflow(instance: ZeroShotNerModel, path: String, spark: SparkSession): Unit = { - - val tfWrapper = - readTensorflowModel(path, spark, "_roberta_classification_tf", initAllTables = false) - instance.setModelIfNotSet(spark, Some(tfWrapper), None) + def readModel(instance: ZeroShotNerModel, path: String, spark: SparkSession): Unit = { + instance.getEngine match { + case TensorFlow.name => { + val tfWrapper = readTensorflowModel(path, spark, "_roberta_classification_tf", initAllTables = false) + instance.setModelIfNotSet(spark, Some(tfWrapper), None) + } + case ONNX.name => { + val onnxWrapper = readOnnxModel( + path, + spark, + "_roberta_classification_onnx", + zipped = true, + useBundle = false, + None) + instance.setModelIfNotSet(spark, None, Some(onnxWrapper)) + } + case _ => + throw new Exception(notSupportedEngineError) + } } - addReader(readTensorflow) + addReader(readModel) } object ZeroShotNerModel extends ReadablePretrainedZeroShotNer with ReadZeroShotNerDLModel { @@ -487,7 +504,12 @@ object ZeroShotNerModel extends ReadablePretrainedZeroShotNer with ReadZeroShotN newModel.setSignatures( model.signatures.get.getOrElse(throw new RuntimeException("Signatures not set"))) - newModel.setModelIfNotSet(spark, model.getModelIfNotSet.tensorflowWrapper, None) + model.getEngine match { + case TensorFlow.name => + newModel.setModelIfNotSet(spark, model.getModelIfNotSet.tensorflowWrapper, None) + case ONNX.name => + newModel.setModelIfNotSet(spark, None, model.getModelIfNotSet.onnxWrapper) + } model .extractParamMap() From 19367f2cd8e03d58baabc203cfc87b084cdb39d9 Mon Sep 17 00:00:00 2001 From: Abdullah mubeen <77073730+AbdullahMubeenAnwar@users.noreply.github.com> Date: Mon, 4 Mar 2024 02:18:19 +0500 Subject: [PATCH 4/7] adding colab notebook for M2M100 (#14191) --- ...Multilingual_Translation_with_M2M100.ipynb | 257 ++++++++++++++++++ 1 file changed, 257 insertions(+) create mode 100644 examples/python/annotation/text/english/language-translation/Multilingual_Translation_with_M2M100.ipynb diff --git a/examples/python/annotation/text/english/language-translation/Multilingual_Translation_with_M2M100.ipynb b/examples/python/annotation/text/english/language-translation/Multilingual_Translation_with_M2M100.ipynb new file mode 100644 index 000000000000..48d94f53211b --- /dev/null +++ b/examples/python/annotation/text/english/language-translation/Multilingual_Translation_with_M2M100.ipynb @@ -0,0 +1,257 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "rAklGDKcIxuT" + }, + "source": [ + "![JohnSnowLabs](https://sparknlp.org/assets/images/logo.png)\n", + "\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp/tree/master/examples/python/annotation/text/english/language-translation/Multilingual_Translation_with_M2M100.ipynb)" + ] + }, + { + "cell_type": "markdown", + "source": [ + "# Multilingual Translation with M2M100" + ], + "metadata": { + "id": "_b6aR_k6Oi9Q" + } + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uI7yhCibA9cf" + }, + "source": [ + "## Colab Setup" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "id": "4WQLLrIUA9cg", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "87c0ea9b-0f20-4361-fa2e-5eb6f8113507" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Installing PySpark 3.2.3 and Spark NLP 5.3.0\n", + "setup Colab for PySpark 3.2.3 and Spark NLP 5.3.0\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m281.5/281.5 MB\u001b[0m \u001b[31m1.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m564.8/564.8 kB\u001b[0m \u001b[31m59.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m199.7/199.7 kB\u001b[0m \u001b[31m23.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25h Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n" + ] + } + ], + "source": [ + "!wget -q http://setup.johnsnowlabs.com/colab.sh -O - | bash" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "id": "KzMHa0HdA9ch", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "f190dcf5-01bb-49d1-e17a-79a4123c3a13" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Spark NLP version 5.3.0\n", + "Apache Spark version: 3.2.3\n" + ] + } + ], + "source": [ + "import sparknlp\n", + "from sparknlp.base import *\n", + "from sparknlp.annotator import *\n", + "from pyspark.ml import Pipeline\n", + "\n", + "spark = sparknlp.start()\n", + "\n", + "print(\"Spark NLP version\", sparknlp.version())\n", + "print(\"Apache Spark version:\", spark.version)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1zvc_5Y1Ixuk" + }, + "source": [ + "# Define Spark NLP pipeline" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2Y9GpdJhXIpD" + }, + "source": [ + "**A sample text in Chinese - we'll translate it to English**" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "id": "POzRV_wSK3YF" + }, + "outputs": [], + "source": [ + "text = \"\"\"除了是北方之王之外，约翰·斯诺还是一位英国医生，也是麻醉和医疗卫生发展的领导者。 他被认为是第一个利用数据治愈 1854 年霍乱爆发的人。\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "id": "0UNHPnccIxul", + "outputId": "d0f00dcc-d7bc-480d-9e82-e924eace7d1a", + "colab": { + "base_uri": "https://localhost:8080/" + } + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "m2m100_418M download started this may take some time.\n", + "Approximate size to download 2.8 GB\n", + "[OK!]\n" + ] + } + ], + "source": [ + "documentAssembler = DocumentAssembler()\\\n", + " .setInputCol(\"text\")\\\n", + " .setOutputCol(\"document\")\n", + "\n", + "m2m100 = M2M100Transformer.pretrained() \\\n", + " .setInputCols([\"document\"]) \\\n", + " .setMaxOutputLength(50) \\\n", + " .setOutputCol(\"generation\") \\\n", + " .setSrcLang(\"zh\") \\\n", + " .setTgtLang(\"en\")\n", + "\n", + "tl_pipeline = Pipeline(\n", + " stages=[documentAssembler, m2m100]\n", + " )" + ] + }, + { + "cell_type": "markdown", + "source": [ + "# Light Pipeline version" + ], + "metadata": { + "id": "v4QtpYx2SK7y" + } + }, + { + "cell_type": "markdown", + "source": [ + "Let's create the light Pipiline" + ], + "metadata": { + "id": "XpcordqsSmE3" + } + }, + { + "cell_type": "code", + "source": [ + "empty_df = spark.createDataFrame([[\"\"]]).toDF('text')\n", + "pipeline_model = tl_pipeline.fit(empty_df)\n", + "model = LightPipeline(pipeline_model)\n", + "res = model.fullAnnotate(text)" + ], + "metadata": { + "id": "CoQAJpOxQ1zb" + }, + "execution_count": 7, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "visualize the results" + ], + "metadata": { + "id": "t-L_JARWSszu" + } + }, + { + "cell_type": "code", + "source": [ + "print ('Original:', text, '\\n\\n')\n", + "\n", + "print ('Translated:\\n')\n", + "for sentence in res[0]['generation']:\n", + " print (sentence.result)" + ], + "metadata": { + "id": "ZnD-QGuJREW8", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "51258f55-1ec2-4e86-f312-1e057dbec5b8" + }, + "execution_count": 16, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Original: 除了是北方之王之外，约翰·斯诺还是一位英国医生，也是麻醉和医疗卫生发展的领导者。 他被认为是第一个利用数据治愈 1854 年霍乱爆发的人。 \n", + "\n", + "\n", + "Translated:\n", + "\n", + "In addition to being the King of the North, John Snow was also a British doctor and a leader in the development of anesthesia and health care. he was considered the first person to use data to cure the 1854 cholera outbreak.\n" + ] + } + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file From 4980740c3b91dca2cc85c33ba095c28c3351c57f Mon Sep 17 00:00:00 2001 From: Maziyar Panahi Date: Mon, 4 Mar 2024 09:34:59 +0100 Subject: [PATCH 5/7] Bump to 5.3.1 and update CHANGELOG [run doc] --- CHANGELOG | 12 +++ README.md | 88 +++++++++---------- build.sbt | 2 +- docs/README.md | 88 +++++++++---------- docs/_layouts/landing.html | 2 +- docs/en/concepts.md | 2 +- docs/en/examples.md | 4 +- docs/en/hardware_acceleration.md | 2 +- docs/en/install.md | 54 ++++++------ docs/en/spark_nlp.md | 2 +- python/README.md | 88 +++++++++---------- python/docs/conf.py | 2 +- python/setup.py | 2 +- python/sparknlp/__init__.py | 4 +- scripts/colab_setup.sh | 2 +- scripts/kaggle_setup.sh | 2 +- scripts/sagemaker_setup.sh | 2 +- .../scala/com/johnsnowlabs/nlp/SparkNLP.scala | 2 +- .../scala/com/johnsnowlabs/util/Build.scala | 2 +- 19 files changed, 187 insertions(+), 175 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index f327548e3f7e..7ae0d8e08396 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,15 @@ +======== +5.3.1 +======== +---------------- +Bug Fixes +---------------- +* Fix M2M100 not working on the second run (closing the ONNX Session by mistake) +* Fix ONNX models failing in clusters like Databricks +* Fix `ZeroShotNerClassification` issue with NerConverter +* adding colab notebook for M2M100 + + ======== 5.3.0 ======== diff --git a/README.md b/README.md index 5f4c9637cd89..dfff9ca155cf 100644 --- a/README.md +++ b/README.md @@ -165,7 +165,7 @@ To use Spark NLP you need the following requirements: **GPU (optional):** -Spark NLP 5.3.0 is built with ONNX 1.17.0 and TensorFlow 2.7.1 deep learning engines. The minimum following NVIDIA® software are only required for GPU support: +Spark NLP 5.3.1 is built with ONNX 1.17.0 and TensorFlow 2.7.1 deep learning engines. The minimum following NVIDIA® software are only required for GPU support: - NVIDIA® GPU drivers version 450.80.02 or higher - CUDA® Toolkit 11.2 @@ -181,7 +181,7 @@ $ java -version $ conda create -n sparknlp python=3.7 -y $ conda activate sparknlp # spark-nlp by default is based on pyspark 3.x -$ pip install spark-nlp==5.3.0 pyspark==3.3.1 +$ pip install spark-nlp==5.3.1 pyspark==3.3.1 ``` In Python console or Jupyter `Python3` kernel: @@ -226,7 +226,7 @@ For more examples, you can visit our dedicated [examples](https://github.com/Joh ## Apache Spark Support -Spark NLP *5.3.0* has been built on top of Apache Spark 3.4 while fully supports Apache Spark 3.0.x, 3.1.x, 3.2.x, 3.3.x, 3.4.x, and 3.5.x +Spark NLP *5.3.1* has been built on top of Apache Spark 3.4 while fully supports Apache Spark 3.0.x, 3.1.x, 3.2.x, 3.3.x, 3.4.x, and 3.5.x | Spark NLP | Apache Spark 3.5.x | Apache Spark 3.4.x | Apache Spark 3.3.x | Apache Spark 3.2.x | Apache Spark 3.1.x | Apache Spark 3.0.x | Apache Spark 2.4.x | Apache Spark 2.3.x | |-----------|--------------------|--------------------|--------------------|--------------------|--------------------|--------------------|--------------------|--------------------| @@ -270,7 +270,7 @@ Find out more about `Spark NLP` versions from our [release notes](https://github ## Databricks Support -Spark NLP 5.3.0 has been tested and is compatible with the following runtimes: +Spark NLP 5.3.1 has been tested and is compatible with the following runtimes: **CPU:** @@ -343,7 +343,7 @@ Spark NLP 5.3.0 has been tested and is compatible with the following runtimes: ## EMR Support -Spark NLP 5.3.0 has been tested and is compatible with the following EMR releases: +Spark NLP 5.3.1 has been tested and is compatible with the following EMR releases: - emr-6.2.0 - emr-6.3.0 @@ -393,11 +393,11 @@ Spark NLP supports all major releases of Apache Spark 3.0.x, Apache Spark 3.1.x, ```sh # CPU -spark-shell --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 +spark-shell --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1 -pyspark --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 +pyspark --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1 -spark-submit --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 +spark-submit --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1 ``` The `spark-nlp` has been published to @@ -406,11 +406,11 @@ the [Maven Repository](https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/s ```sh # GPU -spark-shell --packages com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:5.3.0 +spark-shell --packages com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:5.3.1 -pyspark --packages com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:5.3.0 +pyspark --packages com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:5.3.1 -spark-submit --packages com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:5.3.0 +spark-submit --packages com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:5.3.1 ``` @@ -420,11 +420,11 @@ the [Maven Repository](https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/s ```sh # AArch64 -spark-shell --packages com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:5.3.0 +spark-shell --packages com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:5.3.1 -pyspark --packages com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:5.3.0 +pyspark --packages com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:5.3.1 -spark-submit --packages com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:5.3.0 +spark-submit --packages com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:5.3.1 ``` @@ -434,11 +434,11 @@ the [Maven Repository](https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/s ```sh # M1/M2 (Apple Silicon) -spark-shell --packages com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:5.3.0 +spark-shell --packages com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:5.3.1 -pyspark --packages com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:5.3.0 +pyspark --packages com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:5.3.1 -spark-submit --packages com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:5.3.0 +spark-submit --packages com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:5.3.1 ``` @@ -452,7 +452,7 @@ set in your SparkSession: spark-shell \ --driver-memory 16g \ --conf spark.kryoserializer.buffer.max=2000M \ - --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 + --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1 ``` ## Scala @@ -470,7 +470,7 @@ coordinates: com.johnsnowlabs.nlp spark-nlp_2.12 - 5.3.0 + 5.3.1 ``` @@ -481,7 +481,7 @@ coordinates: com.johnsnowlabs.nlp spark-nlp-gpu_2.12 - 5.3.0 + 5.3.1 ``` @@ -492,7 +492,7 @@ coordinates: com.johnsnowlabs.nlp spark-nlp-aarch64_2.12 - 5.3.0 + 5.3.1 ``` @@ -503,7 +503,7 @@ coordinates: com.johnsnowlabs.nlp spark-nlp-silicon_2.12 - 5.3.0 + 5.3.1 ``` @@ -513,28 +513,28 @@ coordinates: ```sbtshell // https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/spark-nlp -libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp" % "5.3.0" +libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp" % "5.3.1" ``` **spark-nlp-gpu:** ```sbtshell // https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/spark-nlp-gpu -libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-gpu" % "5.3.0" +libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-gpu" % "5.3.1" ``` **spark-nlp-aarch64:** ```sbtshell // https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/spark-nlp-aarch64 -libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-aarch64" % "5.3.0" +libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-aarch64" % "5.3.1" ``` **spark-nlp-silicon:** ```sbtshell // https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/spark-nlp-silicon -libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-silicon" % "5.3.0" +libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-silicon" % "5.3.1" ``` Maven @@ -556,7 +556,7 @@ If you installed pyspark through pip/conda, you can install `spark-nlp` through Pip: ```bash -pip install spark-nlp==5.3.0 +pip install spark-nlp==5.3.1 ``` Conda: @@ -585,7 +585,7 @@ spark = SparkSession.builder .config("spark.driver.memory", "16G") .config("spark.driver.maxResultSize", "0") .config("spark.kryoserializer.buffer.max", "2000M") - .config("spark.jars.packages", "com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0") + .config("spark.jars.packages", "com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1") .getOrCreate() ``` @@ -656,7 +656,7 @@ Use either one of the following options - Add the following Maven Coordinates to the interpreter's library list ```bash -com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 +com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1 ``` - Add a path to pre-built jar from [here](#compiled-jars) in the interpreter's library list making sure the jar is @@ -667,7 +667,7 @@ com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 Apart from the previous step, install the python module through pip ```bash -pip install spark-nlp==5.3.0 +pip install spark-nlp==5.3.1 ``` Or you can install `spark-nlp` from inside Zeppelin by using Conda: @@ -695,7 +695,7 @@ launch the Jupyter from the same Python environment: $ conda create -n sparknlp python=3.8 -y $ conda activate sparknlp # spark-nlp by default is based on pyspark 3.x -$ pip install spark-nlp==5.3.0 pyspark==3.3.1 jupyter +$ pip install spark-nlp==5.3.1 pyspark==3.3.1 jupyter $ jupyter notebook ``` @@ -712,7 +712,7 @@ export PYSPARK_PYTHON=python3 export PYSPARK_DRIVER_PYTHON=jupyter export PYSPARK_DRIVER_PYTHON_OPTS=notebook -pyspark --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 +pyspark --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1 ``` Alternatively, you can mix in using `--jars` option for pyspark + `pip install spark-nlp` @@ -739,7 +739,7 @@ This script comes with the two options to define `pyspark` and `spark-nlp` versi # -s is for spark-nlp # -g will enable upgrading libcudnn8 to 8.1.0 on Google Colab for GPU usage # by default they are set to the latest -!wget https://setup.johnsnowlabs.com/colab.sh -O - | bash /dev/stdin -p 3.2.3 -s 5.3.0 +!wget https://setup.johnsnowlabs.com/colab.sh -O - | bash /dev/stdin -p 3.2.3 -s 5.3.1 ``` [Spark NLP quick start on Google Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp/blob/master/examples/python/quick_start_google_colab.ipynb) @@ -762,7 +762,7 @@ This script comes with the two options to define `pyspark` and `spark-nlp` versi # -s is for spark-nlp # -g will enable upgrading libcudnn8 to 8.1.0 on Kaggle for GPU usage # by default they are set to the latest -!wget https://setup.johnsnowlabs.com/colab.sh -O - | bash /dev/stdin -p 3.2.3 -s 5.3.0 +!wget https://setup.johnsnowlabs.com/colab.sh -O - | bash /dev/stdin -p 3.2.3 -s 5.3.1 ``` [Spark NLP quick start on Kaggle Kernel](https://www.kaggle.com/mozzie/spark-nlp-named-entity-recognition) is a live @@ -781,9 +781,9 @@ demo on Kaggle Kernel that performs named entity recognitions by using Spark NLP 3. In `Libraries` tab inside your cluster you need to follow these steps: - 3.1. Install New -> PyPI -> `spark-nlp==5.3.0` -> Install + 3.1. Install New -> PyPI -> `spark-nlp==5.3.1` -> Install - 3.2. Install New -> Maven -> Coordinates -> `com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0` -> Install + 3.2. Install New -> Maven -> Coordinates -> `com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1` -> Install 4. Now you can attach your notebook to the cluster and use Spark NLP! @@ -834,7 +834,7 @@ A sample of your software configuration in JSON on S3 (must be public access): "spark.kryoserializer.buffer.max": "2000M", "spark.serializer": "org.apache.spark.serializer.KryoSerializer", "spark.driver.maxResultSize": "0", - "spark.jars.packages": "com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0" + "spark.jars.packages": "com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1" } }] ``` @@ -843,7 +843,7 @@ A sample of AWS CLI to launch EMR cluster: ```.sh aws emr create-cluster \ ---name "Spark NLP 5.3.0" \ +--name "Spark NLP 5.3.1" \ --release-label emr-6.2.0 \ --applications Name=Hadoop Name=Spark Name=Hive \ --instance-type m4.4xlarge \ @@ -907,7 +907,7 @@ gcloud dataproc clusters create ${CLUSTER_NAME} \ --enable-component-gateway \ --metadata 'PIP_PACKAGES=spark-nlp spark-nlp-display google-cloud-bigquery google-cloud-storage' \ --initialization-actions gs://goog-dataproc-initialization-actions-${REGION}/python/pip-install.sh \ - --properties spark:spark.serializer=org.apache.spark.serializer.KryoSerializer,spark:spark.driver.maxResultSize=0,spark:spark.kryoserializer.buffer.max=2000M,spark:spark.jars.packages=com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 + --properties spark:spark.serializer=org.apache.spark.serializer.KryoSerializer,spark:spark.driver.maxResultSize=0,spark:spark.kryoserializer.buffer.max=2000M,spark:spark.jars.packages=com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1 ``` 2. On an existing one, you need to install spark-nlp and spark-nlp-display packages from PyPI. @@ -950,7 +950,7 @@ spark = SparkSession.builder .config("spark.kryoserializer.buffer.max", "2000m") .config("spark.jsl.settings.pretrained.cache_folder", "sample_data/pretrained") .config("spark.jsl.settings.storage.cluster_tmp_dir", "sample_data/storage") - .config("spark.jars.packages", "com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0") + .config("spark.jars.packages", "com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1") .getOrCreate() ``` @@ -964,7 +964,7 @@ spark-shell \ --conf spark.kryoserializer.buffer.max=2000M \ --conf spark.jsl.settings.pretrained.cache_folder="sample_data/pretrained" \ --conf spark.jsl.settings.storage.cluster_tmp_dir="sample_data/storage" \ - --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 + --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1 ``` **pyspark:** @@ -977,7 +977,7 @@ pyspark \ --conf spark.kryoserializer.buffer.max=2000M \ --conf spark.jsl.settings.pretrained.cache_folder="sample_data/pretrained" \ --conf spark.jsl.settings.storage.cluster_tmp_dir="sample_data/storage" \ - --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 + --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1 ``` **Databricks:** @@ -1249,7 +1249,7 @@ spark = SparkSession.builder .config("spark.driver.memory", "16G") .config("spark.driver.maxResultSize", "0") .config("spark.kryoserializer.buffer.max", "2000M") - .config("spark.jars", "/tmp/spark-nlp-assembly-5.3.0.jar") + .config("spark.jars", "/tmp/spark-nlp-assembly-5.3.1.jar") .getOrCreate() ``` @@ -1258,7 +1258,7 @@ spark = SparkSession.builder version (3.0.x, 3.1.x, 3.2.x, 3.3.x, 3.4.x, and 3.5.x) - If you are local, you can load the Fat JAR from your local FileSystem, however, if you are in a cluster setup you need to put the Fat JAR on a distributed FileSystem such as HDFS, DBFS, S3, etc. ( - i.e., `hdfs:///tmp/spark-nlp-assembly-5.3.0.jar`) + i.e., `hdfs:///tmp/spark-nlp-assembly-5.3.1.jar`) Example of using pretrained Models and Pipelines in offline: diff --git a/build.sbt b/build.sbt index 3f45d5ee14d6..70c46d429421 100644 --- a/build.sbt +++ b/build.sbt @@ -6,7 +6,7 @@ name := getPackageName(is_silicon, is_gpu, is_aarch64) organization := "com.johnsnowlabs.nlp" -version := "5.3.0" +version := "5.3.1" (ThisBuild / scalaVersion) := scalaVer diff --git a/docs/README.md b/docs/README.md index 3db879d41867..c07d04cf0fe5 100644 --- a/docs/README.md +++ b/docs/README.md @@ -173,7 +173,7 @@ To use Spark NLP you need the following requirements: **GPU (optional):** -Spark NLP 5.3.0 is built with ONNX 1.16.3 and TensorFlow 2.7.1 deep learning engines. The minimum following NVIDIA® software are only required for GPU support: +Spark NLP 5.3.1 is built with ONNX 1.16.3 and TensorFlow 2.7.1 deep learning engines. The minimum following NVIDIA® software are only required for GPU support: - NVIDIA® GPU drivers version 450.80.02 or higher - CUDA® Toolkit 11.2 @@ -189,7 +189,7 @@ $ java -version $ conda create -n sparknlp python=3.7 -y $ conda activate sparknlp # spark-nlp by default is based on pyspark 3.x -$ pip install spark-nlp==5.3.0 pyspark==3.3.1 +$ pip install spark-nlp==5.3.1 pyspark==3.3.1 ``` In Python console or Jupyter `Python3` kernel: @@ -234,7 +234,7 @@ For more examples, you can visit our dedicated [examples](https://github.com/Joh ## Apache Spark Support -Spark NLP *5.3.0* has been built on top of Apache Spark 3.4 while fully supports Apache Spark 3.0.x, 3.1.x, 3.2.x, 3.3.x, 3.4.x, and 3.5.x +Spark NLP *5.3.1* has been built on top of Apache Spark 3.4 while fully supports Apache Spark 3.0.x, 3.1.x, 3.2.x, 3.3.x, 3.4.x, and 3.5.x | Spark NLP | Apache Spark 3.5.x | Apache Spark 3.4.x | Apache Spark 3.3.x | Apache Spark 3.2.x | Apache Spark 3.1.x | Apache Spark 3.0.x | Apache Spark 2.4.x | Apache Spark 2.3.x | |-----------|--------------------|--------------------|--------------------|--------------------|--------------------|--------------------|--------------------|--------------------| @@ -276,7 +276,7 @@ Find out more about `Spark NLP` versions from our [release notes](https://github ## Databricks Support -Spark NLP 5.3.0 has been tested and is compatible with the following runtimes: +Spark NLP 5.3.1 has been tested and is compatible with the following runtimes: **CPU:** @@ -343,7 +343,7 @@ Spark NLP 5.3.0 has been tested and is compatible with the following runtimes: ## EMR Support -Spark NLP 5.3.0 has been tested and is compatible with the following EMR releases: +Spark NLP 5.3.1 has been tested and is compatible with the following EMR releases: - emr-6.2.0 - emr-6.3.0 @@ -390,11 +390,11 @@ Spark NLP supports all major releases of Apache Spark 3.0.x, Apache Spark 3.1.x, ```sh # CPU -spark-shell --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 +spark-shell --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1 -pyspark --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 +pyspark --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1 -spark-submit --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 +spark-submit --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1 ``` The `spark-nlp` has been published to @@ -403,11 +403,11 @@ the [Maven Repository](https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/s ```sh # GPU -spark-shell --packages com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:5.3.0 +spark-shell --packages com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:5.3.1 -pyspark --packages com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:5.3.0 +pyspark --packages com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:5.3.1 -spark-submit --packages com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:5.3.0 +spark-submit --packages com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:5.3.1 ``` @@ -417,11 +417,11 @@ the [Maven Repository](https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/s ```sh # AArch64 -spark-shell --packages com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:5.3.0 +spark-shell --packages com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:5.3.1 -pyspark --packages com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:5.3.0 +pyspark --packages com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:5.3.1 -spark-submit --packages com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:5.3.0 +spark-submit --packages com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:5.3.1 ``` @@ -431,11 +431,11 @@ the [Maven Repository](https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/s ```sh # M1/M2 (Apple Silicon) -spark-shell --packages com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:5.3.0 +spark-shell --packages com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:5.3.1 -pyspark --packages com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:5.3.0 +pyspark --packages com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:5.3.1 -spark-submit --packages com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:5.3.0 +spark-submit --packages com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:5.3.1 ``` @@ -449,7 +449,7 @@ set in your SparkSession: spark-shell \ --driver-memory 16g \ --conf spark.kryoserializer.buffer.max=2000M \ - --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 + --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1 ``` ## Scala @@ -467,7 +467,7 @@ coordinates: com.johnsnowlabs.nlp spark-nlp_2.12 - 5.3.0 + 5.3.1 ``` @@ -478,7 +478,7 @@ coordinates: com.johnsnowlabs.nlp spark-nlp-gpu_2.12 - 5.3.0 + 5.3.1 ``` @@ -489,7 +489,7 @@ coordinates: com.johnsnowlabs.nlp spark-nlp-aarch64_2.12 - 5.3.0 + 5.3.1 ``` @@ -500,7 +500,7 @@ coordinates: com.johnsnowlabs.nlp spark-nlp-silicon_2.12 - 5.3.0 + 5.3.1 ``` @@ -510,28 +510,28 @@ coordinates: ```sbtshell // https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/spark-nlp -libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp" % "5.3.0" +libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp" % "5.3.1" ``` **spark-nlp-gpu:** ```sbtshell // https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/spark-nlp-gpu -libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-gpu" % "5.3.0" +libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-gpu" % "5.3.1" ``` **spark-nlp-aarch64:** ```sbtshell // https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/spark-nlp-aarch64 -libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-aarch64" % "5.3.0" +libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-aarch64" % "5.3.1" ``` **spark-nlp-silicon:** ```sbtshell // https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/spark-nlp-silicon -libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-silicon" % "5.3.0" +libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-silicon" % "5.3.1" ``` Maven @@ -553,7 +553,7 @@ If you installed pyspark through pip/conda, you can install `spark-nlp` through Pip: ```bash -pip install spark-nlp==5.3.0 +pip install spark-nlp==5.3.1 ``` Conda: @@ -582,7 +582,7 @@ spark = SparkSession.builder .config("spark.driver.memory", "16G") .config("spark.driver.maxResultSize", "0") .config("spark.kryoserializer.buffer.max", "2000M") - .config("spark.jars.packages", "com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0") + .config("spark.jars.packages", "com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1") .getOrCreate() ``` @@ -653,7 +653,7 @@ Use either one of the following options - Add the following Maven Coordinates to the interpreter's library list ```bash -com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 +com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1 ``` - Add a path to pre-built jar from [here](#compiled-jars) in the interpreter's library list making sure the jar is @@ -664,7 +664,7 @@ com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 Apart from the previous step, install the python module through pip ```bash -pip install spark-nlp==5.3.0 +pip install spark-nlp==5.3.1 ``` Or you can install `spark-nlp` from inside Zeppelin by using Conda: @@ -692,7 +692,7 @@ launch the Jupyter from the same Python environment: $ conda create -n sparknlp python=3.8 -y $ conda activate sparknlp # spark-nlp by default is based on pyspark 3.x -$ pip install spark-nlp==5.3.0 pyspark==3.3.1 jupyter +$ pip install spark-nlp==5.3.1 pyspark==3.3.1 jupyter $ jupyter notebook ``` @@ -709,7 +709,7 @@ export PYSPARK_PYTHON=python3 export PYSPARK_DRIVER_PYTHON=jupyter export PYSPARK_DRIVER_PYTHON_OPTS=notebook -pyspark --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 +pyspark --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1 ``` Alternatively, you can mix in using `--jars` option for pyspark + `pip install spark-nlp` @@ -736,7 +736,7 @@ This script comes with the two options to define `pyspark` and `spark-nlp` versi # -s is for spark-nlp # -g will enable upgrading libcudnn8 to 8.1.0 on Google Colab for GPU usage # by default they are set to the latest -!wget https://setup.johnsnowlabs.com/colab.sh -O - | bash /dev/stdin -p 3.2.3 -s 5.3.0 +!wget https://setup.johnsnowlabs.com/colab.sh -O - | bash /dev/stdin -p 3.2.3 -s 5.3.1 ``` [Spark NLP quick start on Google Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp/blob/master/examples/python/quick_start_google_colab.ipynb) @@ -759,7 +759,7 @@ This script comes with the two options to define `pyspark` and `spark-nlp` versi # -s is for spark-nlp # -g will enable upgrading libcudnn8 to 8.1.0 on Kaggle for GPU usage # by default they are set to the latest -!wget https://setup.johnsnowlabs.com/colab.sh -O - | bash /dev/stdin -p 3.2.3 -s 5.3.0 +!wget https://setup.johnsnowlabs.com/colab.sh -O - | bash /dev/stdin -p 3.2.3 -s 5.3.1 ``` [Spark NLP quick start on Kaggle Kernel](https://www.kaggle.com/mozzie/spark-nlp-named-entity-recognition) is a live @@ -778,9 +778,9 @@ demo on Kaggle Kernel that performs named entity recognitions by using Spark NLP 3. In `Libraries` tab inside your cluster you need to follow these steps: - 3.1. Install New -> PyPI -> `spark-nlp==5.3.0` -> Install + 3.1. Install New -> PyPI -> `spark-nlp==5.3.1` -> Install - 3.2. Install New -> Maven -> Coordinates -> `com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0` -> Install + 3.2. Install New -> Maven -> Coordinates -> `com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1` -> Install 4. Now you can attach your notebook to the cluster and use Spark NLP! @@ -831,7 +831,7 @@ A sample of your software configuration in JSON on S3 (must be public access): "spark.kryoserializer.buffer.max": "2000M", "spark.serializer": "org.apache.spark.serializer.KryoSerializer", "spark.driver.maxResultSize": "0", - "spark.jars.packages": "com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0" + "spark.jars.packages": "com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1" } }] ``` @@ -840,7 +840,7 @@ A sample of AWS CLI to launch EMR cluster: ```.sh aws emr create-cluster \ ---name "Spark NLP 5.3.0" \ +--name "Spark NLP 5.3.1" \ --release-label emr-6.2.0 \ --applications Name=Hadoop Name=Spark Name=Hive \ --instance-type m4.4xlarge \ @@ -904,7 +904,7 @@ gcloud dataproc clusters create ${CLUSTER_NAME} \ --enable-component-gateway \ --metadata 'PIP_PACKAGES=spark-nlp spark-nlp-display google-cloud-bigquery google-cloud-storage' \ --initialization-actions gs://goog-dataproc-initialization-actions-${REGION}/python/pip-install.sh \ - --properties spark:spark.serializer=org.apache.spark.serializer.KryoSerializer,spark:spark.driver.maxResultSize=0,spark:spark.kryoserializer.buffer.max=2000M,spark:spark.jars.packages=com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 + --properties spark:spark.serializer=org.apache.spark.serializer.KryoSerializer,spark:spark.driver.maxResultSize=0,spark:spark.kryoserializer.buffer.max=2000M,spark:spark.jars.packages=com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1 ``` 2. On an existing one, you need to install spark-nlp and spark-nlp-display packages from PyPI. @@ -947,7 +947,7 @@ spark = SparkSession.builder .config("spark.kryoserializer.buffer.max", "2000m") .config("spark.jsl.settings.pretrained.cache_folder", "sample_data/pretrained") .config("spark.jsl.settings.storage.cluster_tmp_dir", "sample_data/storage") - .config("spark.jars.packages", "com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0") + .config("spark.jars.packages", "com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1") .getOrCreate() ``` @@ -961,7 +961,7 @@ spark-shell \ --conf spark.kryoserializer.buffer.max=2000M \ --conf spark.jsl.settings.pretrained.cache_folder="sample_data/pretrained" \ --conf spark.jsl.settings.storage.cluster_tmp_dir="sample_data/storage" \ - --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 + --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1 ``` **pyspark:** @@ -974,7 +974,7 @@ pyspark \ --conf spark.kryoserializer.buffer.max=2000M \ --conf spark.jsl.settings.pretrained.cache_folder="sample_data/pretrained" \ --conf spark.jsl.settings.storage.cluster_tmp_dir="sample_data/storage" \ - --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 + --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1 ``` **Databricks:** @@ -1246,7 +1246,7 @@ spark = SparkSession.builder .config("spark.driver.memory", "16G") .config("spark.driver.maxResultSize", "0") .config("spark.kryoserializer.buffer.max", "2000M") - .config("spark.jars", "/tmp/spark-nlp-assembly-5.3.0.jar") + .config("spark.jars", "/tmp/spark-nlp-assembly-5.3.1.jar") .getOrCreate() ``` @@ -1255,7 +1255,7 @@ spark = SparkSession.builder version (3.0.x, 3.1.x, 3.2.x, 3.3.x, 3.4.x, and 3.5.x) - If you are local, you can load the Fat JAR from your local FileSystem, however, if you are in a cluster setup you need to put the Fat JAR on a distributed FileSystem such as HDFS, DBFS, S3, etc. ( - i.e., `hdfs:///tmp/spark-nlp-assembly-5.3.0.jar`) + i.e., `hdfs:///tmp/spark-nlp-assembly-5.3.1.jar`) Example of using pretrained Models and Pipelines in offline: diff --git a/docs/_layouts/landing.html b/docs/_layouts/landing.html index 93316984ba5c..5b7438dde12b 100755 --- a/docs/_layouts/landing.html +++ b/docs/_layouts/landing.html @@ -201,7 +201,7 @@

{{ _section.title }}

{% highlight bash %} # Using PyPI - $ pip install spark-nlp==5.3.0 + $ pip install spark-nlp==5.3.1 # Using Anaconda/Conda $ conda install -c johnsnowlabs spark-nlp diff --git a/docs/en/concepts.md b/docs/en/concepts.md index 4bbfe70b5b96..f82e9db6ba78 100644 --- a/docs/en/concepts.md +++ b/docs/en/concepts.md @@ -66,7 +66,7 @@ $ java -version $ conda create -n sparknlp python=3.7 -y $ conda activate sparknlp # spark-nlp by default is based on pyspark 3.x -$ pip install spark-nlp==5.3.0 pyspark==3.3.1 jupyter +$ pip install spark-nlp==5.3.1 pyspark==3.3.1 jupyter $ jupyter notebook ``` diff --git a/docs/en/examples.md b/docs/en/examples.md index 0844f38efd71..8d353dd4dd4b 100644 --- a/docs/en/examples.md +++ b/docs/en/examples.md @@ -18,7 +18,7 @@ $ java -version # should be Java 8 (Oracle or OpenJDK) $ conda create -n sparknlp python=3.7 -y $ conda activate sparknlp -$ pip install spark-nlp==5.3.0 pyspark==3.3.1 +$ pip install spark-nlp==5.3.1 pyspark==3.3.1 ```
@@ -40,7 +40,7 @@ This script comes with the two options to define `pyspark` and `spark-nlp` versi # -p is for pyspark # -s is for spark-nlp # by default they are set to the latest -!bash colab.sh -p 3.2.3 -s 5.3.0 +!bash colab.sh -p 3.2.3 -s 5.3.1 ``` [Spark NLP quick start on Google Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp/blob/master/examples/python/quick_start_google_colab.ipynb) is a live demo on Google Colab that performs named entity recognitions and sentiment analysis by using Spark NLP pretrained pipelines. diff --git a/docs/en/hardware_acceleration.md b/docs/en/hardware_acceleration.md index 32f0beb92871..6ef366a587f1 100644 --- a/docs/en/hardware_acceleration.md +++ b/docs/en/hardware_acceleration.md @@ -49,7 +49,7 @@ Since the new Transformer models such as BERT for Word and Sentence embeddings a | DeBERTa Large | +477%(5.8x) | | Longformer Base | +52%(1.5x) | -Spark NLP 5.3.0 is built with TensorFlow 2.7.1 and the following NVIDIA® software are only required for GPU support: +Spark NLP 5.3.1 is built with TensorFlow 2.7.1 and the following NVIDIA® software are only required for GPU support: - NVIDIA® GPU drivers version 450.80.02 or higher - CUDA® Toolkit 11.2 diff --git a/docs/en/install.md b/docs/en/install.md index a962a1f607c9..16b3e9243906 100644 --- a/docs/en/install.md +++ b/docs/en/install.md @@ -17,22 +17,22 @@ sidebar: ```bash # Install Spark NLP from PyPI -pip install spark-nlp==5.3.0 +pip install spark-nlp==5.3.1 # Install Spark NLP from Anaconda/Conda conda install -c johnsnowlabs spark-nlp # Load Spark NLP with Spark Shell -spark-shell --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 +spark-shell --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1 # Load Spark NLP with PySpark -pyspark --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 +pyspark --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1 # Load Spark NLP with Spark Submit -spark-submit --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 +spark-submit --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1 # Load Spark NLP as external JAR after compiling and building Spark NLP by `sbt assembly` -spark-shell --jars spark-nlp-assembly-5.3.0.jar +spark-shell --jars spark-nlp-assembly-5.3.1.jar ```
@@ -55,7 +55,7 @@ $ java -version # should be Java 8 (Oracle or OpenJDK) $ conda create -n sparknlp python=3.8 -y $ conda activate sparknlp -$ pip install spark-nlp==5.3.0 pyspark==3.3.1 +$ pip install spark-nlp==5.3.1 pyspark==3.3.1 ``` Of course you will need to have jupyter installed in your system: @@ -83,7 +83,7 @@ spark = SparkSession.builder \ .config("spark.driver.memory","16G")\ .config("spark.driver.maxResultSize", "0") \ .config("spark.kryoserializer.buffer.max", "2000M")\ - .config("spark.jars.packages", "com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0")\ + .config("spark.jars.packages", "com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1")\ .getOrCreate() ``` @@ -100,7 +100,7 @@ spark = SparkSession.builder \ com.johnsnowlabs.nlp spark-nlp_2.12 - 5.3.0 + 5.3.1 ``` @@ -111,7 +111,7 @@ spark = SparkSession.builder \ com.johnsnowlabs.nlp spark-nlp-gpu_2.12 - 5.3.0 + 5.3.1 ``` @@ -122,7 +122,7 @@ spark = SparkSession.builder \ com.johnsnowlabs.nlp spark-nlp-silicon_2.12 - 5.3.0 + 5.3.1 ``` @@ -133,7 +133,7 @@ spark = SparkSession.builder \ com.johnsnowlabs.nlp spark-nlp-aarch64_2.12 - 5.3.0 + 5.3.1 ``` @@ -145,28 +145,28 @@ spark = SparkSession.builder \ ```scala // https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/spark-nlp -libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp" % "5.3.0" +libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp" % "5.3.1" ``` **spark-nlp-gpu:** ```scala // https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/spark-nlp-gpu -libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-gpu" % "5.3.0" +libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-gpu" % "5.3.1" ``` **spark-nlp-silicon:** ```scala // https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/spark-nlp-silicon -libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-silicon" % "5.3.0" +libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-silicon" % "5.3.1" ``` **spark-nlp-aarch64:** ```scala // https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/spark-nlp-aarch64 -libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-aarch64" % "5.3.0" +libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-aarch64" % "5.3.1" ``` Maven Central: [https://mvnrepository.com/artifact/com.johnsnowlabs.nlp](https://mvnrepository.com/artifact/com.johnsnowlabs.nlp) @@ -248,7 +248,7 @@ maven coordinates like these: com.johnsnowlabs.nlp spark-nlp-silicon_2.12 - 5.3.0 + 5.3.1 ``` @@ -256,7 +256,7 @@ or in case of sbt: ```scala // https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/spark-nlp -libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-silicon" % "5.3.0" +libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-silicon" % "5.3.1" ``` If everything went well, you can now start Spark NLP with the `m1` flag set to `true`: @@ -293,7 +293,7 @@ spark = sparknlp.start(apple_silicon=True) ## Installation for Linux Aarch64 Systems -Starting from version 5.3.0, Spark NLP supports Linux systems running on an aarch64 +Starting from version 5.3.1, Spark NLP supports Linux systems running on an aarch64 processor architecture. The necessary dependencies have been built on Ubuntu 16.04, so a recent system with an environment of at least that will be needed. @@ -341,7 +341,7 @@ This script comes with the two options to define `pyspark` and `spark-nlp` versi # -p is for pyspark # -s is for spark-nlp # by default they are set to the latest -!wget http://setup.johnsnowlabs.com/colab.sh -O - | bash /dev/stdin -p 3.2.3 -s 5.3.0 +!wget http://setup.johnsnowlabs.com/colab.sh -O - | bash /dev/stdin -p 3.2.3 -s 5.3.1 ``` [Spark NLP quick start on Google Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp/blob/master/examples/python/quick_start_google_colab.ipynb) is a live demo on Google Colab that performs named entity recognitions and sentiment analysis by using Spark NLP pretrained pipelines. @@ -363,7 +363,7 @@ Run the following code in Kaggle Kernel and start using spark-nlp right away. ## Databricks Support -Spark NLP 5.3.0 has been tested and is compatible with the following runtimes: +Spark NLP 5.3.1 has been tested and is compatible with the following runtimes: **CPU:** @@ -445,7 +445,7 @@ Spark NLP 5.3.0 has been tested and is compatible with the following runtimes: 3.1. Install New -> PyPI -> `spark-nlp` -> Install - 3.2. Install New -> Maven -> Coordinates -> `com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0` -> Install + 3.2. Install New -> Maven -> Coordinates -> `com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1` -> Install 4. Now you can attach your notebook to the cluster and use Spark NLP! @@ -465,7 +465,7 @@ Note: You can import these notebooks by using their URLs. ## EMR Support -Spark NLP 5.3.0 has been tested and is compatible with the following EMR releases: +Spark NLP 5.3.1 has been tested and is compatible with the following EMR releases: - emr-6.2.0 - emr-6.3.0 @@ -528,7 +528,7 @@ A sample of your software configuration in JSON on S3 (must be public access): "spark.kryoserializer.buffer.max": "2000M", "spark.serializer": "org.apache.spark.serializer.KryoSerializer", "spark.driver.maxResultSize": "0", - "spark.jars.packages": "com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0" + "spark.jars.packages": "com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1" } } ] @@ -538,7 +538,7 @@ A sample of AWS CLI to launch EMR cluster: ```sh aws emr create-cluster \ ---name "Spark NLP 5.3.0" \ +--name "Spark NLP 5.3.1" \ --release-label emr-6.2.0 \ --applications Name=Hadoop Name=Spark Name=Hive \ --instance-type m4.4xlarge \ @@ -803,7 +803,7 @@ We recommend using `conda` to manage your Python environment on Windows. Now you can use the downloaded binary by navigating to `%SPARK_HOME%\bin` and running -Either create a conda env for python 3.6, install *pyspark==3.3.1 spark-nlp numpy* and use Jupyter/python console, or in the same conda env you can go to spark bin for *pyspark --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0*. +Either create a conda env for python 3.6, install *pyspark==3.3.1 spark-nlp numpy* and use Jupyter/python console, or in the same conda env you can go to spark bin for *pyspark --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1*. @@ -831,12 +831,12 @@ spark = SparkSession.builder \ .config("spark.driver.memory","16G")\ .config("spark.driver.maxResultSize", "0") \ .config("spark.kryoserializer.buffer.max", "2000M")\ - .config("spark.jars", "/tmp/spark-nlp-assembly-5.3.0.jar")\ + .config("spark.jars", "/tmp/spark-nlp-assembly-5.3.1.jar")\ .getOrCreate() ``` - You can download provided Fat JARs from each [release notes](https://github.com/JohnSnowLabs/spark-nlp/releases), please pay attention to pick the one that suits your environment depending on the device (CPU/GPU) and Apache Spark version (3.x) -- If you are local, you can load the Fat JAR from your local FileSystem, however, if you are in a cluster setup you need to put the Fat JAR on a distributed FileSystem such as HDFS, DBFS, S3, etc. (i.e., `hdfs:///tmp/spark-nlp-assembly-5.3.0.jar`) +- If you are local, you can load the Fat JAR from your local FileSystem, however, if you are in a cluster setup you need to put the Fat JAR on a distributed FileSystem such as HDFS, DBFS, S3, etc. (i.e., `hdfs:///tmp/spark-nlp-assembly-5.3.1.jar`) Example of using pretrained Models and Pipelines in offline: diff --git a/docs/en/spark_nlp.md b/docs/en/spark_nlp.md index 58ae3ef27d02..a5b0495be60d 100644 --- a/docs/en/spark_nlp.md +++ b/docs/en/spark_nlp.md @@ -25,7 +25,7 @@ Spark NLP is built on top of **Apache Spark 3.x**. For using Spark NLP you need: **GPU (optional):** -Spark NLP 5.3.0 is built with TensorFlow 2.7.1 and the following NVIDIA® software are only required for GPU support: +Spark NLP 5.3.1 is built with TensorFlow 2.7.1 and the following NVIDIA® software are only required for GPU support: - NVIDIA® GPU drivers version 450.80.02 or higher - CUDA® Toolkit 11.2 diff --git a/python/README.md b/python/README.md index 5f4c9637cd89..dfff9ca155cf 100644 --- a/python/README.md +++ b/python/README.md @@ -165,7 +165,7 @@ To use Spark NLP you need the following requirements: **GPU (optional):** -Spark NLP 5.3.0 is built with ONNX 1.17.0 and TensorFlow 2.7.1 deep learning engines. The minimum following NVIDIA® software are only required for GPU support: +Spark NLP 5.3.1 is built with ONNX 1.17.0 and TensorFlow 2.7.1 deep learning engines. The minimum following NVIDIA® software are only required for GPU support: - NVIDIA® GPU drivers version 450.80.02 or higher - CUDA® Toolkit 11.2 @@ -181,7 +181,7 @@ $ java -version $ conda create -n sparknlp python=3.7 -y $ conda activate sparknlp # spark-nlp by default is based on pyspark 3.x -$ pip install spark-nlp==5.3.0 pyspark==3.3.1 +$ pip install spark-nlp==5.3.1 pyspark==3.3.1 ``` In Python console or Jupyter `Python3` kernel: @@ -226,7 +226,7 @@ For more examples, you can visit our dedicated [examples](https://github.com/Joh ## Apache Spark Support -Spark NLP *5.3.0* has been built on top of Apache Spark 3.4 while fully supports Apache Spark 3.0.x, 3.1.x, 3.2.x, 3.3.x, 3.4.x, and 3.5.x +Spark NLP *5.3.1* has been built on top of Apache Spark 3.4 while fully supports Apache Spark 3.0.x, 3.1.x, 3.2.x, 3.3.x, 3.4.x, and 3.5.x | Spark NLP | Apache Spark 3.5.x | Apache Spark 3.4.x | Apache Spark 3.3.x | Apache Spark 3.2.x | Apache Spark 3.1.x | Apache Spark 3.0.x | Apache Spark 2.4.x | Apache Spark 2.3.x | |-----------|--------------------|--------------------|--------------------|--------------------|--------------------|--------------------|--------------------|--------------------| @@ -270,7 +270,7 @@ Find out more about `Spark NLP` versions from our [release notes](https://github ## Databricks Support -Spark NLP 5.3.0 has been tested and is compatible with the following runtimes: +Spark NLP 5.3.1 has been tested and is compatible with the following runtimes: **CPU:** @@ -343,7 +343,7 @@ Spark NLP 5.3.0 has been tested and is compatible with the following runtimes: ## EMR Support -Spark NLP 5.3.0 has been tested and is compatible with the following EMR releases: +Spark NLP 5.3.1 has been tested and is compatible with the following EMR releases: - emr-6.2.0 - emr-6.3.0 @@ -393,11 +393,11 @@ Spark NLP supports all major releases of Apache Spark 3.0.x, Apache Spark 3.1.x, ```sh # CPU -spark-shell --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 +spark-shell --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1 -pyspark --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 +pyspark --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1 -spark-submit --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 +spark-submit --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1 ``` The `spark-nlp` has been published to @@ -406,11 +406,11 @@ the [Maven Repository](https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/s ```sh # GPU -spark-shell --packages com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:5.3.0 +spark-shell --packages com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:5.3.1 -pyspark --packages com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:5.3.0 +pyspark --packages com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:5.3.1 -spark-submit --packages com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:5.3.0 +spark-submit --packages com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:5.3.1 ``` @@ -420,11 +420,11 @@ the [Maven Repository](https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/s ```sh # AArch64 -spark-shell --packages com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:5.3.0 +spark-shell --packages com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:5.3.1 -pyspark --packages com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:5.3.0 +pyspark --packages com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:5.3.1 -spark-submit --packages com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:5.3.0 +spark-submit --packages com.johnsnowlabs.nlp:spark-nlp-aarch64_2.12:5.3.1 ``` @@ -434,11 +434,11 @@ the [Maven Repository](https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/s ```sh # M1/M2 (Apple Silicon) -spark-shell --packages com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:5.3.0 +spark-shell --packages com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:5.3.1 -pyspark --packages com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:5.3.0 +pyspark --packages com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:5.3.1 -spark-submit --packages com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:5.3.0 +spark-submit --packages com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:5.3.1 ``` @@ -452,7 +452,7 @@ set in your SparkSession: spark-shell \ --driver-memory 16g \ --conf spark.kryoserializer.buffer.max=2000M \ - --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 + --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1 ``` ## Scala @@ -470,7 +470,7 @@ coordinates: com.johnsnowlabs.nlp spark-nlp_2.12 - 5.3.0 + 5.3.1 ``` @@ -481,7 +481,7 @@ coordinates: com.johnsnowlabs.nlp spark-nlp-gpu_2.12 - 5.3.0 + 5.3.1 ``` @@ -492,7 +492,7 @@ coordinates: com.johnsnowlabs.nlp spark-nlp-aarch64_2.12 - 5.3.0 + 5.3.1 ``` @@ -503,7 +503,7 @@ coordinates: com.johnsnowlabs.nlp spark-nlp-silicon_2.12 - 5.3.0 + 5.3.1 ``` @@ -513,28 +513,28 @@ coordinates: ```sbtshell // https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/spark-nlp -libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp" % "5.3.0" +libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp" % "5.3.1" ``` **spark-nlp-gpu:** ```sbtshell // https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/spark-nlp-gpu -libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-gpu" % "5.3.0" +libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-gpu" % "5.3.1" ``` **spark-nlp-aarch64:** ```sbtshell // https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/spark-nlp-aarch64 -libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-aarch64" % "5.3.0" +libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-aarch64" % "5.3.1" ``` **spark-nlp-silicon:** ```sbtshell // https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/spark-nlp-silicon -libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-silicon" % "5.3.0" +libraryDependencies += "com.johnsnowlabs.nlp" %% "spark-nlp-silicon" % "5.3.1" ``` Maven @@ -556,7 +556,7 @@ If you installed pyspark through pip/conda, you can install `spark-nlp` through Pip: ```bash -pip install spark-nlp==5.3.0 +pip install spark-nlp==5.3.1 ``` Conda: @@ -585,7 +585,7 @@ spark = SparkSession.builder .config("spark.driver.memory", "16G") .config("spark.driver.maxResultSize", "0") .config("spark.kryoserializer.buffer.max", "2000M") - .config("spark.jars.packages", "com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0") + .config("spark.jars.packages", "com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1") .getOrCreate() ``` @@ -656,7 +656,7 @@ Use either one of the following options - Add the following Maven Coordinates to the interpreter's library list ```bash -com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 +com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1 ``` - Add a path to pre-built jar from [here](#compiled-jars) in the interpreter's library list making sure the jar is @@ -667,7 +667,7 @@ com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 Apart from the previous step, install the python module through pip ```bash -pip install spark-nlp==5.3.0 +pip install spark-nlp==5.3.1 ``` Or you can install `spark-nlp` from inside Zeppelin by using Conda: @@ -695,7 +695,7 @@ launch the Jupyter from the same Python environment: $ conda create -n sparknlp python=3.8 -y $ conda activate sparknlp # spark-nlp by default is based on pyspark 3.x -$ pip install spark-nlp==5.3.0 pyspark==3.3.1 jupyter +$ pip install spark-nlp==5.3.1 pyspark==3.3.1 jupyter $ jupyter notebook ``` @@ -712,7 +712,7 @@ export PYSPARK_PYTHON=python3 export PYSPARK_DRIVER_PYTHON=jupyter export PYSPARK_DRIVER_PYTHON_OPTS=notebook -pyspark --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 +pyspark --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1 ``` Alternatively, you can mix in using `--jars` option for pyspark + `pip install spark-nlp` @@ -739,7 +739,7 @@ This script comes with the two options to define `pyspark` and `spark-nlp` versi # -s is for spark-nlp # -g will enable upgrading libcudnn8 to 8.1.0 on Google Colab for GPU usage # by default they are set to the latest -!wget https://setup.johnsnowlabs.com/colab.sh -O - | bash /dev/stdin -p 3.2.3 -s 5.3.0 +!wget https://setup.johnsnowlabs.com/colab.sh -O - | bash /dev/stdin -p 3.2.3 -s 5.3.1 ``` [Spark NLP quick start on Google Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp/blob/master/examples/python/quick_start_google_colab.ipynb) @@ -762,7 +762,7 @@ This script comes with the two options to define `pyspark` and `spark-nlp` versi # -s is for spark-nlp # -g will enable upgrading libcudnn8 to 8.1.0 on Kaggle for GPU usage # by default they are set to the latest -!wget https://setup.johnsnowlabs.com/colab.sh -O - | bash /dev/stdin -p 3.2.3 -s 5.3.0 +!wget https://setup.johnsnowlabs.com/colab.sh -O - | bash /dev/stdin -p 3.2.3 -s 5.3.1 ``` [Spark NLP quick start on Kaggle Kernel](https://www.kaggle.com/mozzie/spark-nlp-named-entity-recognition) is a live @@ -781,9 +781,9 @@ demo on Kaggle Kernel that performs named entity recognitions by using Spark NLP 3. In `Libraries` tab inside your cluster you need to follow these steps: - 3.1. Install New -> PyPI -> `spark-nlp==5.3.0` -> Install + 3.1. Install New -> PyPI -> `spark-nlp==5.3.1` -> Install - 3.2. Install New -> Maven -> Coordinates -> `com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0` -> Install + 3.2. Install New -> Maven -> Coordinates -> `com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1` -> Install 4. Now you can attach your notebook to the cluster and use Spark NLP! @@ -834,7 +834,7 @@ A sample of your software configuration in JSON on S3 (must be public access): "spark.kryoserializer.buffer.max": "2000M", "spark.serializer": "org.apache.spark.serializer.KryoSerializer", "spark.driver.maxResultSize": "0", - "spark.jars.packages": "com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0" + "spark.jars.packages": "com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1" } }] ``` @@ -843,7 +843,7 @@ A sample of AWS CLI to launch EMR cluster: ```.sh aws emr create-cluster \ ---name "Spark NLP 5.3.0" \ +--name "Spark NLP 5.3.1" \ --release-label emr-6.2.0 \ --applications Name=Hadoop Name=Spark Name=Hive \ --instance-type m4.4xlarge \ @@ -907,7 +907,7 @@ gcloud dataproc clusters create ${CLUSTER_NAME} \ --enable-component-gateway \ --metadata 'PIP_PACKAGES=spark-nlp spark-nlp-display google-cloud-bigquery google-cloud-storage' \ --initialization-actions gs://goog-dataproc-initialization-actions-${REGION}/python/pip-install.sh \ - --properties spark:spark.serializer=org.apache.spark.serializer.KryoSerializer,spark:spark.driver.maxResultSize=0,spark:spark.kryoserializer.buffer.max=2000M,spark:spark.jars.packages=com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 + --properties spark:spark.serializer=org.apache.spark.serializer.KryoSerializer,spark:spark.driver.maxResultSize=0,spark:spark.kryoserializer.buffer.max=2000M,spark:spark.jars.packages=com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1 ``` 2. On an existing one, you need to install spark-nlp and spark-nlp-display packages from PyPI. @@ -950,7 +950,7 @@ spark = SparkSession.builder .config("spark.kryoserializer.buffer.max", "2000m") .config("spark.jsl.settings.pretrained.cache_folder", "sample_data/pretrained") .config("spark.jsl.settings.storage.cluster_tmp_dir", "sample_data/storage") - .config("spark.jars.packages", "com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0") + .config("spark.jars.packages", "com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1") .getOrCreate() ``` @@ -964,7 +964,7 @@ spark-shell \ --conf spark.kryoserializer.buffer.max=2000M \ --conf spark.jsl.settings.pretrained.cache_folder="sample_data/pretrained" \ --conf spark.jsl.settings.storage.cluster_tmp_dir="sample_data/storage" \ - --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 + --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1 ``` **pyspark:** @@ -977,7 +977,7 @@ pyspark \ --conf spark.kryoserializer.buffer.max=2000M \ --conf spark.jsl.settings.pretrained.cache_folder="sample_data/pretrained" \ --conf spark.jsl.settings.storage.cluster_tmp_dir="sample_data/storage" \ - --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.0 + --packages com.johnsnowlabs.nlp:spark-nlp_2.12:5.3.1 ``` **Databricks:** @@ -1249,7 +1249,7 @@ spark = SparkSession.builder .config("spark.driver.memory", "16G") .config("spark.driver.maxResultSize", "0") .config("spark.kryoserializer.buffer.max", "2000M") - .config("spark.jars", "/tmp/spark-nlp-assembly-5.3.0.jar") + .config("spark.jars", "/tmp/spark-nlp-assembly-5.3.1.jar") .getOrCreate() ``` @@ -1258,7 +1258,7 @@ spark = SparkSession.builder version (3.0.x, 3.1.x, 3.2.x, 3.3.x, 3.4.x, and 3.5.x) - If you are local, you can load the Fat JAR from your local FileSystem, however, if you are in a cluster setup you need to put the Fat JAR on a distributed FileSystem such as HDFS, DBFS, S3, etc. ( - i.e., `hdfs:///tmp/spark-nlp-assembly-5.3.0.jar`) + i.e., `hdfs:///tmp/spark-nlp-assembly-5.3.1.jar`) Example of using pretrained Models and Pipelines in offline: diff --git a/python/docs/conf.py b/python/docs/conf.py index 6485fffbdb93..26f207e80de5 100644 --- a/python/docs/conf.py +++ b/python/docs/conf.py @@ -23,7 +23,7 @@ author = "John Snow Labs" # The full version, including alpha/beta/rc tags -release = "5.3.0" +release = "5.3.1" pyspark_version = "3.2.3" # -- General configuration --------------------------------------------------- diff --git a/python/setup.py b/python/setup.py index 116c82123e7f..803c6c9de1dc 100644 --- a/python/setup.py +++ b/python/setup.py @@ -41,7 +41,7 @@ # project code, see # https://packaging.python.org/en/latest/single_source_version.html - version='5.3.0', # Required + version='5.3.1', # Required # This is a one-line description or tagline of what your project does. This # corresponds to the 'Summary' metadata field: diff --git a/python/sparknlp/__init__.py b/python/sparknlp/__init__.py index 4964ae2bfcdb..d84cb357f722 100644 --- a/python/sparknlp/__init__.py +++ b/python/sparknlp/__init__.py @@ -128,7 +128,7 @@ def start(gpu=False, The initiated Spark session. """ - current_version = "5.3.0" + current_version = "5.3.1" if params is None: params = {} @@ -309,4 +309,4 @@ def version(): str The current Spark NLP version. """ - return '5.3.0' + return '5.3.1' diff --git a/scripts/colab_setup.sh b/scripts/colab_setup.sh index 1a28ecbddb3d..260cefc3e8ff 100644 --- a/scripts/colab_setup.sh +++ b/scripts/colab_setup.sh @@ -1,7 +1,7 @@ #!/bin/bash #default values for pyspark, spark-nlp, and SPARK_HOME -SPARKNLP="5.3.0" +SPARKNLP="5.3.1" PYSPARK="3.2.3" while getopts s:p:g option diff --git a/scripts/kaggle_setup.sh b/scripts/kaggle_setup.sh index 5d7e3391d4ab..936333827a1e 100644 --- a/scripts/kaggle_setup.sh +++ b/scripts/kaggle_setup.sh @@ -1,7 +1,7 @@ #!/bin/bash #default values for pyspark, spark-nlp, and SPARK_HOME -SPARKNLP="5.3.0" +SPARKNLP="5.3.1" PYSPARK="3.2.3" while getopts s:p:g option diff --git a/scripts/sagemaker_setup.sh b/scripts/sagemaker_setup.sh index feb7b6a7463d..706294e69191 100644 --- a/scripts/sagemaker_setup.sh +++ b/scripts/sagemaker_setup.sh @@ -1,7 +1,7 @@ #!/bin/bash # Default values for pyspark, spark-nlp, and SPARK_HOME -SPARKNLP="5.3.0" +SPARKNLP="5.3.1" PYSPARK="3.2.3" echo "Setup SageMaker for PySpark $PYSPARK and Spark NLP $SPARKNLP" diff --git a/src/main/scala/com/johnsnowlabs/nlp/SparkNLP.scala b/src/main/scala/com/johnsnowlabs/nlp/SparkNLP.scala index ef55ded819e3..e5feccda95af 100644 --- a/src/main/scala/com/johnsnowlabs/nlp/SparkNLP.scala +++ b/src/main/scala/com/johnsnowlabs/nlp/SparkNLP.scala @@ -20,7 +20,7 @@ import org.apache.spark.sql.SparkSession object SparkNLP { - val currentVersion = "5.3.0" + val currentVersion = "5.3.1" val MavenSpark3 = s"com.johnsnowlabs.nlp:spark-nlp_2.12:$currentVersion" val MavenGpuSpark3 = s"com.johnsnowlabs.nlp:spark-nlp-gpu_2.12:$currentVersion" val MavenSparkSilicon = s"com.johnsnowlabs.nlp:spark-nlp-silicon_2.12:$currentVersion" diff --git a/src/main/scala/com/johnsnowlabs/util/Build.scala b/src/main/scala/com/johnsnowlabs/util/Build.scala index 6a942e7a407c..7ddd708aa9a5 100644 --- a/src/main/scala/com/johnsnowlabs/util/Build.scala +++ b/src/main/scala/com/johnsnowlabs/util/Build.scala @@ -17,5 +17,5 @@ package com.johnsnowlabs.util object Build { - val version: String = "5.3.0" + val version: String = "5.3.1" } From 76c38dbf0591b4597e1d3239497a8482d7dc084b Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 4 Mar 2024 08:43:55 +0000 Subject: [PATCH 6/7] Update Scala and Python APIs --- docs/api/com/index.html | 8 +- .../com/johnsnowlabs/client/CloudClient.html | 8 +- .../com/johnsnowlabs/client/CloudManager.html | 8 +- .../johnsnowlabs/client/CloudResources$.html | 8 +- .../com/johnsnowlabs/client/CloudStorage.html | 8 +- .../client/aws/AWSAnonymousCredentials.html | 8 +- .../client/aws/AWSBasicCredentials.html | 8 +- .../johnsnowlabs/client/aws/AWSClient.html | 8 +- .../client/aws/AWSCredentialsProvider.html | 8 +- .../johnsnowlabs/client/aws/AWSGateway.html | 8 +- .../client/aws/AWSProfileCredentials.html | 8 +- .../client/aws/AWSTokenCredentials.html | 8 +- .../client/aws/CredentialParams.html | 8 +- .../johnsnowlabs/client/aws/Credentials.html | 8 +- .../com/johnsnowlabs/client/aws/index.html | 8 +- .../client/azure/AzureClient.html | 8 +- .../client/azure/AzureGateway.html | 8 +- .../com/johnsnowlabs/client/azure/index.html | 8 +- .../johnsnowlabs/client/gcp/GCPClient.html | 8 +- .../johnsnowlabs/client/gcp/GCPGateway.html | 8 +- .../com/johnsnowlabs/client/gcp/index.html | 8 +- docs/api/com/johnsnowlabs/client/index.html | 8 +- .../client/util/CloudHelper$.html | 8 +- .../com/johnsnowlabs/client/util/index.html | 8 +- .../johnsnowlabs/collections/SearchTrie$.html | 8 +- .../johnsnowlabs/collections/SearchTrie.html | 8 +- .../collections/StorageSearchTrie$.html | 8 +- .../collections/StorageSearchTrie.html | 8 +- .../com/johnsnowlabs/collections/index.html | 8 +- docs/api/com/johnsnowlabs/index.html | 8 +- docs/api/com/johnsnowlabs/ml/ai/DeBerta.html | 8 +- .../ml/ai/MergeTokenStrategy$.html | 8 +- .../johnsnowlabs/ml/ai/OpenAICompletion.html | 8 +- .../johnsnowlabs/ml/ai/OpenAIEmbeddings.html | 8 +- docs/api/com/johnsnowlabs/ml/ai/index.html | 8 +- .../com/johnsnowlabs/ml/ai/model/Choice.html | 8 +- .../ml/ai/model/CompletionResponse.html | 8 +- .../ml/ai/model/EmbeddingData.html | 8 +- .../ml/ai/model/TextEmbeddingResponse.html | 8 +- .../com/johnsnowlabs/ml/ai/model/Usage.html | 8 +- .../johnsnowlabs/ml/ai/model/UsageData.html | 8 +- .../com/johnsnowlabs/ml/ai/model/index.html | 8 +- .../ml/ai/seq2seq/DecoderProcessor.html | 8 +- .../ml/ai/seq2seq/OnnxT5EncoderDecoder.html | 8 +- .../ml/ai/seq2seq/T5EncoderDecoder.html | 8 +- .../com/johnsnowlabs/ml/ai/seq2seq/index.html | 8 +- .../ml/ai/t5/OnnxT5EncoderDecoder.html | 8 +- .../t5/T5EncoderDecoder$DecoderProcessor.html | 8 +- .../ml/ai/t5/T5EncoderDecoder.html | 8 +- docs/api/com/johnsnowlabs/ml/ai/t5/index.html | 8 +- .../ml/ai/util/Generation/Generate.html | 8 +- .../ai/util/Generation/GenerationConfig.html | 8 +- .../ml/ai/util/Generation/Logit/Logit.html | 8 +- .../ForcedTokenLogitProcessor.html | 8 +- .../Logit/LogitProcess/LogitProcessor.html | 8 +- .../LogitProcess/MinLengthLogitProcessor.html | 8 +- .../NoRepeatNgramsLogitProcessor.html | 8 +- .../RepetitionPenaltyLogitProcessor.html | 8 +- .../LogitProcess/SuppressLogitProcessor.html | 8 +- .../Generation/Logit/LogitProcess/index.html | 8 +- .../Generation/Logit/LogitProcessorList.html | 8 +- .../Logit/LogitWarper/LogitWarper.html | 8 +- .../LogitWarper/TemperatureLogitWarper.html | 8 +- .../Logit/LogitWarper/TopKLogitWarper.html | 8 +- .../Logit/LogitWarper/TopPLogitWarper.html | 8 +- .../Generation/Logit/LogitWarper/index.html | 8 +- .../ml/ai/util/Generation/Logit/index.html | 8 +- .../Generation/Search/BeamHypotheses.html | 8 +- .../ai/util/Generation/Search/BeamScorer.html | 8 +- .../Generation/Search/BeamSearchScorer.html | 8 +- .../ml/ai/util/Generation/Search/index.html | 8 +- .../ml/ai/util/Generation/index.html | 8 +- .../com/johnsnowlabs/ml/ai/util/index.html | 8 +- docs/api/com/johnsnowlabs/ml/crf/Attr.html | 8 +- .../com/johnsnowlabs/ml/crf/AttrFeature.html | 8 +- .../api/com/johnsnowlabs/ml/crf/AttrStat.html | 8 +- .../com/johnsnowlabs/ml/crf/CrfDataset.html | 8 +- .../com/johnsnowlabs/ml/crf/CrfParams.html | 8 +- .../johnsnowlabs/ml/crf/DatasetEncoder.html | 8 +- .../johnsnowlabs/ml/crf/DatasetMetadata.html | 8 +- .../johnsnowlabs/ml/crf/DatasetReader$.html | 8 +- .../johnsnowlabs/ml/crf/EdgeCalculator$.html | 8 +- .../com/johnsnowlabs/ml/crf/FbCalculator.html | 8 +- .../api/com/johnsnowlabs/ml/crf/Instance.html | 8 +- .../johnsnowlabs/ml/crf/InstanceLabels.html | 8 +- .../johnsnowlabs/ml/crf/L2DecayStrategy.html | 8 +- .../johnsnowlabs/ml/crf/LinearChainCrf.html | 8 +- .../ml/crf/LinearChainCrfModel.html | 8 +- .../ml/crf/SerializedDatasetMetadata.html | 8 +- .../ml/crf/SerializedLinearChainCrfModel.html | 8 +- .../ml/crf/SparseArray$$SeqWrapper.html | 8 +- .../com/johnsnowlabs/ml/crf/SparseArray$.html | 8 +- .../com/johnsnowlabs/ml/crf/SparseArray.html | 8 +- .../ml/crf/TextSentenceAttrs.html | 8 +- .../ml/crf/TextSentenceLabels.html | 8 +- .../com/johnsnowlabs/ml/crf/Transition.html | 8 +- .../com/johnsnowlabs/ml/crf/VectorMath$.html | 8 +- .../com/johnsnowlabs/ml/crf/WordAttrs.html | 8 +- docs/api/com/johnsnowlabs/ml/crf/index.html | 8 +- docs/api/com/johnsnowlabs/ml/index.html | 8 +- .../com/johnsnowlabs/ml/onnx/OnnxSession.html | 8 +- .../ml/onnx/OnnxWrapper$$DecoderWrappers.html | 8 +- ...er$$EncoderDecoderWithoutPastWrappers.html | 8 +- .../OnnxWrapper$$EncoderDecoderWrappers.html | 8 +- .../johnsnowlabs/ml/onnx/OnnxWrapper$.html | 8 +- .../com/johnsnowlabs/ml/onnx/OnnxWrapper.html | 8 +- .../johnsnowlabs/ml/onnx/ReadOnnxModel.html | 10 +-- ...sources$$implicits$$OnnxSessionResult.html | 8 +- .../ml/onnx/TensorResources$$implicits$.html | 8 +- .../ml/onnx/TensorResources$.html | 8 +- .../johnsnowlabs/ml/onnx/TensorResources.html | 8 +- .../johnsnowlabs/ml/onnx/WriteOnnxModel.html | 8 +- docs/api/com/johnsnowlabs/ml/onnx/index.html | 8 +- .../tensorflow/ClassifierDatasetEncoder.html | 8 +- .../ClassifierDatasetEncoderParams.html | 8 +- .../ml/tensorflow/DatasetEncoderParams.html | 8 +- .../johnsnowlabs/ml/tensorflow/Logging.html | 8 +- .../ml/tensorflow/ModelSignature.html | 8 +- .../johnsnowlabs/ml/tensorflow/NerBatch$.html | 8 +- .../johnsnowlabs/ml/tensorflow/NerBatch.html | 8 +- .../ml/tensorflow/NerDatasetEncoder.html | 8 +- .../ml/tensorflow/ReadTensorflowModel.html | 8 +- .../ml/tensorflow/SentenceGrouper.html | 8 +- .../ml/tensorflow/TensorResources$.html | 8 +- .../ml/tensorflow/TensorResources.html | 8 +- .../ml/tensorflow/TensorflowClassifier.html | 8 +- .../ml/tensorflow/TensorflowWrapper$.html | 8 +- .../ml/tensorflow/TensorflowWrapper.html | 8 +- .../johnsnowlabs/ml/tensorflow/Variables.html | 8 +- .../ml/tensorflow/WriteTensorflowModel.html | 8 +- .../com/johnsnowlabs/ml/tensorflow/index.html | 8 +- .../sentencepiece/ReadSentencePieceModel.html | 8 +- .../sentencepiece/SentencePieceException.html | 8 +- .../sentencepiece/SentencePieceProcessor.html | 8 +- .../sentencepiece/SentencePieceWrapper$.html | 8 +- .../WriteSentencePieceModel.html | 8 +- .../ml/tensorflow/sentencepiece/index.html | 8 +- ...delSignatureConstants$$AttentionMask$.html | 8 +- ...lSignatureConstants$$AttentionMaskV1$.html | 8 +- ...SignatureConstants$$AudioValuesInput$.html | 8 +- ...s$$CachedDecoderEncoderAttentionMask$.html | 8 +- ...stants$$CachedDecoderEncoderInputIds$.html | 8 +- ...eConstants$$CachedDecoderInputCache1$.html | 8 +- ...eConstants$$CachedDecoderInputCache2$.html | 8 +- ...tureConstants$$CachedDecoderInputIds$.html | 8 +- ...natureConstants$$CachedEncoderOutput$.html | 8 +- ...gnatureConstants$$CachedLogitsOutput$.html | 8 +- ...delSignatureConstants$$CachedOutPut2$.html | 8 +- ...delSignatureConstants$$CachedOutput1$.html | 8 +- .../sign/ModelSignatureConstants$$DType$.html | 8 +- ...atureConstants$$DecoderAttentionMask$.html | 8 +- ...ureConstants$$DecoderCachedCache1Key$.html | 8 +- ...ureConstants$$DecoderCachedCache2Key$.html | 8 +- ...ts$$DecoderCachedEncoderAttentionKey$.html | 8 +- ...stants$$DecoderCachedEncoderStateKey$.html | 8 +- ...eConstants$$DecoderCachedInputIdsKey$.html | 8 +- ...natureConstants$$DecoderCachedOutput$.html | 8 +- ...stants$$DecoderCachedOutputCache1Key$.html | 8 +- ...stants$$DecoderCachedOutputCache2Key$.html | 8 +- ...ureConstants$$DecoderCachedOutputKey$.html | 8 +- ...nstants$$DecoderEncoderAttentionMask$.html | 8 +- ...ureConstants$$DecoderEncoderInputIds$.html | 8 +- ...onstants$$DecoderInitOutputCache1Key$.html | 8 +- ...onstants$$DecoderInitOutputCache2Key$.html | 8 +- ...lSignatureConstants$$DecoderInputIds$.html | 8 +- ...delSignatureConstants$$DecoderOutput$.html | 8 +- .../ModelSignatureConstants$$DimCount$.html | 8 +- ...atureConstants$$EncoderAttentionMask$.html | 8 +- ...gnatureConstants$$EncoderContextMask$.html | 8 +- ...lSignatureConstants$$EncoderInputIds$.html | 8 +- ...delSignatureConstants$$EncoderOutput$.html | 8 +- ...lSignatureConstants$$EndLogitsOutput$.html | 8 +- ...ignatureConstants$$InitCachedOutPut2$.html | 8 +- ...ignatureConstants$$InitCachedOutput1$.html | 8 +- ...nts$$InitDecoderEncoderAttentionMask$.html | 8 +- ...onstants$$InitDecoderEncoderInputIds$.html | 8 +- ...natureConstants$$InitDecoderInputIds$.html | 8 +- ...SignatureConstants$$InitLogitsOutput$.html | 8 +- .../ModelSignatureConstants$$InputIds$.html | 8 +- .../ModelSignatureConstants$$InputIdsV1$.html | 8 +- ...lSignatureConstants$$LastHiddenState$.html | 8 +- ...ignatureConstants$$LastHiddenStateV1$.html | 8 +- ...odelSignatureConstants$$LogitsOutput$.html | 8 +- .../sign/ModelSignatureConstants$$Name$.html | 8 +- ...SignatureConstants$$PixelValuesInput$.html | 8 +- ...odelSignatureConstants$$PoolerOutput$.html | 8 +- ...elSignatureConstants$$PoolerOutputV1$.html | 8 +- ...elSignatureConstants$$SerializedSize$.html | 8 +- ...odelSignatureConstants$$ShapeDimList$.html | 8 +- ...ignatureConstants$$StartLogitsOutput$.html | 8 +- ...lSignatureConstants$$TFInfoDescriptor.html | 8 +- ...lSignatureConstants$$TFInfoNameMapper.html | 8 +- ...stants$$TapasLogitsAggregationOutput$.html | 8 +- ...ignatureConstants$$TapasLogitsOutput$.html | 8 +- ...odelSignatureConstants$$TokenTypeIds$.html | 8 +- ...elSignatureConstants$$TokenTypeIdsV1$.html | 8 +- .../sign/ModelSignatureConstants$.html | 8 +- .../sign/ModelSignatureManager$.html | 8 +- .../ml/tensorflow/sign/index.html | 8 +- ...inAlg$$implicits$$ExtendedDenseMatrix.html | 8 +- .../ml/util/LinAlg$$implicits$.html | 8 +- .../api/com/johnsnowlabs/ml/util/LinAlg$.html | 8 +- .../ml/util/LoadExternalModel$.html | 8 +- .../com/johnsnowlabs/ml/util/ModelArch$.html | 8 +- .../com/johnsnowlabs/ml/util/ModelEngine.html | 8 +- docs/api/com/johnsnowlabs/ml/util/ONNX$.html | 8 +- .../com/johnsnowlabs/ml/util/PyTorch$.html | 8 +- .../com/johnsnowlabs/ml/util/TensorFlow$.html | 8 +- .../com/johnsnowlabs/ml/util/Unknown$.html | 8 +- docs/api/com/johnsnowlabs/ml/util/index.html | 8 +- .../johnsnowlabs/nlp/ActivationFunction$.html | 8 +- .../nlp/Annotation$$AnnotationContainer.html | 8 +- ...nnotation$$extractors$$AnnotationData.html | 8 +- .../nlp/Annotation$$extractors$.html | 8 +- .../api/com/johnsnowlabs/nlp/Annotation$.html | 8 +- docs/api/com/johnsnowlabs/nlp/Annotation.html | 8 +- .../AnnotationAudio$$AnnotationContainer.html | 8 +- .../nlp/AnnotationAudio$$AudioFields.html | 8 +- .../johnsnowlabs/nlp/AnnotationAudio$.html | 8 +- .../com/johnsnowlabs/nlp/AnnotationAudio.html | 8 +- .../AnnotationImage$$AnnotationContainer.html | 8 +- .../nlp/AnnotationImage$$ImageFields.html | 8 +- .../johnsnowlabs/nlp/AnnotationImage$.html | 8 +- .../com/johnsnowlabs/nlp/AnnotationImage.html | 8 +- .../johnsnowlabs/nlp/AnnotatorApproach.html | 8 +- .../com/johnsnowlabs/nlp/AnnotatorModel.html | 8 +- .../com/johnsnowlabs/nlp/AnnotatorType$.html | 8 +- .../com/johnsnowlabs/nlp/AudioAssembler$.html | 8 +- .../com/johnsnowlabs/nlp/AudioAssembler.html | 8 +- docs/api/com/johnsnowlabs/nlp/CanBeLazy.html | 8 +- docs/api/com/johnsnowlabs/nlp/Doc2Chunk$.html | 8 +- docs/api/com/johnsnowlabs/nlp/Doc2Chunk.html | 8 +- .../johnsnowlabs/nlp/DocumentAssembler$.html | 8 +- .../johnsnowlabs/nlp/DocumentAssembler.html | 8 +- .../johnsnowlabs/nlp/EmbeddingsFinisher$.html | 8 +- .../johnsnowlabs/nlp/EmbeddingsFinisher.html | 8 +- .../com/johnsnowlabs/nlp/FeaturesReader.html | 8 +- .../com/johnsnowlabs/nlp/FeaturesWriter.html | 8 +- docs/api/com/johnsnowlabs/nlp/Finisher$.html | 8 +- docs/api/com/johnsnowlabs/nlp/Finisher.html | 8 +- .../com/johnsnowlabs/nlp/GraphFinisher.html | 8 +- .../nlp/HasAudioFeatureProperties.html | 8 +- .../johnsnowlabs/nlp/HasBatchedAnnotate.html | 8 +- .../nlp/HasBatchedAnnotateAudio.html | 8 +- .../nlp/HasBatchedAnnotateImage.html | 8 +- .../nlp/HasCandidateLabelsProperties.html | 8 +- .../nlp/HasCaseSensitiveProperties.html | 8 +- .../HasClassifierActivationProperties.html | 8 +- .../nlp/HasEnableCachingProperties.html | 8 +- docs/api/com/johnsnowlabs/nlp/HasEngine.html | 8 +- .../api/com/johnsnowlabs/nlp/HasFeatures.html | 8 +- .../nlp/HasGeneratorProperties.html | 8 +- .../nlp/HasImageFeatureProperties.html | 8 +- .../nlp/HasInputAnnotationCols.html | 8 +- .../nlp/HasMultipleInputAnnotationCols.html | 8 +- .../nlp/HasOutputAnnotationCol.html | 8 +- .../nlp/HasOutputAnnotatorType.html | 8 +- .../com/johnsnowlabs/nlp/HasPretrained.html | 8 +- .../HasProtectedParams$ProtectedParam.html | 8 +- .../johnsnowlabs/nlp/HasProtectedParams.html | 8 +- .../com/johnsnowlabs/nlp/HasRecursiveFit.html | 8 +- .../nlp/HasRecursiveTransform.html | 8 +- .../johnsnowlabs/nlp/HasSimpleAnnotate.html | 8 +- .../api/com/johnsnowlabs/nlp/IAnnotation.html | 8 +- .../com/johnsnowlabs/nlp/ImageAssembler$.html | 8 +- .../com/johnsnowlabs/nlp/ImageAssembler.html | 8 +- .../com/johnsnowlabs/nlp/JavaAnnotation.html | 8 +- .../com/johnsnowlabs/nlp/LightPipeline.html | 8 +- .../nlp/MultiDocumentAssembler$.html | 8 +- .../nlp/MultiDocumentAssembler.html | 8 +- .../nlp/ParamsAndFeaturesReadable.html | 8 +- .../nlp/ParamsAndFeaturesWritable.html | 8 +- .../com/johnsnowlabs/nlp/RawAnnotator.html | 8 +- .../johnsnowlabs/nlp/RecursivePipeline.html | 8 +- .../nlp/RecursivePipelineModel.html | 8 +- docs/api/com/johnsnowlabs/nlp/SparkNLP$.html | 8 +- .../com/johnsnowlabs/nlp/TableAssembler$.html | 8 +- .../com/johnsnowlabs/nlp/TableAssembler.html | 8 +- .../com/johnsnowlabs/nlp/TokenAssembler$.html | 8 +- .../com/johnsnowlabs/nlp/TokenAssembler.html | 8 +- .../nlp/annotators/Chunk2Doc$.html | 8 +- .../nlp/annotators/Chunk2Doc.html | 8 +- .../nlp/annotators/ChunkTokenizer$.html | 8 +- .../nlp/annotators/ChunkTokenizer.html | 8 +- .../nlp/annotators/ChunkTokenizerModel$.html | 8 +- .../nlp/annotators/ChunkTokenizerModel.html | 8 +- .../johnsnowlabs/nlp/annotators/Chunker$.html | 8 +- .../johnsnowlabs/nlp/annotators/Chunker.html | 8 +- .../nlp/annotators/Date2Chunk$.html | 8 +- .../nlp/annotators/Date2Chunk.html | 8 +- .../nlp/annotators/DateMatcher$.html | 8 +- .../nlp/annotators/DateMatcher.html | 8 +- .../nlp/annotators/DateMatcherTranslator.html | 8 +- .../DateMatcherTranslatorPolicy.html | 8 +- .../nlp/annotators/DateMatcherUtils.html | 8 +- .../DocumentCharacterTextSplitter$.html | 8 +- .../DocumentCharacterTextSplitter.html | 8 +- .../nlp/annotators/DocumentNormalizer$.html | 8 +- .../nlp/annotators/DocumentNormalizer.html | 8 +- .../annotators/DocumentTokenSplitter$.html | 8 +- .../nlp/annotators/DocumentTokenSplitter.html | 8 +- .../nlp/annotators/EnglishStemmer$.html | 8 +- .../nlp/annotators/GraphExtraction.html | 8 +- .../nlp/annotators/Lemmatizer$.html | 8 +- .../nlp/annotators/Lemmatizer.html | 8 +- .../nlp/annotators/LemmatizerModel$.html | 8 +- .../nlp/annotators/LemmatizerModel.html | 8 +- .../nlp/annotators/LookAroundManager$.html | 8 +- .../nlp/annotators/MultiDateMatcher$.html | 8 +- .../nlp/annotators/MultiDateMatcher.html | 8 +- .../nlp/annotators/MultiDatePolicy$.html | 8 +- .../nlp/annotators/NGramGenerator$.html | 8 +- .../nlp/annotators/NGramGenerator.html | 8 +- .../nlp/annotators/Normalizer$.html | 8 +- .../nlp/annotators/Normalizer.html | 8 +- .../nlp/annotators/NormalizerModel$.html | 8 +- ...alizerModel$TokenizerAndNormalizerMap.html | 8 +- .../nlp/annotators/NormalizerModel.html | 8 +- .../annotators/PretrainedAnnotations$.html | 8 +- .../ReadablePretrainedLemmatizer.html | 8 +- ...adablePretrainedStopWordsCleanerModel.html | 8 +- .../ReadablePretrainedTextMatcher.html | 8 +- .../ReadablePretrainedTokenizer.html | 8 +- .../nlp/annotators/RecursiveTokenizer.html | 8 +- .../annotators/RecursiveTokenizerModel$.html | 8 +- .../annotators/RecursiveTokenizerModel.html | 8 +- .../nlp/annotators/RegexMatcher$.html | 8 +- .../nlp/annotators/RegexMatcher.html | 8 +- .../nlp/annotators/RegexMatcherModel$.html | 8 +- .../nlp/annotators/RegexMatcherModel.html | 8 +- .../nlp/annotators/RegexTokenizer$.html | 8 +- .../nlp/annotators/RegexTokenizer.html | 8 +- .../nlp/annotators/SingleDatePolicy$.html | 8 +- .../johnsnowlabs/nlp/annotators/Stemmer$.html | 8 +- .../johnsnowlabs/nlp/annotators/Stemmer.html | 8 +- .../nlp/annotators/StopWordsCleaner$.html | 8 +- .../nlp/annotators/StopWordsCleaner.html | 8 +- .../nlp/annotators/TextMatcher$.html | 8 +- .../nlp/annotators/TextMatcher.html | 8 +- .../nlp/annotators/TextMatcherModel$.html | 8 +- .../nlp/annotators/TextMatcherModel.html | 8 +- .../nlp/annotators/TextSplitter.html | 8 +- .../nlp/annotators/Token2Chunk$.html | 8 +- .../nlp/annotators/Token2Chunk.html | 8 +- .../nlp/annotators/Tokenizer$.html | 8 +- .../nlp/annotators/Tokenizer.html | 8 +- .../nlp/annotators/TokenizerModel$.html | 8 +- .../nlp/annotators/TokenizerModel.html | 8 +- .../nlp/annotators/audio/HubertForCTC$.html | 8 +- .../nlp/annotators/audio/HubertForCTC.html | 8 +- .../audio/ReadHubertForAudioDLModel.html | 8 +- .../audio/ReadWav2Vec2ForAudioDLModel.html | 8 +- .../audio/ReadWhisperForCTCDLModel.html | 8 +- ...ReadablePretrainedHubertForAudioModel.html | 8 +- ...adablePretrainedWav2Vec2ForAudioModel.html | 8 +- .../ReadablePretrainedWhisperForCTCModel.html | 8 +- .../nlp/annotators/audio/Wav2Vec2ForCTC$.html | 8 +- .../nlp/annotators/audio/Wav2Vec2ForCTC.html | 8 +- .../nlp/annotators/audio/WhisperForCTC$.html | 8 +- .../nlp/annotators/audio/WhisperForCTC.html | 8 +- .../audio/feature_extractor/AudioUtils$.html | 8 +- .../PreprocessorAttributes$.html | 8 +- .../WhisperPreprocessor.html | 8 +- .../audio/feature_extractor/index.html | 8 +- .../nlp/annotators/audio/index.html | 8 +- .../nlp/annotators/btm/BigTextMatcher$.html | 8 +- .../nlp/annotators/btm/BigTextMatcher.html | 8 +- .../annotators/btm/BigTextMatcherModel$.html | 8 +- .../annotators/btm/BigTextMatcherModel.html | 8 +- .../btm/ReadablePretrainedBigTextMatcher.html | 8 +- .../nlp/annotators/btm/TMEdgesReadWriter.html | 8 +- .../nlp/annotators/btm/TMEdgesReader.html | 8 +- .../nlp/annotators/btm/TMNodesReader.html | 8 +- .../nlp/annotators/btm/TMNodesWriter.html | 8 +- .../nlp/annotators/btm/TMVocabReadWriter.html | 8 +- .../nlp/annotators/btm/TMVocabReader.html | 8 +- .../nlp/annotators/btm/TrieNode.html | 8 +- .../nlp/annotators/btm/index.html | 8 +- .../dl/AlbertForQuestionAnswering$.html | 8 +- .../dl/AlbertForQuestionAnswering.html | 8 +- .../dl/AlbertForSequenceClassification$.html | 8 +- .../dl/AlbertForSequenceClassification.html | 8 +- .../dl/AlbertForTokenClassification$.html | 8 +- .../dl/AlbertForTokenClassification.html | 8 +- .../dl/BartForZeroShotClassification$.html | 8 +- .../dl/BartForZeroShotClassification.html | 8 +- .../dl/BertForQuestionAnswering$.html | 8 +- .../dl/BertForQuestionAnswering.html | 8 +- .../dl/BertForSequenceClassification$.html | 8 +- .../dl/BertForSequenceClassification.html | 8 +- .../dl/BertForTokenClassification$.html | 8 +- .../dl/BertForTokenClassification.html | 8 +- .../dl/BertForZeroShotClassification$.html | 8 +- .../dl/BertForZeroShotClassification.html | 8 +- .../dl/CamemBertForQuestionAnswering$.html | 8 +- .../dl/CamemBertForQuestionAnswering.html | 8 +- .../CamemBertForSequenceClassification$.html | 8 +- .../CamemBertForSequenceClassification.html | 8 +- .../dl/CamemBertForTokenClassification$.html | 8 +- .../dl/CamemBertForTokenClassification.html | 8 +- .../classifier/dl/ClassifierDLApproach$.html | 8 +- .../classifier/dl/ClassifierDLApproach.html | 8 +- .../classifier/dl/ClassifierDLModel$.html | 8 +- .../classifier/dl/ClassifierDLModel.html | 8 +- .../classifier/dl/ClassifierEncoder.html | 8 +- .../classifier/dl/ClassifierMetrics.html | 8 +- .../dl/DeBertaForQuestionAnswering$.html | 8 +- .../dl/DeBertaForQuestionAnswering.html | 8 +- .../dl/DeBertaForSequenceClassification$.html | 8 +- .../dl/DeBertaForSequenceClassification.html | 8 +- .../dl/DeBertaForTokenClassification$.html | 8 +- .../dl/DeBertaForTokenClassification.html | 8 +- .../dl/DeBertaForZeroShotClassification$.html | 8 +- .../dl/DeBertaForZeroShotClassification.html | 8 +- .../dl/DistilBertForQuestionAnswering$.html | 8 +- .../dl/DistilBertForQuestionAnswering.html | 8 +- .../DistilBertForSequenceClassification$.html | 8 +- .../DistilBertForSequenceClassification.html | 8 +- .../dl/DistilBertForTokenClassification$.html | 8 +- .../dl/DistilBertForTokenClassification.html | 8 +- .../DistilBertForZeroShotClassification$.html | 8 +- .../DistilBertForZeroShotClassification.html | 8 +- .../dl/LongformerForQuestionAnswering$.html | 8 +- .../dl/LongformerForQuestionAnswering.html | 8 +- .../LongformerForSequenceClassification$.html | 8 +- .../LongformerForSequenceClassification.html | 8 +- .../dl/LongformerForTokenClassification$.html | 8 +- .../dl/LongformerForTokenClassification.html | 8 +- .../dl/MPNetForQuestionAnswering$.html | 8 +- .../dl/MPNetForQuestionAnswering.html | 8 +- .../dl/MPNetForSequenceClassification$.html | 8 +- .../dl/MPNetForSequenceClassification.html | 8 +- .../dl/MultiClassifierDLApproach.html | 8 +- .../dl/MultiClassifierDLModel$.html | 8 +- .../classifier/dl/MultiClassifierDLModel.html | 8 +- ...ReadAlbertForQuestionAnsweringDLModel.html | 8 +- .../dl/ReadAlbertForSequenceDLModel.html | 8 +- .../dl/ReadAlbertForTokenDLModel.html | 8 +- .../dl/ReadBartForZeroShotDLModel.html | 8 +- .../ReadBertForQuestionAnsweringDLModel.html | 8 +- .../dl/ReadBertForSequenceDLModel.html | 8 +- .../dl/ReadBertForTokenDLModel.html | 8 +- .../dl/ReadBertForZeroShotDLModel.html | 8 +- .../dl/ReadCamemBertForQADLModel.html | 8 +- .../dl/ReadCamemBertForSequenceDLModel.html | 8 +- .../dl/ReadCamemBertForTokenDLModel.html | 8 +- .../dl/ReadClassifierDLTensorflowModel.html | 8 +- ...eadDeBertaForQuestionAnsweringDLModel.html | 8 +- .../dl/ReadDeBertaForSequenceDLModel.html | 8 +- .../dl/ReadDeBertaForTokenDLModel.html | 8 +- .../dl/ReadDeBertaForZeroShotDLModel.html | 8 +- ...DistilBertForQuestionAnsweringDLModel.html | 8 +- .../dl/ReadDistilBertForSequenceDLModel.html | 8 +- .../dl/ReadDistilBertForTokenDLModel.html | 8 +- .../dl/ReadDistilBertForZeroShotDLModel.html | 8 +- ...LongformerForQuestionAnsweringDLModel.html | 8 +- .../dl/ReadLongformerForSequenceDLModel.html | 8 +- .../dl/ReadLongformerForTokenDLModel.html | 8 +- .../ReadMPNetForQuestionAnsweringDLModel.html | 8 +- .../dl/ReadMPNetForSequenceDLModel.html | 8 +- .../ReadMultiClassifierDLTensorflowModel.html | 8 +- ...eadRoBertaForQuestionAnsweringDLModel.html | 8 +- .../dl/ReadRoBertaForSequenceDLModel.html | 8 +- .../dl/ReadRoBertaForTokenDLModel.html | 8 +- .../dl/ReadRoBertaForZeroShotDLModel.html | 8 +- .../dl/ReadSentimentDLTensorflowModel.html | 8 +- .../ReadTapasForQuestionAnsweringDLModel.html | 8 +- ...XlmRoBertaForQuestionAnsweringDLModel.html | 8 +- .../dl/ReadXlmRoBertaForSequenceDLModel.html | 8 +- .../dl/ReadXlmRoBertaForTokenDLModel.html | 8 +- .../dl/ReadXlmRoBertaForZeroShotDLModel.html | 8 +- .../dl/ReadXlnetForSequenceDLModel.html | 8 +- .../dl/ReadXlnetForTokenDLModel.html | 8 +- .../ReadablePretrainedAlbertForQAModel.html | 8 +- ...dablePretrainedAlbertForSequenceModel.html | 8 +- ...ReadablePretrainedAlbertForTokenModel.html | 8 +- ...eadablePretrainedBartForZeroShotModel.html | 8 +- .../dl/ReadablePretrainedBertForQAModel.html | 8 +- ...eadablePretrainedBertForSequenceModel.html | 8 +- .../ReadablePretrainedBertForTokenModel.html | 8 +- ...eadablePretrainedBertForZeroShotModel.html | 8 +- ...ReadablePretrainedCamemBertForQAModel.html | 8 +- ...lePretrainedCamemBertForSequenceModel.html | 8 +- ...dablePretrainedCamemBertForTokenModel.html | 8 +- .../dl/ReadablePretrainedClassifierDL.html | 8 +- .../ReadablePretrainedDeBertaForQAModel.html | 8 +- ...ablePretrainedDeBertaForSequenceModel.html | 8 +- ...eadablePretrainedDeBertaForTokenModel.html | 8 +- ...ablePretrainedDeBertaForZeroShotModel.html | 8 +- ...eadablePretrainedDistilBertForQAModel.html | 8 +- ...ePretrainedDistilBertForSequenceModel.html | 8 +- ...ablePretrainedDistilBertForTokenModel.html | 8 +- ...ePretrainedDistilBertForZeroShotModel.html | 8 +- ...eadablePretrainedLongformerForQAModel.html | 8 +- ...ePretrainedLongformerForSequenceModel.html | 8 +- ...ablePretrainedLongformerForTokenModel.html | 8 +- .../dl/ReadablePretrainedMPNetForQAModel.html | 8 +- ...adablePretrainedMPNetForSequenceModel.html | 8 +- .../ReadablePretrainedMultiClassifierDL.html | 8 +- .../ReadablePretrainedRoBertaForQAModel.html | 8 +- ...ablePretrainedRoBertaForSequenceModel.html | 8 +- ...eadablePretrainedRoBertaForTokenModel.html | 8 +- ...ablePretrainedRoBertaForZeroShotModel.html | 8 +- .../dl/ReadablePretrainedSentimentDL.html | 8 +- .../dl/ReadablePretrainedTapasForQAModel.html | 8 +- ...eadablePretrainedXlmRoBertaForQAModel.html | 8 +- ...ePretrainedXlmRoBertaForSequenceModel.html | 8 +- ...ablePretrainedXlmRoBertaForTokenModel.html | 8 +- ...ePretrainedXlmRoBertaForZeroShotModel.html | 8 +- ...adablePretrainedXlnetForSequenceModel.html | 8 +- .../ReadablePretrainedXlnetForTokenModel.html | 8 +- .../dl/RoBertaForQuestionAnswering$.html | 8 +- .../dl/RoBertaForQuestionAnswering.html | 8 +- .../dl/RoBertaForSequenceClassification$.html | 8 +- .../dl/RoBertaForSequenceClassification.html | 8 +- .../dl/RoBertaForTokenClassification$.html | 8 +- .../dl/RoBertaForTokenClassification.html | 8 +- .../dl/RoBertaForZeroShotClassification$.html | 8 +- .../dl/RoBertaForZeroShotClassification.html | 8 +- .../classifier/dl/SentimentApproach$.html | 8 +- .../classifier/dl/SentimentDLApproach.html | 8 +- .../classifier/dl/SentimentDLModel$.html | 8 +- .../classifier/dl/SentimentDLModel.html | 8 +- .../dl/TapasForQuestionAnswering$.html | 8 +- .../dl/TapasForQuestionAnswering.html | 8 +- .../dl/XlmRoBertaForQuestionAnswering$.html | 8 +- .../dl/XlmRoBertaForQuestionAnswering.html | 8 +- .../XlmRoBertaForSequenceClassification$.html | 8 +- .../XlmRoBertaForSequenceClassification.html | 8 +- .../dl/XlmRoBertaForTokenClassification$.html | 8 +- .../dl/XlmRoBertaForTokenClassification.html | 8 +- .../XlmRoBertaForZeroShotClassification$.html | 8 +- .../XlmRoBertaForZeroShotClassification.html | 8 +- .../dl/XlnetForSequenceClassification$.html | 8 +- .../dl/XlnetForSequenceClassification.html | 8 +- .../dl/XlnetForTokenClassification$.html | 8 +- .../dl/XlnetForTokenClassification.html | 8 +- .../nlp/annotators/classifier/dl/index.html | 8 +- .../nlp/annotators/classifier/index.html | 8 +- .../nlp/annotators/common/Annotated$.html | 8 +- .../nlp/annotators/common/Annotated.html | 8 +- .../nlp/annotators/common/ChunkSplit$.html | 8 +- .../nlp/annotators/common/ConllSentence.html | 8 +- .../DatasetHelpers$$DataFrameHelper.html | 8 +- .../annotators/common/DatasetHelpers$.html | 8 +- .../annotators/common/DependencyParsed$.html | 8 +- .../common/DependencyParsedSentence.html | 8 +- .../common/EmbeddingsWithSentence$.html | 8 +- .../annotators/common/IndexedTaggedWord.html | 8 +- .../nlp/annotators/common/IndexedToken.html | 8 +- .../nlp/annotators/common/InfixToken$.html | 8 +- .../nlp/annotators/common/InfixToken.html | 8 +- .../LabeledDependency$$DependencyInfo.html | 8 +- .../annotators/common/LabeledDependency$.html | 8 +- .../nlp/annotators/common/NerTagged$.html | 8 +- .../nlp/annotators/common/PosTagged$.html | 8 +- .../nlp/annotators/common/PrefixedToken$.html | 8 +- .../nlp/annotators/common/PrefixedToken.html | 8 +- .../common/PreprocessingParser.html | 8 +- .../nlp/annotators/common/Sentence$.html | 8 +- .../nlp/annotators/common/Sentence.html | 8 +- .../nlp/annotators/common/SentenceSplit$.html | 8 +- .../nlp/annotators/common/SuffixedToken$.html | 8 +- .../nlp/annotators/common/SuffixedToken.html | 8 +- .../nlp/annotators/common/TableData$.html | 8 +- .../nlp/annotators/common/TableData.html | 8 +- .../nlp/annotators/common/Tagged.html | 8 +- .../annotators/common/TaggedSentence$.html | 8 +- .../nlp/annotators/common/TaggedSentence.html | 8 +- .../nlp/annotators/common/TaggedWord.html | 8 +- .../nlp/annotators/common/TokenPiece.html | 8 +- .../common/TokenPieceEmbeddings$.html | 8 +- .../common/TokenPieceEmbeddings.html | 8 +- .../annotators/common/TokenizedSentence.html | 8 +- .../common/TokenizedWithSentence$.html | 8 +- .../annotators/common/WordWithDependency.html | 8 +- .../common/WordpieceEmbeddingsSentence$.html | 8 +- .../common/WordpieceEmbeddingsSentence.html | 8 +- .../common/WordpieceTokenized$.html | 8 +- .../common/WordpieceTokenizedSentence.html | 8 +- .../nlp/annotators/common/index.html | 8 +- .../ReadSpanBertCorefTensorflowModel.html | 8 +- .../ReadablePretrainedSpanBertCorefModel.html | 8 +- .../annotators/coref/SpanBertCorefModel$.html | 8 +- .../annotators/coref/SpanBertCorefModel.html | 8 +- .../nlp/annotators/coref/index.html | 8 +- .../cv/CLIPForZeroShotClassification$.html | 8 +- .../cv/CLIPForZeroShotClassification.html | 8 +- .../cv/ConvNextForImageClassification$.html | 8 +- .../cv/ConvNextForImageClassification.html | 8 +- .../nlp/annotators/cv/HasRescaleFactor.html | 8 +- ...eadCLIPForZeroShotClassificationModel.html | 8 +- .../cv/ReadConvNextForImageDLModel.html | 8 +- .../cv/ReadSwinForImageDLModel.html | 8 +- .../annotators/cv/ReadViTForImageDLModel.html | 8 +- .../cv/ReadVisionEncoderDecoderDLModel.html | 8 +- ...nedCLIPForZeroShotClassificationModel.html | 8 +- ...adablePretrainedConvNextForImageModel.html | 8 +- .../ReadablePretrainedSwinForImageModel.html | 8 +- .../ReadablePretrainedViTForImageModel.html | 8 +- ...lePretrainedVisionEncoderDecoderModel.html | 8 +- .../cv/SwinForImageClassification$.html | 8 +- .../cv/SwinForImageClassification.html | 8 +- .../cv/ViTForImageClassification$.html | 8 +- .../cv/ViTForImageClassification.html | 8 +- ...sionEncoderDecoderForImageCaptioning$.html | 8 +- ...isionEncoderDecoderForImageCaptioning.html | 8 +- .../johnsnowlabs/nlp/annotators/cv/index.html | 8 +- .../er/AhoCorasickAutomaton$Node.html | 8 +- .../annotators/er/AhoCorasickAutomaton.html | 8 +- .../nlp/annotators/er/EntityPattern.html | 8 +- .../annotators/er/EntityRulerApproach.html | 8 +- .../annotators/er/EntityRulerFeatures.html | 8 +- .../nlp/annotators/er/EntityRulerModel$.html | 8 +- .../nlp/annotators/er/EntityRulerModel.html | 8 +- .../nlp/annotators/er/EntityRulerUtil$.html | 8 +- .../annotators/er/FlattenEntityPattern.html | 8 +- .../nlp/annotators/er/PatternsReadWriter.html | 8 +- .../nlp/annotators/er/PatternsReader.html | 8 +- .../er/ReadablePretrainedEntityRuler.html | 8 +- .../er/RegexPatternsReadWriter.html | 8 +- .../annotators/er/RegexPatternsReader.html | 8 +- .../johnsnowlabs/nlp/annotators/er/index.html | 8 +- .../johnsnowlabs/nlp/annotators/index.html | 8 +- .../nlp/annotators/keyword/index.html | 8 +- .../keyword/yake/YakeKeywordExtraction$.html | 8 +- .../keyword/yake/YakeKeywordExtraction.html | 8 +- .../annotators/keyword/yake/YakeParams.html | 8 +- .../nlp/annotators/keyword/yake/index.html | 8 +- .../annotators/keyword/yake/util/Token.html | 8 +- .../keyword/yake/util/Utilities$.html | 8 +- .../annotators/keyword/yake/util/index.html | 8 +- .../annotators/ld/dl/LanguageDetectorDL$.html | 8 +- .../annotators/ld/dl/LanguageDetectorDL.html | 8 +- ...ReadLanguageDetectorDLTensorflowModel.html | 8 +- ...ablePretrainedLanguageDetectorDLModel.html | 8 +- .../nlp/annotators/ld/dl/index.html | 8 +- .../johnsnowlabs/nlp/annotators/ld/index.html | 8 +- .../nlp/annotators/ner/ModelMetrics$.html | 8 +- .../nlp/annotators/ner/NamedEntity.html | 8 +- .../nlp/annotators/ner/NerApproach.html | 8 +- .../nlp/annotators/ner/NerConverter$.html | 8 +- .../nlp/annotators/ner/NerConverter.html | 8 +- .../nlp/annotators/ner/NerOverwriter$.html | 8 +- .../nlp/annotators/ner/NerOverwriter.html | 8 +- .../nlp/annotators/ner/NerTagsEncoding$.html | 8 +- .../nlp/annotators/ner/Verbose$.html | 8 +- .../ner/crf/DictionaryFeatures$.html | 8 +- .../ner/crf/DictionaryFeatures.html | 8 +- .../ner/crf/FeatureGenerator$TokenType$.html | 8 +- .../annotators/ner/crf/FeatureGenerator.html | 8 +- .../annotators/ner/crf/NerCrfApproach$.html | 8 +- .../annotators/ner/crf/NerCrfApproach.html | 8 +- .../nlp/annotators/ner/crf/NerCrfModel$.html | 8 +- .../nlp/annotators/ner/crf/NerCrfModel.html | 8 +- .../ner/crf/ReadablePretrainedNerCrf.html | 8 +- .../nlp/annotators/ner/crf/index.html | 8 +- .../nlp/annotators/ner/dl/LoadsContrib$.html | 8 +- .../nlp/annotators/ner/dl/NerDLApproach$.html | 8 +- .../nlp/annotators/ner/dl/NerDLApproach.html | 8 +- .../nlp/annotators/ner/dl/NerDLModel$.html | 8 +- .../nlp/annotators/ner/dl/NerDLModel.html | 8 +- .../ner/dl/NerDLModelPythonReader$.html | 8 +- .../ner/dl/ReadZeroShotNerDLModel.html | 74 ++++++++++++++++--- .../ner/dl/ReadablePretrainedNerDL.html | 8 +- .../ner/dl/ReadablePretrainedZeroShotNer.html | 8 +- .../nlp/annotators/ner/dl/ReadsNERGraph.html | 8 +- .../annotators/ner/dl/WithGraphResolver.html | 8 +- .../annotators/ner/dl/ZeroShotNerModel$.html | 70 +++++++++++++++--- .../annotators/ner/dl/ZeroShotNerModel.html | 8 +- .../nlp/annotators/ner/dl/index.html | 12 +-- .../nlp/annotators/ner/index.html | 8 +- ...lizableFormat$$SerializableDateFormat.html | 8 +- .../AnnotatorParam$SerializableFormat$.html | 8 +- .../nlp/annotators/param/AnnotatorParam.html | 8 +- .../annotators/param/EvaluationDLParams.html | 8 +- .../param/ExternalResourceParam.html | 8 +- .../param/SerializedAnnotatorComponent.html | 8 +- .../param/WritableAnnotatorComponent.html | 8 +- .../nlp/annotators/param/index.html | 8 +- .../parser/dep/DependencyParserApproach$.html | 8 +- .../parser/dep/DependencyParserApproach.html | 8 +- .../parser/dep/DependencyParserModel$.html | 8 +- .../parser/dep/DependencyParserModel.html | 8 +- .../GreedyTransition/DependencyMaker$.html | 8 +- .../DependencyMaker$CurrentState.html | 8 +- .../DependencyMaker$ParseState.html | 8 +- .../dep/GreedyTransition/DependencyMaker.html | 8 +- .../GreedyTransitionApproach$.html | 8 +- .../parser/dep/GreedyTransition/index.html | 8 +- .../GreedyTransition/package$$Feature.html | 8 +- .../GreedyTransition/package$$WordData.html | 8 +- .../parser/dep/Perceptron$WeightLearner.html | 8 +- .../nlp/annotators/parser/dep/Perceptron.html | 8 +- .../dep/ReadablePretrainedDependency.html | 8 +- .../annotators/parser/dep/TagDictionary$.html | 8 +- .../nlp/annotators/parser/dep/Tagger$.html | 8 +- .../nlp/annotators/parser/dep/Tagger.html | 8 +- .../nlp/annotators/parser/dep/index.html | 8 +- .../nlp/annotators/parser/index.html | 8 +- .../annotators/parser/typdep/ConllData.html | 8 +- .../parser/typdep/DependencyArcList.html | 8 +- .../parser/typdep/DependencyInstance.html | 8 +- .../parser/typdep/DependencyPipe.html | 8 +- .../parser/typdep/LocalFeatureData.html | 8 +- .../parser/typdep/LowRankTensor.html | 8 +- .../nlp/annotators/parser/typdep/Options.html | 8 +- .../annotators/parser/typdep/Parameters.html | 8 +- .../parser/typdep/PredictionParameters.html | 8 +- .../ReadablePretrainedTypedDependency.html | 8 +- .../parser/typdep/TrainDependencies.html | 8 +- .../annotators/parser/typdep/TrainFile.html | 8 +- .../parser/typdep/TypedDependencyParser.html | 8 +- .../TypedDependencyParserApproach$.html | 8 +- .../typdep/TypedDependencyParserApproach.html | 8 +- .../typdep/TypedDependencyParserModel$.html | 8 +- .../typdep/TypedDependencyParserModel.html | 8 +- .../typdep/feature/FeatureTemplate.html | 8 +- .../feature/SyntacticFeatureFactory.html | 8 +- .../parser/typdep/feature/index.html | 8 +- .../nlp/annotators/parser/typdep/index.html | 8 +- .../parser/typdep/io/Conll09Reader.html | 8 +- .../parser/typdep/io/ConllUReader.html | 8 +- .../parser/typdep/io/ConllWriter.html | 8 +- .../parser/typdep/io/DependencyReader.html | 8 +- .../annotators/parser/typdep/io/index.html | 8 +- .../parser/typdep/util/Alphabet.html | 8 +- .../parser/typdep/util/Collector.html | 8 +- .../parser/typdep/util/DependencyLabel.html | 8 +- .../parser/typdep/util/Dictionary.html | 8 +- .../parser/typdep/util/DictionarySet.html | 8 +- .../parser/typdep/util/FeatureVector.html | 8 +- .../parser/typdep/util/ScoreCollector.html | 8 +- .../annotators/parser/typdep/util/Utils.html | 8 +- .../annotators/parser/typdep/util/index.html | 8 +- .../nlp/annotators/pos/index.html | 8 +- .../pos/perceptron/AveragedPerceptron.html | 8 +- .../pos/perceptron/PerceptronApproach$.html | 8 +- .../pos/perceptron/PerceptronApproach.html | 8 +- .../PerceptronApproachDistributed$.html | 8 +- .../PerceptronApproachDistributed.html | 8 +- .../pos/perceptron/PerceptronModel$.html | 8 +- .../pos/perceptron/PerceptronModel.html | 8 +- .../perceptron/PerceptronPredictionUtils.html | 8 +- .../perceptron/PerceptronTrainingUtils.html | 8 +- .../pos/perceptron/PerceptronUtils.html | 8 +- .../ReadablePretrainedPerceptron.html | 8 +- .../StringMapStringDoubleAccumulator.html | 8 +- .../perceptron/TrainingPerceptronLegacy.html | 8 +- .../TupleKeyLongDoubleMapAccumulator.html | 8 +- .../nlp/annotators/pos/perceptron/index.html | 8 +- .../sbd/SentenceDetectorParams.html | 8 +- .../nlp/annotators/sbd/index.html | 8 +- .../sbd/pragmatic/CustomPragmaticMethod.html | 8 +- .../sbd/pragmatic/DefaultPragmaticMethod.html | 8 +- .../sbd/pragmatic/MixedPragmaticMethod.html | 8 +- .../pragmatic/PragmaticContentFormatter$.html | 8 +- .../pragmatic/PragmaticContentFormatter.html | 8 +- .../sbd/pragmatic/PragmaticDictionaries$.html | 8 +- .../sbd/pragmatic/PragmaticMethod.html | 8 +- .../pragmatic/PragmaticSentenceExtractor.html | 8 +- .../sbd/pragmatic/PragmaticSymbols$.html | 8 +- .../annotators/sbd/pragmatic/RuleSymbols.html | 8 +- .../sbd/pragmatic/SentenceDetector$.html | 8 +- .../sbd/pragmatic/SentenceDetector.html | 8 +- .../nlp/annotators/sbd/pragmatic/index.html | 8 +- .../nlp/annotators/sda/index.html | 8 +- .../sda/pragmatic/PragmaticScorer.html | 8 +- .../sda/pragmatic/SentimentDetector$.html | 8 +- .../sda/pragmatic/SentimentDetector.html | 8 +- .../pragmatic/SentimentDetectorModel$.html | 8 +- .../sda/pragmatic/SentimentDetectorModel.html | 8 +- .../nlp/annotators/sda/pragmatic/index.html | 8 +- .../sda/vivekn/ReadablePretrainedVivekn.html | 8 +- .../sda/vivekn/ViveknSentimentApproach.html | 8 +- .../sda/vivekn/ViveknSentimentModel$.html | 8 +- .../sda/vivekn/ViveknSentimentModel.html | 8 +- .../sda/vivekn/ViveknSentimentUtils.html | 8 +- .../nlp/annotators/sda/vivekn/index.html | 8 +- .../sentence_detector_dl/Metrics.html | 8 +- .../ReadablePretrainedSentenceDetectorDL.html | 8 +- .../ReadsSentenceDetectorDLGraph.html | 8 +- .../SentenceDetectorDLApproach.html | 8 +- .../SentenceDetectorDLEncoder$.html | 8 +- .../SentenceDetectorDLEncoder.html | 8 +- .../SentenceDetectorDLEncoderParam.html | 8 +- .../SentenceDetectorDLModel$.html | 8 +- .../SentenceDetectorDLModel.html | 8 +- .../sentence_detector_dl/index.html | 8 +- .../annotators/seq2seq/BartTransformer$.html | 8 +- .../annotators/seq2seq/BartTransformer.html | 8 +- .../annotators/seq2seq/GPT2Transformer$.html | 8 +- .../annotators/seq2seq/GPT2Transformer.html | 8 +- .../seq2seq/LLAMA2Transformer$.html | 8 +- .../annotators/seq2seq/LLAMA2Transformer.html | 8 +- .../seq2seq/M2M100Transformer$.html | 8 +- .../annotators/seq2seq/M2M100Transformer.html | 8 +- .../seq2seq/MarianTransformer$.html | 8 +- .../annotators/seq2seq/MarianTransformer.html | 8 +- .../seq2seq/ReadBartTransformerDLModel.html | 8 +- .../seq2seq/ReadGPT2TransformerDLModel.html | 8 +- .../seq2seq/ReadLLAMA2TransformerDLModel.html | 8 +- .../seq2seq/ReadM2M100TransformerDLModel.html | 8 +- .../seq2seq/ReadMarianMTDLModel.html | 8 +- .../seq2seq/ReadT5TransformerDLModel.html | 8 +- ...eadablePretrainedBartTransformerModel.html | 8 +- ...eadablePretrainedGPT2TransformerModel.html | 8 +- ...dablePretrainedLLAMA2TransformerModel.html | 8 +- ...dablePretrainedM2M100TransformerModel.html | 8 +- .../ReadablePretrainedMarianMTModel.html | 8 +- .../ReadablePretrainedT5TransformerModel.html | 8 +- .../annotators/seq2seq/T5Transformer$.html | 8 +- .../nlp/annotators/seq2seq/T5Transformer.html | 8 +- .../nlp/annotators/seq2seq/index.html | 8 +- .../DocumentSimilarityRankerApproach$.html | 8 +- .../DocumentSimilarityRankerApproach.html | 8 +- .../DocumentSimilarityRankerModel$.html | 8 +- .../DocumentSimilarityRankerModel.html | 8 +- .../similarity/IndexedNeighbors.html | 8 +- .../IndexedNeighborsWithDistance.html | 8 +- .../similarity/NeighborAnnotation.html | 8 +- .../similarity/NeighborsResultSet.html | 8 +- .../ReadableDocumentSimilarityRanker.html | 8 +- .../nlp/annotators/similarity/index.html | 8 +- .../spell/context/CandidateStrategy$.html | 8 +- ...ntextSpellCheckerApproach$ArrayHelper.html | 8 +- .../context/ContextSpellCheckerApproach.html | 8 +- .../context/ContextSpellCheckerModel$.html | 8 +- .../ContextSpellCheckerModel$StringTools.html | 8 +- .../context/ContextSpellCheckerModel.html | 8 +- .../spell/context/HasTransducerFeatures.html | 8 +- .../spell/context/LangModelSentence.html | 8 +- .../ReadablePretrainedContextSpell.html | 8 +- .../context/ReadsLanguageModelGraph.html | 8 +- .../spell/context/WeightedLevenshtein.html | 8 +- .../nlp/annotators/spell/context/index.html | 8 +- .../spell/context/parser/AgeToken.html | 8 +- .../spell/context/parser/DateToken.html | 8 +- .../context/parser/GenericRegexParser.html | 8 +- .../context/parser/GenericVocabParser.html | 8 +- .../spell/context/parser/LocationClass.html | 8 +- .../spell/context/parser/MainVocab.html | 8 +- .../spell/context/parser/MedicationClass.html | 8 +- .../spell/context/parser/NamesClass.html | 8 +- .../spell/context/parser/NumberToken.html | 8 +- .../spell/context/parser/RegexParser.html | 8 +- .../context/parser/SerializableClass.html | 8 +- .../context/parser/SpecialClassParser.html | 8 +- .../context/parser/TransducerSeqFeature.html | 8 +- .../spell/context/parser/UnitToken.html | 8 +- .../spell/context/parser/VocabParser.html | 8 +- .../spell/context/parser/index.html | 8 +- .../nlp/annotators/spell/index.html | 8 +- .../spell/norvig/NorvigSweetingApproach$.html | 8 +- .../spell/norvig/NorvigSweetingApproach.html | 8 +- .../spell/norvig/NorvigSweetingModel$.html | 8 +- .../spell/norvig/NorvigSweetingModel.html | 8 +- .../spell/norvig/NorvigSweetingParams.html | 8 +- .../norvig/ReadablePretrainedNorvig.html | 8 +- .../nlp/annotators/spell/norvig/index.html | 8 +- .../ReadablePretrainedSymmetric.html | 8 +- .../symmetric/SymmetricDeleteApproach$.html | 8 +- .../symmetric/SymmetricDeleteApproach.html | 8 +- .../symmetric/SymmetricDeleteModel$.html | 8 +- .../SymmetricDeleteModel$SuggestedWord.html | 8 +- .../spell/symmetric/SymmetricDeleteModel.html | 8 +- .../symmetric/SymmetricDeleteParams.html | 8 +- .../nlp/annotators/spell/symmetric/index.html | 8 +- .../nlp/annotators/spell/util/Utilities$.html | 8 +- .../nlp/annotators/spell/util/index.html | 8 +- .../nlp/annotators/tapas/TapasCellDate$.html | 8 +- .../nlp/annotators/tapas/TapasCellDate.html | 8 +- .../nlp/annotators/tapas/TapasCellValue$.html | 8 +- .../nlp/annotators/tapas/TapasCellValue.html | 8 +- .../nlp/annotators/tapas/TapasEncoder.html | 8 +- .../nlp/annotators/tapas/TapasInputData.html | 8 +- .../tapas/TapasNumericRelation$.html | 8 +- .../tapas/TapasNumericValueSpan$.html | 8 +- .../tapas/TapasNumericValueSpan.html | 8 +- .../nlp/annotators/tapas/index.html | 8 +- .../tokenizer/bpe/BartTokenizer.html | 8 +- .../tokenizer/bpe/BpeTokenizer$.html | 8 +- .../tokenizer/bpe/CLIPTokenizer.html | 8 +- .../tokenizer/bpe/Gpt2Tokenizer.html | 8 +- .../tokenizer/bpe/RobertaTokenizer.html | 8 +- .../tokenizer/bpe/SpecialToken.html | 8 +- .../tokenizer/bpe/WhisperTokenDecoder.html | 8 +- .../nlp/annotators/tokenizer/bpe/index.html | 8 +- .../nlp/annotators/tokenizer/index.html | 8 +- .../ws/ReadablePretrainedWordSegmenter.html | 8 +- .../nlp/annotators/ws/TagsType$.html | 8 +- .../annotators/ws/WordSegmenterApproach$.html | 8 +- .../annotators/ws/WordSegmenterApproach.html | 8 +- .../annotators/ws/WordSegmenterModel$.html | 8 +- .../nlp/annotators/ws/WordSegmenterModel.html | 8 +- .../johnsnowlabs/nlp/annotators/ws/index.html | 8 +- .../nlp/embeddings/AlbertEmbeddings$.html | 8 +- .../nlp/embeddings/AlbertEmbeddings.html | 8 +- .../nlp/embeddings/BGEEmbeddings$.html | 8 +- .../nlp/embeddings/BGEEmbeddings.html | 8 +- .../nlp/embeddings/BertEmbeddings$.html | 8 +- .../nlp/embeddings/BertEmbeddings.html | 8 +- .../embeddings/BertSentenceEmbeddings$.html | 8 +- .../embeddings/BertSentenceEmbeddings.html | 8 +- .../nlp/embeddings/CamemBertEmbeddings$.html | 8 +- .../nlp/embeddings/CamemBertEmbeddings.html | 8 +- .../nlp/embeddings/ChunkEmbeddings$.html | 8 +- .../nlp/embeddings/ChunkEmbeddings.html | 8 +- .../nlp/embeddings/DeBertaEmbeddings$.html | 8 +- .../nlp/embeddings/DeBertaEmbeddings.html | 8 +- .../nlp/embeddings/DistilBertEmbeddings$.html | 8 +- .../nlp/embeddings/DistilBertEmbeddings.html | 8 +- .../nlp/embeddings/Doc2VecApproach$.html | 8 +- .../nlp/embeddings/Doc2VecApproach.html | 8 +- .../nlp/embeddings/Doc2VecModel$.html | 8 +- .../nlp/embeddings/Doc2VecModel.html | 8 +- .../nlp/embeddings/E5Embeddings$.html | 8 +- .../nlp/embeddings/E5Embeddings.html | 8 +- .../nlp/embeddings/ElmoEmbeddings$.html | 8 +- .../nlp/embeddings/ElmoEmbeddings.html | 8 +- .../EmbeddingsCoverage$CoverageResult.html | 8 +- .../nlp/embeddings/EmbeddingsCoverage.html | 8 +- .../embeddings/HasEmbeddingsProperties.html | 8 +- .../nlp/embeddings/InstructorEmbeddings$.html | 8 +- .../nlp/embeddings/InstructorEmbeddings.html | 8 +- .../nlp/embeddings/LongformerEmbeddings$.html | 8 +- .../nlp/embeddings/LongformerEmbeddings.html | 8 +- .../nlp/embeddings/MPNetEmbeddings$.html | 8 +- .../nlp/embeddings/MPNetEmbeddings.html | 8 +- .../PoolingStrategy$$AnnotatorType$.html | 8 +- .../nlp/embeddings/PoolingStrategy$.html | 8 +- .../nlp/embeddings/ReadAlbertDLModel.html | 8 +- .../nlp/embeddings/ReadBGEDLModel.html | 8 +- .../nlp/embeddings/ReadBertDLModel.html | 8 +- .../embeddings/ReadBertSentenceDLModel.html | 8 +- .../nlp/embeddings/ReadCamemBertDLModel.html | 8 +- .../nlp/embeddings/ReadDeBertaDLModel.html | 8 +- .../nlp/embeddings/ReadDistilBertDLModel.html | 8 +- .../nlp/embeddings/ReadE5DLModel.html | 8 +- .../nlp/embeddings/ReadElmoDLModel.html | 8 +- .../nlp/embeddings/ReadInstructorDLModel.html | 8 +- .../nlp/embeddings/ReadLongformerDLModel.html | 8 +- .../nlp/embeddings/ReadMPNetDLModel.html | 8 +- .../nlp/embeddings/ReadRobertaDLModel.html | 8 +- .../ReadRobertaSentenceDLModel.html | 8 +- .../nlp/embeddings/ReadUSEDLModel.html | 8 +- .../nlp/embeddings/ReadXlmRobertaDLModel.html | 8 +- .../ReadXlmRobertaSentenceDLModel.html | 8 +- .../nlp/embeddings/ReadXlnetDLModel.html | 8 +- .../ReadablePretrainedAlbertModel.html | 8 +- .../ReadablePretrainedBGEModel.html | 8 +- .../ReadablePretrainedBertModel.html | 8 +- .../ReadablePretrainedBertSentenceModel.html | 8 +- .../ReadablePretrainedCamemBertModel.html | 8 +- .../ReadablePretrainedDeBertaModel.html | 8 +- .../ReadablePretrainedDistilBertModel.html | 8 +- .../embeddings/ReadablePretrainedDoc2Vec.html | 8 +- .../embeddings/ReadablePretrainedE5Model.html | 8 +- .../ReadablePretrainedElmoModel.html | 8 +- .../ReadablePretrainedInstructorModel.html | 8 +- .../ReadablePretrainedLongformerModel.html | 8 +- .../ReadablePretrainedMPNetModel.html | 8 +- .../ReadablePretrainedRobertaModel.html | 8 +- ...eadablePretrainedRobertaSentenceModel.html | 8 +- .../ReadablePretrainedUSEModel.html | 8 +- .../ReadablePretrainedWord2Vec.html | 8 +- .../ReadablePretrainedWordEmbeddings.html | 8 +- .../ReadablePretrainedXlmRobertaModel.html | 8 +- ...ablePretrainedXlmRobertaSentenceModel.html | 8 +- .../ReadablePretrainedXlnetModel.html | 8 +- .../nlp/embeddings/ReadsFromBytes.html | 8 +- .../nlp/embeddings/RoBertaEmbeddings$.html | 8 +- .../nlp/embeddings/RoBertaEmbeddings.html | 8 +- .../RoBertaSentenceEmbeddings$.html | 8 +- .../embeddings/RoBertaSentenceEmbeddings.html | 8 +- .../nlp/embeddings/SentenceEmbeddings$.html | 8 +- .../nlp/embeddings/SentenceEmbeddings.html | 8 +- .../embeddings/UniversalSentenceEncoder$.html | 8 +- .../embeddings/UniversalSentenceEncoder.html | 8 +- .../nlp/embeddings/Word2VecApproach$.html | 8 +- .../nlp/embeddings/Word2VecApproach.html | 8 +- .../nlp/embeddings/Word2VecModel$.html | 8 +- .../nlp/embeddings/Word2VecModel.html | 8 +- .../nlp/embeddings/WordEmbeddings$.html | 8 +- .../nlp/embeddings/WordEmbeddings.html | 8 +- .../WordEmbeddingsBinaryIndexer$.html | 8 +- .../nlp/embeddings/WordEmbeddingsModel$.html | 8 +- .../nlp/embeddings/WordEmbeddingsModel.html | 8 +- .../nlp/embeddings/WordEmbeddingsReader.html | 8 +- .../WordEmbeddingsTextIndexer$.html | 8 +- .../nlp/embeddings/WordEmbeddingsWriter.html | 8 +- .../nlp/embeddings/XlmRoBertaEmbeddings$.html | 8 +- .../nlp/embeddings/XlmRoBertaEmbeddings.html | 8 +- .../XlmRoBertaSentenceEmbeddings$.html | 8 +- .../XlmRoBertaSentenceEmbeddings.html | 8 +- .../nlp/embeddings/XlnetEmbeddings$.html | 8 +- .../nlp/embeddings/XlnetEmbeddings.html | 8 +- .../johnsnowlabs/nlp/embeddings/index.html | 8 +- .../DocumentSimilarityRankerFinisher$.html | 8 +- .../DocumentSimilarityRankerFinisher.html | 8 +- .../com/johnsnowlabs/nlp/finisher/index.html | 8 +- .../nlp/functions$$EachAnnotations.html | 8 +- .../nlp/functions$$ExplodeAnnotations.html | 8 +- .../nlp/functions$$FilterAnnotations.html | 8 +- .../nlp/functions$$MapAnnotations.html | 8 +- docs/api/com/johnsnowlabs/nlp/functions$.html | 8 +- docs/api/com/johnsnowlabs/nlp/index.html | 8 +- .../nlp/pretrained/PretrainedPipeline$.html | 8 +- .../nlp/pretrained/PretrainedPipeline.html | 8 +- .../pretrained/PythonResourceDownloader$.html | 8 +- .../nlp/pretrained/RepositoryMetadata.html | 8 +- .../nlp/pretrained/ResourceDownloader$.html | 8 +- .../nlp/pretrained/ResourceDownloader.html | 8 +- .../nlp/pretrained/ResourceMetadata$.html | 8 +- .../nlp/pretrained/ResourceMetadata.html | 8 +- .../nlp/pretrained/ResourceRequest.html | 8 +- .../nlp/pretrained/ResourceType$.html | 8 +- .../nlp/pretrained/S3ResourceDownloader.html | 8 +- .../johnsnowlabs/nlp/pretrained/index.html | 8 +- .../com/johnsnowlabs/nlp/recursive/index.html | 8 +- .../nlp/recursive/package$$Recursive.html | 8 +- .../recursive/package$$RecursiveModel.html | 8 +- .../nlp/serialization/ArrayFeature.html | 8 +- .../nlp/serialization/Feature.html | 8 +- .../nlp/serialization/MapFeature.html | 8 +- .../SerializedExternalResource.html | 8 +- .../nlp/serialization/SetFeature.html | 8 +- .../nlp/serialization/StructFeature.html | 8 +- .../nlp/serialization/TransducerFeature.html | 8 +- .../johnsnowlabs/nlp/serialization/index.html | 8 +- .../com/johnsnowlabs/nlp/training/CoNLL.html | 8 +- .../nlp/training/CoNLL2003NerReader.html | 8 +- .../nlp/training/CoNLLDocument.html | 8 +- .../CoNLLHelper$$CoNLLSentenceCols.html | 8 +- .../training/CoNLLHelper$$CoNLLTokenCols.html | 8 +- .../nlp/training/CoNLLHelper$.html | 8 +- .../com/johnsnowlabs/nlp/training/CoNLLU.html | 8 +- .../nlp/training/CoNLLUCols$.html | 8 +- .../nlp/training/CoNLLUDocument.html | 8 +- .../com/johnsnowlabs/nlp/training/POS.html | 8 +- .../johnsnowlabs/nlp/training/PubTator.html | 8 +- .../nlp/training/SpacyToAnnotation.html | 8 +- .../com/johnsnowlabs/nlp/training/index.html | 8 +- .../johnsnowlabs/nlp/util/FinisherUtil$.html | 8 +- .../johnsnowlabs/nlp/util/GraphBuilder.html | 8 +- .../nlp/util/LfuCache$CachedItem.html | 8 +- .../nlp/util/LfuCache$DoubleLinked.html | 8 +- .../nlp/util/LfuCache$FrequencyList.html | 8 +- .../com/johnsnowlabs/nlp/util/LfuCache.html | 8 +- .../nlp/util/LruMap$KeyPriority.html | 8 +- .../nlp/util/LruMap$KeyPriorityOrdering$.html | 8 +- .../api/com/johnsnowlabs/nlp/util/LruMap.html | 8 +- .../nlp/util/SparkNlpConfig$.html | 8 +- docs/api/com/johnsnowlabs/nlp/util/index.html | 8 +- .../nlp/util/io/CloudStorageType$.html | 8 +- .../nlp/util/io/ExternalResource$.html | 8 +- .../nlp/util/io/ExternalResource.html | 8 +- .../nlp/util/io/MatchStrategy$.html | 8 +- .../nlp/util/io/OutputHelper$.html | 8 +- .../com/johnsnowlabs/nlp/util/io/ReadAs$.html | 8 +- .../util/io/ResourceHelper$$SourceStream.html | 8 +- .../nlp/util/io/ResourceHelper$.html | 8 +- .../com/johnsnowlabs/nlp/util/io/index.html | 8 +- .../nlp/util/regex/RegexRule.html | 8 +- .../util/regex/RuleFactory$$RuleMatch.html | 8 +- .../nlp/util/regex/RuleFactory$.html | 8 +- .../nlp/util/regex/RuleFactory.html | 8 +- .../nlp/util/regex/TransformStrategy$.html | 8 +- .../johnsnowlabs/nlp/util/regex/index.html | 8 +- .../com/johnsnowlabs/storage/BytesKey.html | 8 +- .../com/johnsnowlabs/storage/Database$.html | 8 +- .../com/johnsnowlabs/storage/Database.html | 8 +- .../johnsnowlabs/storage/HasConnection.html | 8 +- .../com/johnsnowlabs/storage/HasStorage.html | 8 +- .../johnsnowlabs/storage/HasStorageModel.html | 8 +- .../storage/HasStorageOptions.html | 8 +- .../storage/HasStorageReader.html | 8 +- .../johnsnowlabs/storage/HasStorageRef$.html | 8 +- .../johnsnowlabs/storage/HasStorageRef.html | 8 +- .../storage/RocksDBConnection$.html | 8 +- .../storage/RocksDBConnection.html | 8 +- .../storage/StorageBatchWriter.html | 8 +- .../johnsnowlabs/storage/StorageFormat.html | 8 +- .../johnsnowlabs/storage/StorageHelper$.html | 8 +- .../johnsnowlabs/storage/StorageLocator$.html | 8 +- .../johnsnowlabs/storage/StorageLocator.html | 8 +- .../storage/StorageReadWriter.html | 8 +- .../johnsnowlabs/storage/StorageReadable.html | 8 +- .../johnsnowlabs/storage/StorageReader.html | 8 +- .../johnsnowlabs/storage/StorageWriter.html | 8 +- docs/api/com/johnsnowlabs/storage/index.html | 8 +- .../api/com/johnsnowlabs/util/Benchmark$.html | 8 +- docs/api/com/johnsnowlabs/util/Build$.html | 8 +- .../johnsnowlabs/util/CoNLLGenerator$.html | 8 +- .../com/johnsnowlabs/util/ConfigHelper$.html | 8 +- .../com/johnsnowlabs/util/ConfigLoader$.html | 8 +- .../com/johnsnowlabs/util/FileHelper$.html | 8 +- .../com/johnsnowlabs/util/JsonBuilder$.html | 8 +- .../com/johnsnowlabs/util/JsonParser$.html | 8 +- .../johnsnowlabs/util/PipelineModels$.html | 8 +- .../johnsnowlabs/util/TrainingHelper$.html | 8 +- docs/api/com/johnsnowlabs/util/Version$.html | 8 +- docs/api/com/johnsnowlabs/util/Version.html | 8 +- .../johnsnowlabs/util/ZipArchiveUtil$.html | 8 +- docs/api/com/johnsnowlabs/util/index.html | 8 +- .../util/spark/LongMapAccumulator.html | 8 +- .../util/spark/MapAccumulator.html | 8 +- .../johnsnowlabs/util/spark/SparkUtil$.html | 8 +- .../com/johnsnowlabs/util/spark/index.html | 8 +- docs/api/index.html | 8 +- docs/api/index.js | 2 +- docs/api/python/.buildinfo | 2 +- docs/api/python/genindex.html | 2 +- docs/api/python/getting_started/index.html | 20 ++--- docs/api/python/index.html | 2 +- docs/api/python/modules/index.html | 2 +- docs/api/python/modules/sparknlp.html | 6 +- .../python/modules/sparknlp/annotation.html | 2 +- .../modules/sparknlp/annotation_audio.html | 2 +- .../modules/sparknlp/annotation_image.html | 2 +- .../annotator/audio/hubert_for_ctc.html | 2 +- .../annotator/audio/wav2vec2_for_ctc.html | 2 +- .../annotator/audio/whisper_for_ctc.html | 2 +- .../sparknlp/annotator/chunk2_doc.html | 2 +- .../modules/sparknlp/annotator/chunker.html | 2 +- .../albert_for_question_answering.html | 2 +- .../albert_for_sequence_classification.html | 2 +- .../albert_for_token_classification.html | 2 +- .../bart_for_zero_shot_classification.html | 2 +- .../bert_for_question_answering.html | 2 +- .../bert_for_sequence_classification.html | 2 +- .../bert_for_token_classification.html | 2 +- .../bert_for_zero_shot_classification.html | 2 +- .../camembert_for_question_answering.html | 2 +- ...camembert_for_sequence_classification.html | 2 +- .../camembert_for_token_classification.html | 2 +- .../classifier_dl/classifier_dl.html | 2 +- .../deberta_for_question_answering.html | 2 +- .../deberta_for_sequence_classification.html | 2 +- .../deberta_for_token_classification.html | 2 +- .../deberta_for_zero_shot_classification.html | 2 +- .../distil_bert_for_question_answering.html | 2 +- ...stil_bert_for_sequence_classification.html | 2 +- .../distil_bert_for_token_classification.html | 2 +- ...til_bert_for_zero_shot_classification.html | 2 +- .../longformer_for_question_answering.html | 2 +- ...ongformer_for_sequence_classification.html | 2 +- .../longformer_for_token_classification.html | 2 +- .../mpnet_for_question_answering.html | 2 +- .../mpnet_for_sequence_classification.html | 2 +- .../classifier_dl/multi_classifier_dl.html | 2 +- .../roberta_for_question_answering.html | 2 +- .../roberta_for_sequence_classification.html | 2 +- .../roberta_for_token_classification.html | 2 +- .../roberta_for_zero_shot_classification.html | 2 +- .../annotator/classifier_dl/sentiment_dl.html | 2 +- .../tapas_for_question_answering.html | 2 +- .../xlm_roberta_for_question_answering.html | 2 +- ...m_roberta_for_sequence_classification.html | 2 +- .../xlm_roberta_for_token_classification.html | 2 +- ..._roberta_for_zero_shot_classification.html | 2 +- .../xlnet_for_sequence_classification.html | 2 +- .../xlnet_for_token_classification.html | 2 +- .../annotator/coref/spanbert_coref.html | 2 +- .../cv/clip_for_zero_shot_classification.html | 2 +- .../cv/convnext_for_image_classification.html | 2 +- .../cv/swin_for_image_classification.html | 2 +- ..._encoder_decoder_for_image_captioning.html | 2 +- .../cv/vit_for_image_classification.html | 2 +- .../sparknlp/annotator/date2_chunk.html | 2 +- .../dependency/dependency_parser.html | 2 +- .../dependency/typed_dependency_parser.html | 2 +- .../document_character_text_splitter.html | 2 +- .../annotator/document_normalizer.html | 2 +- .../annotator/document_token_splitter.html | 2 +- .../document_token_splitter_test.html | 2 +- .../embeddings/albert_embeddings.html | 2 +- .../annotator/embeddings/bert_embeddings.html | 2 +- .../embeddings/bert_sentence_embeddings.html | 2 +- .../annotator/embeddings/bge_embeddings.html | 2 +- .../embeddings/camembert_embeddings.html | 2 +- .../embeddings/chunk_embeddings.html | 2 +- .../embeddings/deberta_embeddings.html | 2 +- .../embeddings/distil_bert_embeddings.html | 2 +- .../annotator/embeddings/doc2vec.html | 2 +- .../annotator/embeddings/e5_embeddings.html | 2 +- .../annotator/embeddings/elmo_embeddings.html | 2 +- .../embeddings/instructor_embeddings.html | 2 +- .../embeddings/longformer_embeddings.html | 2 +- .../embeddings/mpnet_embeddings.html | 2 +- .../embeddings/roberta_embeddings.html | 2 +- .../roberta_sentence_embeddings.html | 2 +- .../embeddings/sentence_embeddings.html | 2 +- .../universal_sentence_encoder.html | 2 +- .../annotator/embeddings/word2vec.html | 2 +- .../annotator/embeddings/word_embeddings.html | 2 +- .../embeddings/xlm_roberta_embeddings.html | 2 +- .../xlm_roberta_sentence_embeddings.html | 2 +- .../embeddings/xlnet_embeddings.html | 2 +- .../sparknlp/annotator/er/entity_ruler.html | 2 +- .../sparknlp/annotator/graph_extraction.html | 2 +- .../yake_keyword_extraction.html | 2 +- .../annotator/ld_dl/language_detector_dl.html | 2 +- .../sparknlp/annotator/lemmatizer.html | 2 +- .../annotator/matcher/big_text_matcher.html | 2 +- .../annotator/matcher/date_matcher.html | 2 +- .../annotator/matcher/multi_date_matcher.html | 2 +- .../annotator/matcher/regex_matcher.html | 2 +- .../annotator/matcher/text_matcher.html | 2 +- .../sparknlp/annotator/n_gram_generator.html | 2 +- .../sparknlp/annotator/ner/ner_approach.html | 2 +- .../sparknlp/annotator/ner/ner_converter.html | 2 +- .../sparknlp/annotator/ner/ner_crf.html | 2 +- .../sparknlp/annotator/ner/ner_dl.html | 2 +- .../annotator/ner/ner_overwriter.html | 2 +- .../annotator/ner/zero_shot_ner_model.html | 2 +- .../sparknlp/annotator/normalizer.html | 2 +- .../annotator/openai/openai_completion.html | 2 +- .../annotator/openai/openai_embeddings.html | 2 +- .../annotator/param/classifier_encoder.html | 2 +- .../annotator/param/evaluation_dl_params.html | 2 +- .../sparknlp/annotator/pos/perceptron.html | 2 +- .../annotator/sentence/sentence_detector.html | 2 +- .../sentence/sentence_detector_dl.html | 2 +- .../sentiment/sentiment_detector.html | 2 +- .../annotator/sentiment/vivekn_sentiment.html | 2 +- .../annotator/seq2seq/bart_transformer.html | 2 +- .../annotator/seq2seq/gpt2_transformer.html | 2 +- .../annotator/seq2seq/llama2_transformer.html | 2 +- .../annotator/seq2seq/m2m100_transformer.html | 2 +- .../annotator/seq2seq/marian_transformer.html | 2 +- .../annotator/seq2seq/t5_transformer.html | 2 +- .../document_similarity_ranker.html | 2 +- .../spell_check/context_spell_checker.html | 2 +- .../spell_check/norvig_sweeting.html | 2 +- .../spell_check/symmetric_delete.html | 2 +- .../modules/sparknlp/annotator/stemmer.html | 2 +- .../annotator/stop_words_cleaner.html | 2 +- .../annotator/tf_ner_dl_graph_builder.html | 2 +- .../annotator/token/chunk_tokenizer.html | 2 +- .../annotator/token/recursive_tokenizer.html | 2 +- .../annotator/token/regex_tokenizer.html | 2 +- .../sparknlp/annotator/token/tokenizer.html | 2 +- .../sparknlp/annotator/token2_chunk.html | 2 +- .../sparknlp/annotator/ws/word_segmenter.html | 2 +- .../sparknlp/base/audio_assembler.html | 2 +- .../modules/sparknlp/base/doc2_chunk.html | 2 +- .../sparknlp/base/document_assembler.html | 2 +- .../sparknlp/base/embeddings_finisher.html | 2 +- .../modules/sparknlp/base/finisher.html | 2 +- .../modules/sparknlp/base/graph_finisher.html | 2 +- .../sparknlp/base/has_recursive_fit.html | 2 +- .../base/has_recursive_transform.html | 2 +- .../sparknlp/base/image_assembler.html | 2 +- .../modules/sparknlp/base/light_pipeline.html | 2 +- .../base/multi_document_assembler.html | 2 +- .../sparknlp/base/recursive_pipeline.html | 2 +- .../sparknlp/base/table_assembler.html | 2 +- .../sparknlp/base/token_assembler.html | 2 +- .../sparknlp/common/annotator_approach.html | 2 +- .../sparknlp/common/annotator_model.html | 2 +- .../sparknlp/common/annotator_properties.html | 2 +- .../sparknlp/common/match_strategy.html | 2 +- .../modules/sparknlp/common/properties.html | 2 +- .../modules/sparknlp/common/read_as.html | 2 +- .../common/recursive_annotator_approach.html | 2 +- .../python/modules/sparknlp/common/utils.html | 2 +- .../python/modules/sparknlp/functions.html | 2 +- .../sparknlp/internal/annotator_java_ml.html | 2 +- .../internal/annotator_transformer.html | 2 +- .../internal/extended_java_wrapper.html | 2 +- .../internal/params_getters_setters.html | 2 +- .../modules/sparknlp/internal/recursive.html | 2 +- .../modules/sparknlp/logging/comet.html | 2 +- .../pretrained/pretrained_pipeline.html | 2 +- .../pretrained/resource_downloader.html | 2 +- .../modules/sparknlp/training/conll.html | 2 +- .../modules/sparknlp/training/conllu.html | 2 +- .../python/modules/sparknlp/training/pos.html | 2 +- .../modules/sparknlp/training/pub_tator.html | 2 +- .../training/spacy_to_annotation.html | 2 +- docs/api/python/py-modindex.html | 2 +- .../sparknlp/annotation/index.html | 2 +- .../sparknlp/annotation_audio/index.html | 2 +- .../sparknlp/annotation_image/index.html | 2 +- .../annotator/audio/hubert_for_ctc/index.html | 2 +- .../sparknlp/annotator/audio/index.html | 2 +- .../audio/wav2vec2_for_ctc/index.html | 2 +- .../audio/whisper_for_ctc/index.html | 2 +- .../sparknlp/annotator/chunk2_doc/index.html | 2 +- .../sparknlp/annotator/chunker/index.html | 2 +- .../albert_for_question_answering/index.html | 2 +- .../index.html | 2 +- .../index.html | 2 +- .../index.html | 2 +- .../bert_for_question_answering/index.html | 2 +- .../index.html | 2 +- .../bert_for_token_classification/index.html | 2 +- .../index.html | 2 +- .../index.html | 2 +- .../index.html | 2 +- .../index.html | 2 +- .../classifier_dl/classifier_dl/index.html | 2 +- .../deberta_for_question_answering/index.html | 2 +- .../index.html | 2 +- .../index.html | 2 +- .../index.html | 2 +- .../index.html | 2 +- .../index.html | 2 +- .../index.html | 2 +- .../index.html | 2 +- .../annotator/classifier_dl/index.html | 2 +- .../index.html | 2 +- .../index.html | 2 +- .../index.html | 2 +- .../mpnet_for_question_answering/index.html | 2 +- .../index.html | 2 +- .../multi_classifier_dl/index.html | 2 +- .../roberta_for_question_answering/index.html | 2 +- .../index.html | 2 +- .../index.html | 2 +- .../index.html | 2 +- .../classifier_dl/sentiment_dl/index.html | 2 +- .../tapas_for_question_answering/index.html | 2 +- .../index.html | 2 +- .../index.html | 2 +- .../index.html | 2 +- .../index.html | 2 +- .../index.html | 2 +- .../xlnet_for_token_classification/index.html | 2 +- .../sparknlp/annotator/coref/index.html | 2 +- .../annotator/coref/spanbert_coref/index.html | 2 +- .../index.html | 2 +- .../index.html | 2 +- .../sparknlp/annotator/cv/index.html | 2 +- .../swin_for_image_classification/index.html | 2 +- .../index.html | 2 +- .../vit_for_image_classification/index.html | 2 +- .../sparknlp/annotator/date2_chunk/index.html | 2 +- .../dependency/dependency_parser/index.html | 2 +- .../sparknlp/annotator/dependency/index.html | 2 +- .../typed_dependency_parser/index.html | 2 +- .../index.html | 2 +- .../annotator/document_normalizer/index.html | 2 +- .../document_token_splitter/index.html | 2 +- .../document_token_splitter_test/index.html | 2 +- .../embeddings/albert_embeddings/index.html | 2 +- .../embeddings/bert_embeddings/index.html | 2 +- .../bert_sentence_embeddings/index.html | 2 +- .../embeddings/bge_embeddings/index.html | 2 +- .../camembert_embeddings/index.html | 2 +- .../embeddings/chunk_embeddings/index.html | 2 +- .../embeddings/deberta_embeddings/index.html | 2 +- .../distil_bert_embeddings/index.html | 2 +- .../annotator/embeddings/doc2vec/index.html | 2 +- .../embeddings/e5_embeddings/index.html | 2 +- .../embeddings/elmo_embeddings/index.html | 2 +- .../sparknlp/annotator/embeddings/index.html | 2 +- .../instructor_embeddings/index.html | 2 +- .../longformer_embeddings/index.html | 2 +- .../embeddings/mpnet_embeddings/index.html | 2 +- .../embeddings/roberta_embeddings/index.html | 2 +- .../roberta_sentence_embeddings/index.html | 2 +- .../embeddings/sentence_embeddings/index.html | 2 +- .../universal_sentence_encoder/index.html | 2 +- .../annotator/embeddings/word2vec/index.html | 2 +- .../embeddings/word_embeddings/index.html | 2 +- .../xlm_roberta_embeddings/index.html | 2 +- .../index.html | 2 +- .../embeddings/xlnet_embeddings/index.html | 2 +- .../annotator/er/entity_ruler/index.html | 2 +- .../sparknlp/annotator/er/index.html | 2 +- .../annotator/graph_extraction/index.html | 2 +- .../autosummary/sparknlp/annotator/index.html | 2 +- .../annotator/keyword_extraction/index.html | 2 +- .../yake_keyword_extraction/index.html | 2 +- .../sparknlp/annotator/ld_dl/index.html | 2 +- .../ld_dl/language_detector_dl/index.html | 2 +- .../sparknlp/annotator/lemmatizer/index.html | 2 +- .../matcher/big_text_matcher/index.html | 2 +- .../annotator/matcher/date_matcher/index.html | 2 +- .../sparknlp/annotator/matcher/index.html | 2 +- .../matcher/multi_date_matcher/index.html | 2 +- .../matcher/regex_matcher/index.html | 2 +- .../annotator/matcher/text_matcher/index.html | 2 +- .../annotator/n_gram_generator/index.html | 2 +- .../sparknlp/annotator/ner/index.html | 2 +- .../annotator/ner/ner_approach/index.html | 2 +- .../annotator/ner/ner_converter/index.html | 2 +- .../sparknlp/annotator/ner/ner_crf/index.html | 2 +- .../sparknlp/annotator/ner/ner_dl/index.html | 2 +- .../annotator/ner/ner_overwriter/index.html | 2 +- .../ner/zero_shot_ner_model/index.html | 2 +- .../sparknlp/annotator/normalizer/index.html | 2 +- .../sparknlp/annotator/openai/index.html | 2 +- .../openai/openai_completion/index.html | 2 +- .../openai/openai_embeddings/index.html | 2 +- .../param/classifier_encoder/index.html | 2 +- .../param/evaluation_dl_params/index.html | 2 +- .../sparknlp/annotator/param/index.html | 2 +- .../sparknlp/annotator/pos/index.html | 2 +- .../annotator/pos/perceptron/index.html | 2 +- .../sparknlp/annotator/sentence/index.html | 2 +- .../sentence/sentence_detector/index.html | 2 +- .../sentence/sentence_detector_dl/index.html | 2 +- .../sparknlp/annotator/sentiment/index.html | 2 +- .../sentiment/sentiment_detector/index.html | 2 +- .../sentiment/vivekn_sentiment/index.html | 2 +- .../seq2seq/bart_transformer/index.html | 2 +- .../seq2seq/gpt2_transformer/index.html | 2 +- .../sparknlp/annotator/seq2seq/index.html | 2 +- .../seq2seq/llama2_transformer/index.html | 2 +- .../seq2seq/m2m100_transformer/index.html | 2 +- .../seq2seq/marian_transformer/index.html | 2 +- .../seq2seq/t5_transformer/index.html | 2 +- .../document_similarity_ranker/index.html | 2 +- .../sparknlp/annotator/similarity/index.html | 2 +- .../context_spell_checker/index.html | 2 +- .../sparknlp/annotator/spell_check/index.html | 2 +- .../spell_check/norvig_sweeting/index.html | 2 +- .../spell_check/symmetric_delete/index.html | 2 +- .../sparknlp/annotator/stemmer/index.html | 2 +- .../annotator/stop_words_cleaner/index.html | 2 +- .../tf_ner_dl_graph_builder/index.html | 2 +- .../token/chunk_tokenizer/index.html | 2 +- .../sparknlp/annotator/token/index.html | 2 +- .../token/recursive_tokenizer/index.html | 2 +- .../token/regex_tokenizer/index.html | 2 +- .../annotator/token/tokenizer/index.html | 2 +- .../annotator/token2_chunk/index.html | 2 +- .../sparknlp/annotator/ws/index.html | 2 +- .../annotator/ws/word_segmenter/index.html | 2 +- .../sparknlp/base/audio_assembler/index.html | 2 +- .../sparknlp/base/doc2_chunk/index.html | 2 +- .../base/document_assembler/index.html | 2 +- .../base/embeddings_finisher/index.html | 2 +- .../sparknlp/base/finisher/index.html | 2 +- .../sparknlp/base/graph_finisher/index.html | 2 +- .../base/has_recursive_fit/index.html | 2 +- .../base/has_recursive_transform/index.html | 2 +- .../sparknlp/base/image_assembler/index.html | 2 +- .../autosummary/sparknlp/base/index.html | 2 +- .../sparknlp/base/light_pipeline/index.html | 2 +- .../base/multi_document_assembler/index.html | 2 +- .../base/recursive_pipeline/index.html | 2 +- .../sparknlp/base/table_assembler/index.html | 2 +- .../sparknlp/base/token_assembler/index.html | 2 +- .../common/annotator_approach/index.html | 2 +- .../common/annotator_model/index.html | 2 +- .../common/annotator_properties/index.html | 2 +- .../sparknlp/common/annotator_type/index.html | 2 +- .../common/coverage_result/index.html | 2 +- .../autosummary/sparknlp/common/index.html | 2 +- .../sparknlp/common/match_strategy/index.html | 2 +- .../sparknlp/common/properties/index.html | 2 +- .../sparknlp/common/read_as/index.html | 2 +- .../recursive_annotator_approach/index.html | 2 +- .../sparknlp/common/storage/index.html | 2 +- .../sparknlp/common/utils/index.html | 2 +- .../autosummary/sparknlp/functions/index.html | 2 +- .../reference/autosummary/sparknlp/index.html | 2 +- .../internal/annotator_java_ml/index.html | 2 +- .../internal/annotator_transformer/index.html | 2 +- .../internal/extended_java_wrapper/index.html | 2 +- .../autosummary/sparknlp/internal/index.html | 2 +- .../params_getters_setters/index.html | 2 +- .../sparknlp/internal/recursive/index.html | 2 +- .../sparknlp/logging/comet/index.html | 2 +- .../autosummary/sparknlp/logging/index.html | 2 +- .../sparknlp/pretrained/index.html | 2 +- .../pretrained/pretrained_pipeline/index.html | 2 +- .../pretrained/resource_downloader/index.html | 2 +- .../sparknlp/pretrained/utils/index.html | 2 +- .../sparknlp/training/conll/index.html | 2 +- .../sparknlp/training/conllu/index.html | 2 +- .../autosummary/sparknlp/training/index.html | 2 +- .../sparknlp/training/pos/index.html | 2 +- .../sparknlp/training/pub_tator/index.html | 2 +- .../training/spacy_to_annotation/index.html | 2 +- .../sparknlp/training/tfgraphs/index.html | 2 +- .../sparknlp/upload_to_hub/index.html | 2 +- .../autosummary/sparknlp/util/index.html | 2 +- docs/api/python/reference/index.html | 2 +- docs/api/python/search.html | 2 +- docs/api/python/searchindex.js | 2 +- .../python/static/documentation_options.js | 2 +- docs/api/python/third_party/Comet.html | 2 +- docs/api/python/third_party/MLflow.html | 2 +- docs/api/python/third_party/index.html | 2 +- docs/api/python/user_guide/annotation.html | 2 +- docs/api/python/user_guide/annotators.html | 2 +- .../python/user_guide/custom_pipelines.html | 2 +- docs/api/python/user_guide/helpers.html | 2 +- docs/api/python/user_guide/index.html | 2 +- .../python/user_guide/light_pipelines.html | 2 +- .../user_guide/pretrained_pipelines.html | 2 +- docs/api/python/user_guide/training.html | 2 +- docs/api/scala/collection/compat/index.html | 8 +- docs/api/scala/collection/index.html | 8 +- docs/api/scala/index.html | 8 +- .../annotators/ner/dl/ZeroShotNerModel.scala | 3 +- 1501 files changed, 4969 insertions(+), 4868 deletions(-) diff --git a/docs/api/com/index.html b/docs/api/com/index.html index 4c1d3a2ceb34..42cdc142ffa4 100644 --- a/docs/api/com/index.html +++ b/docs/api/com/index.html @@ -3,9 +3,9 @@ - Spark NLP 5.3.0 ScalaDoc - com - - + Spark NLP 5.3.1 ScalaDoc - com + + @@ -28,7 +28,7 @@