Update Models Hub Branch

.github/workflows/create_search_index.yml

@@ -94,7 +94,7 @@ jobs:
             ./docs/backup-benchmarking.json \
             ./docs/backup-references.json
       - name: Upload artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: jekyll-build
           path: |

.github/workflows/publish_docs.yaml

@@ -3,60 +3,77 @@ name: Publish APIs
 on:
   push:
     branches:
-      - '*release*'
-      - 'release/**'
+      - "*release*"
+      - "release/**"
   pull_request:
     branches:
-      - 'main'
-      - 'master'
-      - '*release*'
-      - 'release/**'
+      - "main"
+      - "master"
+      - "*release*"
+      - "release/**"
+
 env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
 jobs:
   build:
     if: "contains(toJSON(github.event.commits.*.message), '[run doc]')"
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
-    - name: checkout repo
-      uses: actions/checkout@v2
-    - name: Set up JDK 8
-      uses: actions/setup-java@v1
-      with:
-        java-version: 1.8
-    - name: Install Python 3.7
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.7.7
-        architecture: x64
-    - name: Build Scala APIs
-      run: |
-        sbt doc
-    - name: Install PyPI dependencies
-      run: |
-        python -m pip install --upgrade pip
-        cd ./python/docs && pip install -r requirements_doc.txt
-    - name: Build Python APIs
-      run: |
-        cd ./python/docs
-        make html
-    - name: Commit changes
-      id: commit
-      run: |
-        git config --local user.email "action@github.com"
-        git config --local user.name "github-actions"
-        git add --all
-        if [-z "$(git status --porcelain)"]; then
-           echo "::set-output name=push::false"
-        else
-           git commit -m "Update Scala and Python APIs" -a
-           echo "::set-output name=push::true"
-        fi
-      shell: bash
-    - name: Push changes
-      if: steps.commit.outputs.push == 'true'
-      uses: ad-m/github-push-action@master
-      with:
-         github_token: ${{ secrets.GITHUB_TOKEN }}
-         branch: ${{ github.ref }}
+      - name: Checkout repo
+        uses: actions/checkout@v2
+
+      - name: Set up JDK 8
+        uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.8"
+          architecture: "x64"
+
+      - name: Install SBT
+        run: |
+          echo "deb https://repo.scala-sbt.org/scalasbt/debian all main" | sudo tee /etc/apt/sources.list.d/sbt.list
+          echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | sudo tee -a /etc/apt/sources.list.d/sbt.list
+          curl -sL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x99E82A75642AC823" | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/sbt.gpg > /dev/null
+          sudo apt-get update
+          sudo apt-get install -y sbt
+
+      - name: Build Scala APIs
+        run: sbt doc
+
+      - name: Install PyPI dependencies
+        run: |
+          python -m pip install --upgrade pip
+          cd ./python/docs && pip install -r requirements_doc.txt
+
+      - name: Build Python APIs
+        run: |
+          cd ./python/docs
+          # Run with verbose output to debug any issues
+          SPHINX_APIDOC_OPTIONS=members,undoc-members,show-inheritance sphinx-apidoc -e -f -o ./_api ../sparknlp ../sparknlp/tests
+          make html SPHINXOPTS="-v"
+
+      - name: Commit changes
+        id: commit
+        run: |
+          git config --local user.email "action@github.com"
+          git config --local user.name "github-actions"
+          git add --all
+          if [ -z "$(git status --porcelain)" ]; then
+            echo "push=false" >> $GITHUB_OUTPUT
+          else
+            git commit -m "Update Scala and Python APIs" -a
+            echo "push=true" >> $GITHUB_OUTPUT
+          fi
+        shell: bash
+
+      - name: Push changes
+        if: ${{ steps.commit.outputs.push == 'true' }}
+        uses: ad-m/github-push-action@master
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          branch: ${{ github.ref }}

CHANGELOG

@@ -1,3 +1,57 @@
+=======
+6.0.0
+=======
+----------------
+New Features & Enhancements
+----------------
+* Introducing new large language models:
+  * OLMo model support (SPARKNLP-1006)
+  * Phi 3.5 Vision model support (SPARKNLP-1060)
+  * LLAVA model support (SPARKNLP-1033)
+  * CoHere model support (SPARKNLP-1032)
+  * Qwen2-VL model support (SPARKNLP-1077)
+  * Llama 3.2 Vision models (SPARKNLP-1078)
+  * Deepseek Janus model support (SPARKNLP-1088)
+  * Added LLAVA v1.5 7b quantized model
+  * Added StarCoder2 3b int8 model
+
+* New MultipleChoice Transformers:
+  * AlbertForMultipleChoice (SPARKNLP-1105)
+  * DistilBertForMultipleChoice (SPARKNLP-1106)
+  * RoBertaForMultipleChoice (SPARKNLP-1107)
+  * XlmRoBertaForMultipleChoice (SPARKNLP-1108)
+
+* New file format support:
+  * Excel files reader (SPARKNLP-1102)
+  * PowerPoint files reader (SPARKNLP-1103)
+  * PDF reader (SPARKNLP-1098)
+  * Text reader (SPARKNLP-1113)
+
+* Other improvements:
+  * AutoGGUFVisionModel for vision model support (SPARKNLP-1079)
+  * Added Extractor to SparkNLP (SPARKNLP-1109)
+  * Updated Python and Scala model names
+  * Improved error handling for AutoGGUF models
+
+----------------
+Bug Fixes
+----------------
+* Fixed typo in MXBAI notebook
+
+========
+5.5.3
+========
+----------------
+Bug Fixes & Enhancements
+----------------
+* BGEEmbeddings: The default pretrained model for BGEEmbeddings has been changed from "bge_base" to "bge_small_en_v1.5". Users relying on the old default will need to explicitly specify "bge_base" in the pretrained method.
+* Added useCLSToken parameter to allow users to choose between CLS token pooling and attention-based average pooling for sentence embeddings.
+* BGEEmbeddings: BGEEmbeddings now supports a `useCLSToken` parameter, which defaults to True. This affects the embedding calculation strategy. Existing users should verify their usage and potentially set this parameter explicitly.
+* Added HasClsTokenProperties in Scala: Introduced the HasClsTokenProperties trait in Scala providing useCLSToken parameter functionality for relevant annotators.
+* Fixing wrong padding in attention mask in `MPNet`, `BGE`, `E5`, `Mxbai`, `Nomic`, `SnowFlake`, and `UAE`. This resulted in wrong inference results in some cases and not equal to the ONNX version in transformers/sentence-transformers.
+* Various Performance Optimizations: Multiple changes across different models (Albert, Bart, CLIP, CamemBert, ConvNextClassifier, DeBerta, DistilBert, E5, Instructor, MPNet, Mxbai, Nomic, RoBerta, SnowFlake, UAE, ViTClassifier, VisionEncoderDecoder, Wav2Vec2, XlmRoBertaClassification, XlmRoberta) appear to focus on performance improvements and code cleanup, especially related to OpenVINO and ONNX inference. These may lead to faster inference times.
+* Fixing Llama3 download issue in Python.
+
 ========
 5.5.2
 ========

README.md

@@ -19,7 +19,7 @@
 
 Spark NLP is a state-of-the-art Natural Language Processing library built on top of Apache Spark. It provides **simple**, **performant** & **accurate** NLP annotations for machine learning pipelines that **scale** easily in a distributed environment.
 
-Spark NLP comes with **83000+** pretrained **pipelines** and **models** in more than **200+** languages.
+Spark NLP comes with **100000+** pretrained **pipelines** and **models** in more than **200+** languages.
 It also offers tasks such as **Tokenization**, **Word Segmentation**, **Part-of-Speech Tagging**, Word and Sentence **Embeddings**, **Named Entity Recognition**, **Dependency Parsing**, **Spell Checking**, **Text Classification**, **Sentiment Analysis**, **Token Classification**, **Machine Translation** (+180 languages), **Summarization**, **Question Answering**, **Table Question Answering**, **Text Generation**, **Image Classification**, **Image to Text (captioning)**, **Automatic Speech Recognition**, **Zero-Shot Learning**, and many more [NLP tasks](#features).
 
 **Spark NLP** is the only open-source NLP library in **production** that offers state-of-the-art transformers such as **BERT**, **CamemBERT**, **ALBERT**, **ELECTRA**, **XLNet**, **DistilBERT**, **RoBERTa**, **DeBERTa**, **XLM-RoBERTa**, **Longformer**, **ELMO**, **Universal Sentence Encoder**, **Llama-2**, **M2M100**, **BART**, **Instructor**, **E5**, **Google T5**, **MarianMT**, **OpenAI GPT2**, **Vision Transformers (ViT)**, **OpenAI Whisper**, **Llama**, **Mistral**, **Phi**, **Qwen2**, and many more not only to **Python** and **R**, but also to **JVM** ecosystem (**Java**, **Scala**, and **Kotlin**) at **scale** by extending **Apache Spark** natively.
@@ -63,7 +63,7 @@ $ java -version
 $ conda create -n sparknlp python=3.7 -y
 $ conda activate sparknlp
 # spark-nlp by default is based on pyspark 3.x
-$ pip install spark-nlp==5.5.2 pyspark==3.3.1
+$ pip install spark-nlp==6.0.0 pyspark==3.3.1
 ```
 
 In Python console or Jupyter `Python3` kernel:
@@ -129,10 +129,11 @@ For a quick example of using pipelines and models take a look at our official [d
 
 ### Apache Spark Support
 
-Spark NLP *5.5.2* has been built on top of Apache Spark 3.4 while fully supports Apache Spark 3.0.x, 3.1.x, 3.2.x, 3.3.x, 3.4.x, and 3.5.x
+Spark NLP *6.0.0* has been built on top of Apache Spark 3.4 while fully supports Apache Spark 3.0.x, 3.1.x, 3.2.x, 3.3.x, 3.4.x, and 3.5.x
 
 | Spark NLP | Apache Spark 3.5.x | Apache Spark 3.4.x | Apache Spark 3.3.x | Apache Spark 3.2.x | Apache Spark 3.1.x | Apache Spark 3.0.x | Apache Spark 2.4.x | Apache Spark 2.3.x |
 |-----------|--------------------|--------------------|--------------------|--------------------|--------------------|--------------------|--------------------|--------------------|
+| 6.0.x     | YES                | YES                | YES                | YES                | YES                | YES                | NO                 | NO                 |
 | 5.5.x     | YES                | YES                | YES                | YES                | YES                | YES                | NO                 | NO                 |
 | 5.4.x     | YES                | YES                | YES                | YES                | YES                | YES                | NO                 | NO                 |
 | 5.3.x     | YES                | YES                | YES                | YES                | YES                | YES                | NO                 | NO                 |
@@ -146,6 +147,7 @@ Find out more about `Spark NLP` versions from our [release notes](https://github
 
 | Spark NLP | Python 3.6 | Python 3.7 | Python 3.8 | Python 3.9 | Python 3.10| Scala 2.11 | Scala 2.12 |
 |-----------|------------|------------|------------|------------|------------|------------|------------|
+| 6.0.x     | NO         | YES        | YES        | YES        | YES        | NO         | YES        |
 | 5.5.x     | NO         | YES        | YES        | YES        | YES        | NO         | YES        |
 | 5.4.x     | NO         | YES        | YES        | YES        | YES        | NO         | YES        |
 | 5.3.x     | NO         | YES        | YES        | YES        | YES        | NO         | YES        |
@@ -157,7 +159,7 @@ Find out more about 4.x `SparkNLP` versions in our official [documentation](http
 
 ### Databricks Support
 
-Spark NLP 5.5.2 has been tested and is compatible with the following runtimes:
+Spark NLP 6.0.0 has been tested and is compatible with the following runtimes:
 
 | **CPU**            | **GPU**            |
 |--------------------|--------------------|
@@ -174,7 +176,7 @@ We are compatible with older runtimes. For a full list check databricks support
 
 ### EMR Support
 
-Spark NLP 5.5.2 has been tested and is compatible with the following EMR releases:
+Spark NLP 6.0.0 has been tested and is compatible with the following EMR releases:
 
 | **EMR Release**    |
 |--------------------|
@@ -184,6 +186,13 @@ Spark NLP 5.5.2 has been tested and is compatible with the following EMR release
 | emr-7.0.0          |
 | emr-7.1.0          |
 | emr-7.2.0          |
+| emr-7.3.0          |
+| emr-7.4.0          |
+| emr-7.5.0          |
+| emr-7.6.0          |
+| emr-7.7.0          |
+| emr-7.8.0          |
+
 
 We are compatible with older EMR releases. For a full list check EMR support in our official [documentation](https://sparknlp.org/docs/en/install#emr-support)
 
@@ -205,7 +214,7 @@ deployed to Maven central. To add any of our packages as a dependency in your ap
 from our official documentation.
 
 If you are interested, there is a simple SBT project for Spark NLP to guide you on how to use it in your
-projects [Spark NLP SBT S5.5.2r](https://github.com/maziyarpanahi/spark-nlp-starter)
+projects [Spark NLP SBT S6.0.0r](https://github.com/maziyarpanahi/spark-nlp-starter)
 
 ### Python
 
@@ -250,7 +259,7 @@ In Spark NLP we can define S3 locations to:
 
 Please check [these instructions](https://sparknlp.org/docs/en/install#s3-integration) from our official documentation.
 
-## Document5.5.2
+## Documentation
 
 ### Examples
 
@@ -283,7 +292,7 @@ the Spark NLP library:
     keywords = {Spark, Natural language processing, Deep learning, Tensorflow, Cluster},
     abstract = {Spark NLP is a Natural Language Processing (NLP) library built on top of Apache Spark ML. It provides simple, performant & accurate NLP annotations for machine learning pipelines that can scale easily in a distributed environment. Spark NLP comes with 1100+ pretrained pipelines and models in more than 192+ languages. It supports nearly all the NLP tasks and modules that can be used seamlessly in a cluster. Downloaded more than 2.7 million times and experiencing 9x growth since January 2020, Spark NLP is used by 54% of healthcare organizations as the world’s most widely used NLP library in the enterprise.}
     }
-}5.5.2
+}
 ```
 
 ## Community support

build.sbt

@@ -6,7 +6,7 @@ name := getPackageName(is_silicon, is_gpu, is_aarch64)
 
 organization := "com.johnsnowlabs.nlp"
 
-version := "5.5.2"
+version := "6.0.0"
 
 (ThisBuild / scalaVersion) := scalaVer
 
@@ -163,7 +163,8 @@ lazy val utilDependencies = Seq(
   poiDocx
     exclude ("org.apache.logging.log4j", "log4j-api"),
   scratchpad
-    exclude ("org.apache.logging.log4j", "log4j-api")
+    exclude ("org.apache.logging.log4j", "log4j-api"),
+  pdfBox
 )
 
 lazy val typedDependencyParserDependencies = Seq(junit)

conda/meta.yaml

@@ -1,13 +1,13 @@
 {% set name = "spark-nlp" %}
-{% set version = "5.5.2" %}
+{% set version = "6.0.0" %}
 
 package:
   name: {{ name|lower }}
   version: {{ version }}
 
 source:
   url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/spark-nlp-{{ version }}.tar.gz
-  sha256: b620487092256d02bf8d277374c564cd22384d437c97a4bb5b3b0f1fdfc696e8
+  sha256: 58f4f530105d5c5522fc37ce4d3b63af1e2463b43e000cf69838e0854b468365
 
 build:
   noarch: python

docs/_layouts/landing.html

@@ -201,7 +201,7 @@ <h3 class="grey h3_title">{{ _section.title }}</h3>
                   <div class="highlight-box">
     {% highlight bash %}
     # Using PyPI
-    $ pip install spark-nlp==5.5.2
+    $ pip install spark-nlp==6.0.0
 
     # Using Anaconda/Conda
     $ conda install -c johnsnowlabs spark-nlp