Skip to content

Native S3 Huggingface tokenizers #199

Native S3 Huggingface tokenizers

Native S3 Huggingface tokenizers #199

name: Native S3 Huggingface tokenizers
on:
workflow_dispatch:
push:
branches:
- 'master'
paths:
- extensions/tokenizers/rust/**
jobs:
build-tokenizers-jni-osx:
runs-on: macos-13
steps:
- uses: actions/checkout@v4
- name: Set up JDK 17
uses: actions/setup-java@v4
with:
distribution: 'corretto'
java-version: 17
- uses: actions-rs/toolchain@v1
with:
toolchain: stable
- uses: actions/cache@v4
with:
path: ~/.gradle/caches
key: ${{ runner.os }}-gradle-${{ hashFiles('*/build.gradle.kts', 'engines/**/build.gradle.kts', 'extensions/**/build.gradle.kts') }}
restore-keys: |
${{ runner.os }}-gradle-
- name: Release JNI prep
run: |
./gradlew :extensions:tokenizers:compileJNI
PYTORCH_VERSION=2.2.2 ./gradlew -Pjni :extensions:tokenizers:test
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-east-2
- name: Copy files to S3 with the AWS CLI
run: |
DJL_VERSION=$(awk -F '=' '/djl / {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)
TOKENIZERS_VERSION="$(awk -F '=' '/tokenizers/ {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)"
aws s3 sync extensions/tokenizers/jnilib/$DJL_VERSION/osx-x86_64 s3://djl-ai/publish/tokenizers/${TOKENIZERS_VERSION}/jnilib/$DJL_VERSION/osx-x86_64/
aws cloudfront create-invalidation --distribution-id E371VB8JQ6NRVY --paths "/tokenizers/${TOKENIZERS_VERSION}/jnilib/*"
build-tokenizers-jni-linux:
runs-on: ubuntu-latest
container:
image: amazonlinux:2
env:
JAVA_HOME: /usr/lib/jvm/java-17-amazon-corretto
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
steps:
- name: Install Environment
run: |
yum -y groupinstall "Development Tools"
yum -y install patch perl-IPC-Cmd cmake3
yum -y install java-17-amazon-corretto-devel
ln -s /usr/bin/cmake3 /usr/bin/cmake
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
pip3 install awscli wheel setuptools --upgrade
- uses: taiki-e/checkout-action@v1
- name: Release JNI prep
run: |
source "$HOME/.cargo/env"
./gradlew :extensions:tokenizers:compileJNI
PYTORCH_PRECXX11=true ./gradlew -Pjni :extensions:tokenizers:test
- name: Build djl-converter wheel
working-directory: extensions/tokenizers/src/main/python/
run: ./setup.py bdist_wheel
- name: Copy files to S3 with the AWS CLI
run: |
DJL_VERSION=$(awk -F '=' '/djl / {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)
TOKENIZERS_VERSION="$(awk -F '=' '/tokenizers/ {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)"
aws s3 sync extensions/tokenizers/jnilib/$DJL_VERSION/linux-x86_64/cpu s3://djl-ai/publish/tokenizers/${TOKENIZERS_VERSION}/jnilib/$DJL_VERSION/linux-x86_64/cpu/
aws cloudfront create-invalidation --distribution-id E371VB8JQ6NRVY --paths "/tokenizers/${TOKENIZERS_VERSION}/jnilib/*"
aws s3 cp extensions/tokenizers/src/main/python/dist/djl_converter-*.whl s3://djl-ai/publish/djl_converter/
aws cloudfront create-invalidation --distribution-id E371VB8JQ6NRVY --paths "/djl_converter/*"
build-tokenizer-jni-windows:
runs-on: windows-latest
steps:
- name: Install Environment
run: |
choco install -y mingw
- uses: actions-rs/toolchain@v1
with:
toolchain: stable
- uses: actions/checkout@v4
- name: Set up JDK 17
uses: actions/setup-java@v4
with:
distribution: 'corretto'
java-version: 17
- uses: actions/cache@v4
with:
path: ~/.gradle/caches
key: ${{ runner.os }}-gradle-${{ hashFiles('*/build.gradle.kts', 'engines/**/build.gradle.kts', 'extensions/**/build.gradle.kts') }}
restore-keys: |
${{ runner.os }}-gradle-
- name: Release CPU JNI
shell: cmd
run: |
call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" amd64
gradlew :extensions:tokenizer:compileJNI
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-east-2
- name: Copy files to S3 with the AWS CLI
shell: bash
run: |
DJL_VERSION=$(awk -F '=' '/djl / {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)
TOKENIZERS_VERSION="$(awk -F '=' '/tokenizers/ {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)"
aws s3 sync extensions/tokenizers/jnilib/$DJL_VERSION/win-x86_64 s3://djl-ai/publish/tokenizers/${TOKENIZERS_VERSION}/jnilib/$DJL_VERSION/win-x86_64/
aws cloudfront create-invalidation --distribution-id E371VB8JQ6NRVY --paths "/tokenizers/${TOKENIZERS_VERSION}/jnilib/*"
build-tokenizers-jni-arm64-osx:
if: ${{ github.repository == 'deepjavalibrary/djl' && always() }}
runs-on: macos-latest-xlarge
steps:
- uses: actions/checkout@v4
- name: Set up JDK 17
uses: actions/setup-java@v4
with:
java-version: 17
distribution: corretto
architecture: aarch64
- uses: actions-rs/toolchain@v1
with:
toolchain: stable
- uses: actions/cache@v4
with:
path: ~/.gradle/caches
key: ${{ runner.os }}-gradle-${{ hashFiles('*/build.gradle.kts', 'engines/**/build.gradle.kts', 'extensions/**/build.gradle.kts') }}
restore-keys: |
${{ runner.os }}-gradle-
- name: Release JNI prep
run: |
./gradlew :extensions:tokenizers:compileJNI
./gradlew -Pjni :extensions:tokenizers:test
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-east-2
- name: Copy files to S3 with the AWS CLI
shell: bash
run: |
DJL_VERSION=$(awk -F '=' '/djl / {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)
TOKENIZERS_VERSION="$(awk -F '=' '/tokenizers/ {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)"
aws s3 sync extensions/tokenizers/jnilib/$DJL_VERSION/osx-aarch64 s3://djl-ai/publish/tokenizers/${TOKENIZERS_VERSION}/jnilib/$DJL_VERSION/osx-aarch64/
aws cloudfront create-invalidation --distribution-id E371VB8JQ6NRVY --paths "/tokenizers/${TOKENIZERS_VERSION}/jnilib/*"
create-runners:
if: github.repository == 'deepjavalibrary/djl'
runs-on: [ self-hosted, scheduler ]
steps:
- name: Create new Graviton instance
id: create_aarch64
run: |
cd /home/ubuntu/djl_benchmark_script/scripts
token=$( curl -X POST -H "Authorization: token ${{ secrets.ACTION_RUNNER_PERSONAL_TOKEN }}" \
https://api.github.com/repos/deepjavalibrary/djl/actions/runners/registration-token \
--fail \
| jq '.token' | tr -d '"' )
./start_instance.sh action_graviton $token djl
- name: Create new G5 instance
id: create_gpu
run: |
cd /home/ubuntu/djl_benchmark_script/scripts
token=$( curl -X POST -H "Authorization: token ${{ secrets.ACTION_RUNNER_PERSONAL_TOKEN }}" \
https://api.github.com/repos/deepjavalibrary/djl/actions/runners/registration-token \
--fail \
| jq '.token' | tr -d '"' )
./start_instance.sh action_g5 $token djl
outputs:
aarch64_instance_id: ${{ steps.create_aarch64.outputs.action_graviton_instance_id }}
gpu_instance_id: ${{ steps.create_gpu.outputs.action_g5_instance_id }}
build-tokenizer-jni-aarch64:
if: ${{ github.repository == 'deepjavalibrary/djl' && always() }}
runs-on: [ self-hosted, aarch64 ]
timeout-minutes: 30
needs: create-runners
container:
image: amazonlinux:2
env:
JAVA_HOME: /usr/lib/jvm/java-17-amazon-corretto.aarch64
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
steps:
- name: Install Environment
run: |
yum -y groupinstall "Development Tools"
yum -y install patch perl-IPC-Cmd cmake3
yum -y install java-17-amazon-corretto-devel
ln -s /usr/bin/cmake3 /usr/bin/cmake
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
pip3 install awscli --upgrade
- uses: taiki-e/checkout-action@v1
- name: Release JNI prep
run: |
source "$HOME/.cargo/env"
./gradlew :extensions:tokenizers:compileJNI
PYTORCH_PRECXX11=true ./gradlew -Pjni :extensions:tokenizers:test
- name: Copy files to S3 with the AWS CLI
run: |
DJL_VERSION=$(awk -F '=' '/djl / {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)
TOKENIZERS_VERSION="$(awk -F '=' '/tokenizers/ {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)"
aws s3 sync extensions/tokenizers/jnilib/$DJL_VERSION/linux-aarch64 s3://djl-ai/publish/tokenizers/${TOKENIZERS_VERSION}/jnilib/$DJL_VERSION/linux-aarch64/
aws cloudfront create-invalidation --distribution-id E371VB8JQ6NRVY --paths "/tokenizers/${TOKENIZERS_VERSION}/jnilib/*"
build-tokenizers-jni-cu124:
if: github.repository == 'deepjavalibrary/djl'
runs-on: [ self-hosted, g5 ]
timeout-minutes: 30
needs: create-runners
container:
image: nvidia/cuda:12.4.1-cudnn-devel-ubuntu20.04
options: --gpus all --runtime=nvidia
steps:
- name: Install Environment
run: |
apt-get -y update
apt-get -y install curl git
- name: Set up Python3
uses: actions/setup-python@v5
with:
python-version: '3.10'
- name: Install Python Dependencies
run: |
python -m pip install --upgrade pip
pip install awscli wheel setuptools --upgrade
- uses: actions-rs/toolchain@v1
with:
toolchain: stable
- uses: actions/checkout@v4
- name: Set up JDK 17
uses: actions/setup-java@v4
with:
distribution: 'corretto'
java-version: 17
- uses: actions/cache@v3
with:
path: ~/.gradle/caches
key: ${{ runner.os }}-gradle-${{ hashFiles('*/build.gradle.kts', 'engines/**/build.gradle.kts', 'extensions/**/build.gradle.kts') }}
restore-keys: |
${{ runner.os }}-gradle-
- name: Release JNI prep
run: |
CUDA_VERSION=cu124
. "$HOME/.cargo/env"
./gradlew :extensions:tokenizers:compileJNI -Pcuda=$CUDA_VERSION
./gradlew -Pjni :extensions:tokenizers:test
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v2
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-east-2
- name: Copy files to S3 with the AWS CLI
run: |
DJL_VERSION=$(awk -F '=' '/djl / {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)
TOKENIZERS_VERSION="$(awk -F '=' '/tokenizers/ {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)"
aws s3 sync extensions/tokenizers/jnilib/$DJL_VERSION/linux-x86_64/cu124 s3://djl-ai/publish/tokenizers/${TOKENIZERS_VERSION}/jnilib/$DJL_VERSION/linux-x86_64/cu124/
aws cloudfront create-invalidation --distribution-id E371VB8JQ6NRVY --paths "/tokenizers/${TOKENIZERS_VERSION}/jnilib/*"
stop-runners:
if: ${{ github.repository == 'deepjavalibrary/djl' && always() }}
runs-on: [ self-hosted, scheduler ]
needs: [ create-runners, build-tokenizer-jni-aarch64, build-tokenizers-jni-cu124 ]
steps:
- name: Stop all instances
run: |
cd /home/ubuntu/djl_benchmark_script/scripts
instance_id=${{ needs.create-runners.outputs.aarch64_instance_id }}
./stop_instance.sh $instance_id
cd /home/ubuntu/djl_benchmark_script/scripts
instance_id=${{ needs.create-runners.outputs.gpu_instance_id }}
./stop_instance.sh $instance_id