Native JNI S3 Paddle #1023
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Native JNI S3 Paddle | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 10 * * *' | |
| jobs: | |
| build-paddle-jni-cpu: | |
| if: github.repository == 'deepjavalibrary/djl' | |
| runs-on: ${{ matrix.operating-system }} | |
| strategy: | |
| matrix: | |
| operating-system: [ macos-latest, windows-latest ] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up JDK 11 | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: 'corretto' | |
| java-version: 11 | |
| - uses: actions/cache@v3 | |
| with: | |
| path: ~/.gradle/caches | |
| key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }} | |
| restore-keys: | | |
| ${{ runner.os }}-gradle- | |
| - name: Release JNI prep | |
| run: ./gradlew :engines:paddlepaddle:paddlepaddle-native:compileJNI | |
| - name: Upload compiled jni library | |
| uses: actions/upload-artifact@v3 | |
| if: always() | |
| with: | |
| name: jnilib-${{ runner.os }} | |
| path: engines/paddlepaddle/paddlepaddle-native/jnilib | |
| # A few things tried to build GPU jni (2.0.2) shared library: | |
| # 1. cpu JNI .so file works fine with cu101 and cu102 | |
| # 2. cu101 and cu102 generate identical .so file, no point to build both of them | |
| # 3. centos:7 + GCC 8 is broken: symbol not found | |
| # 4. amazonlinux:2 + GCC 7.3.1 is broken: symbol not found (CPU works fine) | |
| # 5. ubuntu:18.04 + GCC 7.5.1 works fine | |
| # 6. ubuntu:16.04 + GCC 8 works fine (old pipeline use this, assume this works) | |
| build-paddle-jni-linux: | |
| if: github.repository == 'deepjavalibrary/djl' | |
| runs-on: ubuntu-latest | |
| container: amazonlinux:2 | |
| steps: | |
| - name: Install Environment | |
| run: | | |
| yum -y update | |
| yum -y groupinstall "Development Tools" | |
| yum -y install patch cmake3 | |
| ln -sf /usr/bin/cmake3 /usr/bin/cmake | |
| pip3 install awscli --upgrade | |
| - uses: actions/checkout@v3 | |
| - name: Set up JDK 11 | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: 'corretto' | |
| java-version: 11 | |
| - uses: actions/cache@v3 | |
| with: | |
| path: ~/.gradle/caches | |
| key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }} | |
| restore-keys: | | |
| ${{ runner.os }}-gradle- | |
| - name: Release JNI prep | |
| run: | | |
| ./gradlew :engines:paddlepaddle:paddlepaddle-native:compileJNI | |
| - name: Upload compiled jni library | |
| uses: actions/upload-artifact@v3 | |
| if: always() | |
| with: | |
| name: jnilib-${{ runner.os }} | |
| path: engines/paddlepaddle/paddlepaddle-native/jnilib | |
| publish: | |
| if: github.repository == 'deepjavalibrary/djl' | |
| runs-on: ubuntu-latest | |
| needs: [ build-paddle-jni-cpu, build-paddle-jni-linux ] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Download compiledJNI Mac | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: jnilib-macOS | |
| path: jnilib | |
| - name: Download compiledJNI Windows | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: jnilib-Windows | |
| path: jnilib | |
| - name: Download compiledJNI Linux CPU | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: jnilib-Linux | |
| path: jnilib | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v1-node16 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-east-2 | |
| - name: Copy files to S3 with the AWS CLI | |
| run: | | |
| PADDLE_VERSION="$(cat gradle.properties | awk -F '=' '/paddlepaddle_version/ {print $2}')" | |
| aws s3 sync jnilib s3://djl-ai/publish/paddlepaddle-${PADDLE_VERSION}/jnilib | |
| aws cloudfront create-invalidation --distribution-id E371VB8JQ6NRVY --paths "/paddlepaddle-${PADDLE_VERSION}/jnilib*" |