java_nightly.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: Upload Java Nightly builds

on:
  workflow_dispatch:
    inputs:
      prefix:
        description: Job prefix to use.
        required: false
        default: ''
      keep:
        description: Number of versions to keep.
        required: false
        default: 14
  schedule:
    - cron: '0 14 * * *'

permissions:
  contents: read

jobs:
  upload:
    if: github.repository == 'apache/arrow'
    env:
      PREFIX: ${{ github.event.inputs.prefix || ''}}
      CROSSBOW_GITHUB_TOKEN: ${{ github.token }}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Arrow
        uses: actions/checkout@v3
        with:
          fetch-depth: 1
          path: arrow
          repository: apache/arrow
          ref: master
          submodules: recursive
      - name: Checkout Crossbow
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
          path: crossbow
          repository: ursacomputing/crossbow
          ref: master
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          cache: 'pip'
          python-version: 3.8
      - name: Install Archery
        shell: bash
        run: pip install -e arrow/dev/archery[all]
      - run: mkdir -p binaries
      - name: Download Artifacts
        run: |
          if [ -z $PREFIX ]; then
            PREFIX=nightly-packaging-$(date +%Y-%m-%d)-0
          fi
          echo $PREFIX
          archery crossbow download-artifacts -f java-jars -t binaries  $PREFIX
      - name: Cache Repo
        uses: actions/cache@v3
        with:
          path: repo
          key: java-nightly-${{ github.run_id }}
          restore-keys: java-nightly
      - name: Sync from Remote
        uses: ./arrow/.github/actions/sync-nightlies
        with:
          switches: -avzh --update --delete --progress
          local_path: repo
          remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/java
          remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }}
          remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
          remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
          remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
          remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }}
      - shell: bash
        name: Show local repo sync from remote
        run: |
          for i in `ls -t repo/org/apache/arrow`; do
            echo "- $i: $(find repo/org/apache/arrow/$i -mindepth 1 -maxdepth 1 -type d \
            | wc -l \
            | xargs) versions available"
          done
      - shell: bash
        name: Build Repository
        run: |
          DATE=$(date +%Y-%m-%d)
          if [ -z $PREFIX ]; then
            PREFIX=nightly-packaging-${DATE}-0
          fi
          PATTERN_TO_GET_LIB_AND_VERSION='([a-z].+)-([0-9]+.[0-9]+.[0-9]+-SNAPSHOT)'
          mkdir -p repo/org/apache/arrow/
          for LIBRARY in $(ls binaries/$PREFIX/java-jars | grep -E '.jar|.pom' | grep SNAPSHOT); do
            [[ $LIBRARY =~ $PATTERN_TO_GET_LIB_AND_VERSION ]]
            mkdir -p repo/org/apache/arrow/${BASH_REMATCH[1]}/${BASH_REMATCH[2]}
            mkdir -p repo/org/apache/arrow/${BASH_REMATCH[1]}/${DATE}
            # Copy twice to maintain a latest snapshot and some earlier versions
            cp binaries/$PREFIX/java-jars/$LIBRARY repo/org/apache/arrow/${BASH_REMATCH[1]}/${BASH_REMATCH[2]}
            touch repo/org/apache/arrow/${BASH_REMATCH[1]}/${BASH_REMATCH[2]}
            cp binaries/$PREFIX/java-jars/$LIBRARY repo/org/apache/arrow/${BASH_REMATCH[1]}/${DATE}
            echo "Artifacts $LIBRARY configured"
          done
      - name: Prune Repository
        shell: bash
        env:
          KEEP: ${{ github.event.inputs.keep || 14 }}
        run: |
          for i in `ls -t repo/org/apache/arrow`; do
            find repo/org/apache/arrow/$i -mindepth 1 -maxdepth 1 -type d -print0 \
            | xargs -0 ls -t -d \
            | tail -n +$((KEEP + 1)) \
            | xargs rm -rf
          done
      - name: Show repo contents
        run: tree repo
      - name: Sync to Remote
        if: ${{ github.repository == 'apache/arrow' }}
        uses: ./arrow/.github/actions/sync-nightlies
        with:
          upload: true
          switches: -avzh --update --delete --progress
          local_path: repo
          remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/java
          remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }}
          remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
          remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
          remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
          remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }}