ARROW-7685: [Developer] Add support for GitHub Actions to Crossbow

If we use GitHub Actions, we don't need to set additional secret
variables to crossbow repository because we can use built-in
GITHUB_TOKEN secret variable.

We'll be able to use self-hosted runner with GitHub Actions. It means
that we'll be able to build .deb/.rpm packages on ARM machine instead
on QEMU.

linux-packages tasks are migrated to GitHub Actions from Azure
Pipelines as an example.

Closes #6286 from kou/crossbow-github-actions and squashes the following commits:

3a75ac4 <Krisztián Szűcs> set workflow name
35dd859 <Krisztián Szűcs> use CROSSBOW_GITHUB_TOKEN secret
75c574a <Sutou Kouhei>  Add support for GitHub Actions to Crossbow

Lead-authored-by: Sutou Kouhei <kou@clear-code.com>
Co-authored-by: Krisztián Szűcs <szucs.krisztian@gmail.com>
Signed-off-by: Krisztián Szűcs <szucs.krisztian@gmail.com>

dev/tasks/crossbow.py

@@ -113,6 +113,7 @@ def unflatten_tree(files):
 # - circle requires the configuration to be present on all branch, even ones
 #   that are configured to be skipped
 # - azure skips branches without azure-pipelines.yml by default
+# - github skips branches without .github/workflows/ by default
 
 _default_travis_yml = """
 branches:
@@ -736,7 +737,7 @@ class Task(Serializable):
 
     def __init__(self, platform, ci, template, artifacts=None, params=None):
         assert platform in {'win', 'osx', 'linux'}
-        assert ci in {'circle', 'travis', 'appveyor', 'azure'}
+        assert ci in {'circle', 'travis', 'appveyor', 'azure', 'github'}
         self.ci = ci
         self.platform = platform
         self.template = template
@@ -766,7 +767,8 @@ def filename(self):
             'circle': '.circleci/config.yml',
             'travis': '.travis.yml',
             'appveyor': 'appveyor.yml',
-            'azure': 'azure-pipelines.yml'
+            'azure': 'azure-pipelines.yml',
+            'github': '.github/workflows/crossbow.yml',
         }
         return config_files[self.ci]
 

dev/tasks/linux-packages/github.linux.arm64.yml

@@ -0,0 +1,94 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# NOTE: must set "Crossbow" as name to have the badge links working in the
+# github comment reports!
+name: Crossbow
+
+on:
+  push:
+    branches:
+      - "*-github-*"
+
+jobs:
+  package:
+    name: Package
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Ruby
+        uses: actions/setup-ruby@master
+      - name: Checkout Arrow
+        run: |
+          git clone --no-checkout {{ arrow.remote }} arrow
+          git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
+          git -C arrow checkout FETCH_HEAD
+          git -C arrow submodule update --init --recursive
+      # We can remove this with binfmt-support 2.1.7 or later and
+      # qemu-user-static 2.12 or later. It requires Debian buster or later,
+      # or Ubuntu 18.10 or later.
+      - name: Prepare qemu-user-static
+        run: |
+          sudo apt install -y qemu-user-static unar
+          wget http://archive.ubuntu.com/ubuntu/pool/universe/q/qemu/qemu-user-static_3.1+dfsg-2ubuntu3.7_amd64.deb
+          unar *.deb
+          rm *.deb
+          pushd qemu-user-static*
+          unar data.*
+          for dockerfile in ../arrow/dev/tasks/linux-packages/*/*/*/Dockerfile; do
+            cp data/usr/bin/qemu-aarch64-static $(dirname ${dockerfile})
+          done
+          popd
+      - name: Build
+        run: |
+          set -e
+          pushd arrow/dev/tasks/linux-packages
+          rake version:update
+          {{ build_command }}
+          popd
+        env:
+          ARROW_VERSION: {{ arrow.version }}
+      # Using GitHub release tries to find a common ancestor between the
+      # currently pushed tag and the latest tag of the GitHub repository
+      # (don't know why).
+      # The tag upload took 43 minutes because of this scan, so use an
+      # alternative upload script.
+      - name: Set up Python
+        uses: actions/setup-python@master
+      - name: Set up Crossbow
+        run: |
+          pip install \
+            click \
+            github3.py \
+            jinja2 \
+            jira \
+            pygit2 \
+            ruamel.yaml \
+            setuptools_scm \
+            toolz
+      - name: Upload artifacts
+        run: |
+          python arrow/dev/tasks/crossbow.py \
+            --queue-path . \
+            --queue-remote {{ queue.remote_url }} \
+            upload-artifacts \
+          {%- for extension in upload_extensions %}
+            --pattern "**/*{{ extension }}" \
+          {%- endfor %}
+            --sha {{ task.branch }} \
+            --tag {{ task.tag }}
+        env:
+          CROSSBOW_GITHUB_TOKEN: {{ '${{ secrets.CROSSBOW_GITHUB_TOKEN }}' }}

dev/tasks/linux-packages/github.linux.yml

@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# NOTE: must set "Crossbow" as name to have the badge links working in the
+# github comment reports!
+name: Crossbow
+
+on:
+  push:
+    branches:
+      - "*-github-*"
+
+jobs:
+  package:
+    name: Package
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Ruby
+        uses: actions/setup-ruby@master
+      - name: Checkout Arrow
+        run: |
+          git clone --no-checkout {{ arrow.remote }} arrow
+          git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
+          git -C arrow checkout FETCH_HEAD
+          git -C arrow submodule update --init --recursive
+      - name: Build
+        run: |
+          set -e
+          pushd arrow/dev/tasks/linux-packages
+          rake version:update
+          {{ build_command }}
+          popd
+        env:
+          ARROW_VERSION: {{ arrow.version }}
+      # Using GitHub release tries to find a common ancestor between the
+      # currently pushed tag and the latest tag of the GitHub repository
+      # (don't know why).
+      # The tag upload took 43 minutes because of this scan, so use an
+      # alternative upload script.
+      - name: Set up Python
+        uses: actions/setup-python@master
+      - name: Set up Crossbow
+        run: |
+          pip install \
+            click \
+            github3.py \
+            jinja2 \
+            jira \
+            pygit2 \
+            ruamel.yaml \
+            setuptools_scm \
+            toolz
+      - name: Upload artifacts
+        run: |
+          python arrow/dev/tasks/crossbow.py \
+            --queue-path . \
+            --queue-remote {{ queue.remote_url }} \
+            upload-artifacts \
+          {%- for extension in upload_extensions %}
+            --pattern "**/*{{ extension }}" \
+          {%- endfor %}
+            --sha {{ task.branch }} \
+            --tag {{ task.tag }}
+        env:
+          CROSSBOW_GITHUB_TOKEN: {{ '${{ secrets.CROSSBOW_GITHUB_TOKEN }}' }}