ARROW-8463: [CI] Balance the nightly test builds between CircleCI, Azure and Github

dev/tasks/docker-tests/github.linux.yml

@@ -0,0 +1,65 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# NOTE: must set "Crossbow" as name to have the badge links working in the
+# github comment reports!
+name: Crossbow
+
+on:
+  push:
+    branches:
+      - "*-github-*"
+
+jobs:
+  test:
+    name: Docker Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Arrow
+        shell: bash
+        run: |
+          git clone --no-checkout {{ arrow.remote }} arrow
+          git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
+          git -C arrow checkout FETCH_HEAD
+          git -C arrow submodule update --init --recursive
+      - name: Run Docker Build
+        shell: bash
+        run: |
+          pushd arrow
+          {% if env is defined %}
+          {%- for key, value in env.items() %}
+            export {{ key }}={{ value }}
+          {%- endfor %}
+          {% endif %}
+
+          {% if build is defined %}
+          {%- for image in build %}
+            docker-compose pull --ignore-pull-failures {{ image }}
+            docker-compose build {{ image }}
+          {%- endfor %}
+          {% endif %}
+
+          {% if nocache is defined %}
+          {%- for image in nocache %}
+            docker-compose build --no-cache {{ image }}
+          {%- endfor %}
+          {% endif %}
+
+          {%- for image in run %}
+            docker-compose run --rm -e SETUPTOOLS_SCM_PRETEND_VERSION="{{ arrow.no_rc_version }}" {{ image }}
+          {%- endfor %}
+          popd

dev/tasks/tasks.yml

@@ -1602,29 +1602,29 @@ tasks:
   ############################## Docker tests #################################
 
   test-conda-cpp:
-    ci: circle
+    ci: github
     platform: linux
-    template: docker-tests/circle.linux.yml
+    template: docker-tests/github.linux.yml
     params:
       build:
         - conda-cpp
       run:
         - conda-cpp
 
   test-conda-cpp-valgrind:
-    ci: circle
+    ci: github
     platform: linux
-    template: docker-tests/circle.linux.yml
+    template: docker-tests/github.linux.yml
     params:
       build:
         - conda-cpp
       run:
         - conda-cpp-valgrind
 
   test-debian-10-cpp:
-    ci: circle
+    ci: github
     platform: linux
-    template: docker-tests/circle.linux.yml
+    template: docker-tests/github.linux.yml
     params:
       env:
         DEBIAN: 10
@@ -1634,9 +1634,9 @@ tasks:
         - debian-cpp
 
   test-ubuntu-16.04-cpp:
-    ci: circle
+    ci: github
     platform: linux
-    template: docker-tests/circle.linux.yml
+    template: docker-tests/github.linux.yml
     params:
       env:
         UBUNTU: 16.04
@@ -1646,9 +1646,9 @@ tasks:
         - ubuntu-cpp
 
   test-ubuntu-18.04-cpp:
-    ci: circle
+    ci: github
     platform: linux
-    template: docker-tests/circle.linux.yml
+    template: docker-tests/github.linux.yml
     params:
       env:
         UBUNTU: 18.04
@@ -1658,9 +1658,9 @@ tasks:
         - ubuntu-cpp
 
   test-fedora-30-cpp:
-    ci: circle
+    ci: github
     platform: linux
-    template: docker-tests/circle.linux.yml
+    template: docker-tests/github.linux.yml
     params:
       env:
         FEDORA: 30
@@ -1670,39 +1670,39 @@ tasks:
         - fedora-cpp
 
   test-ubuntu-18.04-cpp-release:
-    ci: circle
+    ci: github
     platform: linux
-    template: docker-tests/circle.linux.yml
+    template: docker-tests/github.linux.yml
     params:
       build:
         - ubuntu-cpp
       run:
         - -e ARROW_BUILD_TYPE=release ubuntu-cpp
 
   test-ubuntu-18.04-cpp-static:
-    ci: circle
+    ci: github
     platform: linux
-    template: docker-tests/circle.linux.yml
+    template: docker-tests/github.linux.yml
     params:
       build:
         - ubuntu-cpp
       run:
         - -e ARROW_BUILD_SHARED=OFF -e ARROW_BUILD_STATIC=ON -e ARROW_TEST_LINKAGE=static ubuntu-cpp
 
   test-ubuntu-18.04-cpp-cmake32:
-    ci: circle
+    ci: github
     platform: linux
-    template: docker-tests/circle.linux.yml
+    template: docker-tests/github.linux.yml
     params:
       build:
         - ubuntu-cpp-cmake32
       run:
         - ubuntu-cpp-cmake32
 
   test-debian-c-glib:
-    ci: circle
+    ci: github
     platform: linux
-    template: docker-tests/circle.linux.yml
+    template: docker-tests/github.linux.yml
     params:
       build:
         - debian-cpp
@@ -1711,9 +1711,9 @@ tasks:
         - debian-c-glib
 
   test-ubuntu-c-glib:
-    ci: circle
+    ci: github
     platform: linux
-    template: docker-tests/circle.linux.yml
+    template: docker-tests/github.linux.yml
     params:
       build:
         - ubuntu-cpp
@@ -1722,9 +1722,9 @@ tasks:
         - ubuntu-c-glib
 
   test-debian-ruby:
-    ci: circle
+    ci: azure
     platform: linux
-    template: docker-tests/circle.linux.yml
+    template: docker-tests/azure.linux.yml
     params:
       build:
         - debian-cpp
@@ -1734,9 +1734,9 @@ tasks:
         - debian-ruby
 
   test-ubuntu-ruby:
-    ci: circle
+    ci: azure
     platform: linux
-    template: docker-tests/circle.linux.yml
+    template: docker-tests/azure.linux.yml
     params:
       build:
         - ubuntu-cpp
@@ -1746,9 +1746,9 @@ tasks:
         - ubuntu-ruby
 
   test-conda-python-3.6:
-    ci: circle
+    ci: azure
     platform: linux
-    template: docker-tests/circle.linux.yml
+    template: docker-tests/azure.linux.yml
     params:
       env:
         PYTHON: 3.6
@@ -1759,9 +1759,9 @@ tasks:
         - conda-python
 
   test-conda-python-3.7:
-    ci: circle
+    ci: azure
     platform: linux
-    template: docker-tests/circle.linux.yml
+    template: docker-tests/azure.linux.yml
     params:
       env:
         PYTHON: 3.7
@@ -1772,9 +1772,9 @@ tasks:
         - conda-python
 
   test-conda-python-3.8:
-    ci: circle
+    ci: azure
     platform: linux
-    template: docker-tests/circle.linux.yml
+    template: docker-tests/azure.linux.yml
     params:
       env:
         PYTHON: 3.8
@@ -1785,9 +1785,9 @@ tasks:
         - conda-python
 
   test-debian-10-python-3:
-    ci: circle
+    ci: azure
     platform: linux
-    template: docker-tests/circle.linux.yml
+    template: docker-tests/azure.linux.yml
     params:
       env:
         DEBIAN: 10
@@ -1798,9 +1798,9 @@ tasks:
         - debian-python
 
   test-ubuntu-18.04-python-3:
-    ci: circle
+    ci: azure
     platform: linux
-    template: docker-tests/circle.linux.yml
+    template: docker-tests/azure.linux.yml
     params:
       env:
         UBUNTU: 18.04
@@ -1811,9 +1811,9 @@ tasks:
         - ubuntu-python
 
   test-fedora-30-python-3:
-    ci: circle
+    ci: azure
     platform: linux
-    template: docker-tests/circle.linux.yml
+    template: docker-tests/azure.linux.yml
     params:
       env:
         FEDORA: 30