Improve master -> quarkus-master branch scheduled synchronization wor…

…kflow

.github/test-categories.yaml

@@ -0,0 +1,113 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Association of test categories and integration test modules. Used by GitHub workflows.
+categories:
+  cloud:
+    - aws
+    - aws2
+    - azure
+    - consul
+  core:
+    - core
+    - core-impl
+    - core-main
+    - core-main-caffeine-lrucache
+    - core-main-collector
+    - core-main-xml-io
+    - core-main-xml-jaxb
+  database:
+    - couchdb
+    - influxdb
+    - jdbc
+    - kudu
+    - mongodb
+    - sql
+  dataformats:
+    - base64
+    - bindy
+    - csv
+    - dataformat
+    - jaxb
+    - xstream
+    - soap
+  foundation:
+    - bean
+    - controlbus
+    - exec
+    - file
+    - hystrix
+    - jsonpath
+    - quartz
+    - scheduler
+    - seda
+    - send-dynamic-http
+    - ref
+  memoryhog-dataformat-json:
+    - dataformats-json
+  memoryhog-dozer:
+    - dozer
+  memoryhog-infinispan:
+    - dozer
+  memoryhog-olingo4:
+    - olingo4
+  memoryhog-xml:
+    - rest-binding-mode-xml
+    - xml
+  messaging:
+    - activemq
+    - amqp
+    - kafka
+    - messaging
+  misc:
+    - braintree
+    - compression
+    - graphql
+    - mustache
+    - pdf
+    - qute
+    - stream
+    - tarfile
+    - infinispan
+  networking:
+    - ftp
+    - http
+    - mail
+    - netty
+    - servlet
+    - websocket-jsr356
+  platform:
+    - microprofile
+    - opentracing
+    - platform-http
+    - platform-http-engine
+    - reactive-streams
+  saas:
+    - box
+    - github
+    - google
+    - jira
+    - salesforce
+    - sap-netweaver
+    - servicenow
+    - slack
+  social:
+    - telegram
+    - twitter
+  validation:
+    - bean-validator
+    - validator

.github/workflows/ci-build.yaml

@@ -49,15 +49,15 @@ jobs:
       MAVEN_OPTS: -Xmx3000m
     steps:
       - name: Setup apache-snapshots profile
-        if: github.base_ref == 'camel-master'
+        if: github.ref == 'refs/heads/camel-master' || github.base_ref == 'camel-master'
         run: |
           echo '::set-env name=BRANCH_OPTIONS::-Papache-snapshots'
       - name: Setup oss-snapshots profile
-        if: github.base_ref == 'quarkus-master'
+        if: github.ref == 'refs/heads/quarkus-master' || github.base_ref == 'quarkus-master'
         run: |
-          echo '::set-env name=BRANCH_OPTIONS::-Poss-snapshots'
+          echo '::set-env name=BRANCH_OPTIONS::-Poss-snapshots -Dquarkus.version=999-SNAPSHOT'
       - name: Build Quarkus
-        if: github.base_ref == 'quarkus-master'
+        if: github.ref == 'refs/heads/quarkus-master' || github.base_ref == 'quarkus-master'
         run: |
           git clone --depth 1 --branch master https://github.com/quarkusio/quarkus.git \
             && cd quarkus \
@@ -90,13 +90,13 @@ jobs:
       MAVEN_OPTS: -Xmx3000m
     steps:
       - name: Setup apache-snapshots profile
-        if: github.base_ref == 'camel-master'
+        if: github.ref == 'refs/heads/camel-master' || github.base_ref == 'camel-master'
         run: |
           echo '::set-env name=BRANCH_OPTIONS::-Papache-snapshots'
       - name: Setup oss-snapshots profile
-        if: github.base_ref == 'quarkus-master'
+        if: github.ref == 'refs/heads/quarkus-master' || github.base_ref == 'quarkus-master'
         run: |
-          echo '::set-env name=BRANCH_OPTIONS::-Poss-snapshots'
+          echo '::set-env name=BRANCH_OPTIONS::-Poss-snapshots -Dquarkus.version=999-SNAPSHOT'
       - uses: actions/checkout@v2
       - name: Set up JDK ${{ matrix.java }}
         uses: joschi/setup-jdk@v1
@@ -120,127 +120,16 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        category: [Core, Foundation, Platform, Messaging, Dataformats, Networking, Database, Cloud, Misc, SAAS, Social, Validation, Memoryhog Dozer, Memoryhog Dataformat JSON, Memoryhog Infinispan, Memoryhog Olingo4, Memoryhog XML]
-        include:
-          - category: Cloud
-            test-modules: >
-              aws
-              aws2
-              azure
-              consul
-          - category: Core
-            test-modules: >
-              core
-              core-impl
-              core-main
-              core-main-caffeine-lrucache
-              core-main-collector
-              core-main-xml-io
-              core-main-xml-jaxb
-          - category: Dataformats
-            test-modules: >
-              base64
-              bindy
-              csv
-              dataformat
-              jaxb
-              xstream
-              soap
-          - category: Foundation
-            test-modules: >
-              bean
-              controlbus
-              exec
-              file
-              hystrix
-              jsonpath
-              quartz
-              scheduler
-              seda
-              send-dynamic-http
-              ref
-          - category: Memoryhog Dataformat JSON
-            test-modules: >
-              dataformats-json
-          - category: Memoryhog Dozer
-            test-modules: >
-              dozer
-          - category: Memoryhog Infinispan
-            test-modules: >
-              infinispan
-          - category: Memoryhog Olingo4
-            test-modules: >
-              olingo4
-          - category: Memoryhog XML
-            test-modules: >
-              xml
-              rest-binding-mode-xml
-          - category: Messaging
-            test-modules: >
-              activemq
-              amqp
-              kafka
-              messaging
-          - category: Misc
-            test-modules: >
-              braintree
-              compression
-              graphql
-              mustache
-              pdf
-              qute
-              stream
-              tarfile
-          - category: Networking
-            test-modules: >
-              ftp
-              http
-              mail
-              netty
-              servlet
-              websocket-jsr356
-          - category: Platform
-            test-modules: >
-              microprofile
-              opentracing
-              platform-http
-              platform-http-engine
-              reactive-streams
-          - category: SAAS
-            test-modules: >
-              box
-              github
-              google
-              jira
-              salesforce
-              sap-netweaver
-              servicenow
-              slack
-          - category: Database
-            test-modules: >
-              couchdb
-              influxdb
-              jdbc
-              kudu
-              mongodb
-              sql
-          - category: Social
-            test-modules: >
-              telegram
-              twitter
-          - category: Validation
-            test-modules: >
-              bean-validator
-              validator
+        category: [core, foundation, platform, messaging, database, dataformats, networking, cloud, misc, saas, social, validation, memoryhog-dozer, memoryhog-dataformat-json, memoryhog-infinispan, memoryhog-olingo4, memoryhog-xml]
     steps:
       - name: Setup apache-snapshots profile
-        if: github.base_ref == 'camel-master'
+        if: github.ref == 'refs/heads/camel-master' || github.base_ref == 'camel-master'
         run: |
           echo '::set-env name=BRANCH_OPTIONS::-Papache-snapshots'
       - name: Setup oss-snapshots profile
-        if: github.base_ref == 'quarkus-master'
+        if: github.ref == 'refs/heads/quarkus-master' || github.base_ref == 'quarkus-master'
         run: |
-          echo '::set-env name=BRANCH_OPTIONS::-Poss-snapshots'
+          echo '::set-env name=BRANCH_OPTIONS::-Poss-snapshots -Dquarkus.version=999-SNAPSHOT'
       - name: Checkout
         uses: actions/checkout@v2
       - name: Set up JDK 11
@@ -254,10 +143,28 @@ jobs:
           key: maven-${{ github.run_id }}-${{ github.run_number }}
           restore-keys: |
             maven-${{ github.run_id }}-${{ github.run_number }}
+      - name: Install yq
+        run: |
+          sudo wget -O /usr/local/bin/yq "https://github.com/mikefarah/yq/releases/download/3.2.1/yq_linux_amd64"
+          sudo chmod +x /usr/local/bin/yq
       - name: Integration Tests
         env:
           TEST_MODULES: ${{matrix.test-modules}}
         run: |
+          TEST_MODULES=""
+          for MODULE in $(cat .github/test-categories.yaml | yq r - categories.${{ matrix.category }}); do
+            if [ "${MODULE}" == "-" ]; then
+              continue
+            fi
+            TEST_MODULES="${TEST_MODULES} ${MODULE}"
+          done
+          TEST_MODULES=$(echo ${TEST_MODULES} | sed 's/^[ \t]*//;s/[ \t]*$//')
+
+          if [ -z "${TEST_MODULES}" ]; then
+            echo "No test modules were found for category ${{ matrix.category }}"
+            exit 1
+          fi
+
           for i in $TEST_MODULES
           do modules+=("integration-tests/$i"); done
           IFS=,

.github/workflows/sync-quarkus-master.yaml → .github/workflows/purge-stale-artifacts-cron.yaml

@@ -15,24 +15,23 @@
 # limitations under the License.
 #
 
-name: Sync Quarkus Master Branch
+name: Purge Stale Build Artifacts
 
 on:
   schedule:
-    # Run every day at 2AM
-    - cron:  '0 2 * * *'
+    # Run every 4 hours
+    - cron: '0 */4 * * *'
 
 jobs:
-  sync-quarkus-master-branch:
+  pruge-stale-build-artifacts:
     if: github.repository == 'apache/camel-quarkus'
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v1
-      - name: GitHub Pull Request Action
-        uses: repo-sync/pull-request@v2.0.1
-        with:
-          source_branch: master
-          destination_branch: quarkus-master
-          pr_title: Automatic sync branch master to quarkus-master
-          github_token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Purge Stale Artifacts
+        run: |
+          ARTIFACTS=$(curl -s -H "Accept: application/vnd.github.everest-preview+json" -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/artifacts)
+          for ID in $(echo ${ARTIFACTS} | jq '.artifacts[] | select (.expired == false and .name == "maven-repo") | .id'); do
+              echo "Deleting artifact ${ID} from ${GITHUB_REPOSITORY}"
+              curl -s -X DELETE -H "Accept: application/vnd.github.everest-preview+json" -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/artifacts/${ID}
+              sleep 5
+          done