[GEODE-5282] Split monolithic DistributedTest job into two parallel tasks

ci/pipelines/deploy_meta.sh

@@ -18,6 +18,9 @@
 GEODE_BRANCH=$(git rev-parse --abbrev-ref HEAD)
 SANITIZED_GEODE_BRANCH=$(echo ${GEODE_BRANCH} | tr "/" "-")
 TARGET=geode
+GEODE_FORK=${1:-apache}
+
+echo "Deploying pipline for ${GEODE_FORK}/${GEODE_BRANCH}"
 
 set -x
-fly -t ${TARGET} set-pipeline -p meta-${SANITIZED_GEODE_BRANCH} -c meta.yml --var geode-build-branch=${GEODE_BRANCH}
+fly -t ${TARGET} set-pipeline -p meta-${SANITIZED_GEODE_BRANCH} -c meta.yml --var geode-build-branch=${GEODE_BRANCH} --var geode-fork=${GEODE_FORK}

ci/pipelines/geode-build/base.yml

@@ -38,15 +38,15 @@ resources:
 - name: geode
   type: git
   source:
-    uri: https://github.com/apache/geode.git
+    uri: (( concat "https://github.com/" metadata.geode-fork "/geode.git" ))
     branch: (( grab metadata.geode-build-branch ))
     ignore_paths:
     - ci/*
 - name: geode-ci
   type: git
   source:
     depth: 1
-    uri: https://github.com/apache/geode.git
+    uri: (( concat "https://github.com/" metadata.geode-fork "/geode.git" ))
     branch: (( grab metadata.geode-build-branch ))
     paths:
     - ci/*
@@ -194,3 +194,124 @@ jobs:
           PUBLIC_BUCKET: ((!public-bucket))
         run:
           path: geode-ci/ci/scripts/update-passing-ref.sh
+
+- name: DistributedTest
+  serial: true
+  public: true
+  plan:
+    - aggregate:
+      - get: geode
+        passed: [Build]
+        trigger: true
+      - get: geode-ci
+      - get: geode-build-version
+        passed: [Build]
+      - get: docker-geode-build-image
+        params:
+          rootfs: true
+    - aggregate:
+      - task: run-distributed-core
+        image: docker-geode-build-image
+        tags: [large]
+        privileged: true
+        timeout: 8h
+        config:
+          inputs:
+            - name: geode
+            - name: geode-ci
+            - name: docker-geode-build-image
+            - name: geode-build-version
+          platform: linux
+          outputs:
+            - name: built-geode
+          params:
+            MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
+            SERVICE_ACCOUNT: ((!concourse-gcp-account))
+            PUBLIC_BUCKET: ((!public-bucket))
+            PARALLEL_DUNIT: true
+            DUNIT_PARALLEL_FORKS: 7
+            CALL_STACK_TIMEOUT: 25200
+          run:
+            args:
+            - geode-core:distributedTest
+            - distributedtestcore
+            path: geode-ci/ci/scripts/test-run.sh
+        on_failure:
+          aggregate:
+            - put: send-notification-email
+              params:
+                subject: built-geode/subject
+                body: built-geode/body
+        ensure:
+          aggregate:
+          - task: archive-results-core
+            image: docker-geode-build-image
+            config:
+              inputs:
+                - name: geode
+                - name: geode-ci
+                - name: geode-build-version
+                - name: built-geode
+              platform: linux
+              params:
+                MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
+                SERVICE_ACCOUNT: ((!concourse-gcp-account))
+                PUBLIC_BUCKET: ((!public-bucket))
+              run:
+                args:
+                - geode-core:distributedTest
+                - distributedtestcore
+                path: geode-ci/ci/scripts/test-archive.sh
+      - task: run-distributed-everything-else
+        image: docker-geode-build-image
+        tags: [large]
+        privileged: true
+        timeout: 8h
+        config:
+          inputs:
+            - name: geode
+            - name: geode-ci
+            - name: docker-geode-build-image
+            - name: geode-build-version
+          platform: linux
+          outputs:
+            - name: built-geode
+          params:
+            MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
+            SERVICE_ACCOUNT: ((!concourse-gcp-account))
+            PUBLIC_BUCKET: ((!public-bucket))
+            PARALLEL_DUNIT: true
+            DUNIT_PARALLEL_FORKS: 7
+            CALL_STACK_TIMEOUT: 25200
+            GRADLE_TASK_OPTIONS: -x geode-core:distributedTest
+          run:
+            args:
+            - distributedTest
+            - distributedtesteverythingelse
+            path: geode-ci/ci/scripts/test-run.sh
+        on_failure:
+          aggregate:
+            - put: send-notification-email
+              params:
+                subject: built-geode/subject
+                body: built-geode/body
+        ensure:
+          aggregate:
+          - task: archive-results-everythingelse
+            image: docker-geode-build-image
+            config:
+              inputs:
+                - name: geode
+                - name: geode-ci
+                - name: geode-build-version
+                - name: built-geode
+              platform: linux
+              params:
+                MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
+                SERVICE_ACCOUNT: ((!concourse-gcp-account))
+                PUBLIC_BUCKET: ((!public-bucket))
+              run:
+                args:
+                - disitributedTest
+                - distributedtesteverythingelse
+                path: geode-ci/ci/scripts/test-archive.sh

ci/pipelines/geode-build/deploy_pipeline.sh

@@ -60,7 +60,8 @@ for i in ${SCRIPTDIR}/test-stubs/*.yml; do
   echo "Merging ${i} into ${TMP_DIR}/${X}"
   ${SPRUCE} merge --prune metadata \
     <(echo "metadata:"; \
-      echo "  geode-build-branch: ${GEODE_BRANCH}") \
+      echo "  geode-build-branch: ${GEODE_BRANCH}"; \
+      echo "  geode-fork: ${GEODE_FORK}") \
     ${SCRIPTDIR}/test-template.yml \
     ${i} > ${TMP_DIR}/${X}
 done
@@ -70,6 +71,7 @@ ${SPRUCE} merge --prune metadata \
   ${SCRIPTDIR}/base.yml \
   <(echo "metadata:"; \
     echo "  geode-build-branch: ${GEODE_BRANCH}"; \
+    echo "  geode-fork: ${GEODE_FORK}"; \
     echo "  ") \
   ${TMP_DIR}/*.yml > ${TMP_DIR}/final.yml
 

ci/pipelines/meta.yml

@@ -32,24 +32,24 @@ resources:
     - name: main
       username: ((!concourse-username))
       password: ((!concourse-password))
-- name: geode-develop-pipeline
+- name: geode-pipeline
   type: git
   source:
-    uri: https://github.com/apache/geode.git
+    uri: https://github.com/((!geode-fork))/geode.git
     branch: ((!geode-build-branch))
     paths:
     - ci/*
 - name: geode-docker-images-pipeline
   type: git
   source:
-    uri: https://github.com/apache/geode.git
+    uri: https://github.com/((!geode-fork))/geode.git
     branch: ((!geode-build-branch))
     paths:
     - ci/pipelines/docker-images.yml
 - name: geode-metrics-pipeline
   type: git
   source:
-    uri: https://github.com/apache/geode.git
+    uri: https://github.com/((!geode-fork))/geode.git
     branch: ((!geode-build-branch))
     paths:
     - ci/pipelines/metrics.yml
@@ -79,6 +79,7 @@ jobs:
       params:
         OUTPUT_DIRECTORY: results
         GEODE_BRANCH: ((!geode-build-branch))
+        GEODE_FORK: ((!geode-fork))
         CONCOURSE_USERNAME: ((!concourse-username))
         CONCOURSE_PASSWORD: ((!concourse-password))
       run:

ci/scripts/test-run.sh

@@ -130,7 +130,7 @@ pushd ${GEODE_BUILD}
   set -x
 #    ./gradlew --no-daemon -x javadoc -x spotlessCheck :geode-assembly:acceptanceTest --tests org.apache.geode.management.internal.cli.commands.PutCommandWithJsonTest
   ./gradlew ${PARALLEL_DUNIT} ${DUNIT_PARALLEL_FORKS} ${DUNIT_DOCKER_IMAGE} \
-      --system-prop "java.io.tmpdir=${TMPDIR}" ${DEFAULT_GRADLE_TASK_OPTIONS} ${GRADLE_TASK}
+      --system-prop "java.io.tmpdir=${TMPDIR}" ${DEFAULT_GRADLE_TASK_OPTIONS} ${GRADLE_TASK_OPTIONS} ${GRADLE_TASK}
   export GRADLE_EXIT_STATUS=$?
   set +x
 popd