From e9a61bb8b06a73bb03a2d34b7c094b8f356ebf60 Mon Sep 17 00:00:00 2001 From: Sean Goller Date: Mon, 4 Jun 2018 09:30:54 -0700 Subject: [PATCH] Split monolithic DistributedTest job into two parallel tasks Tasks execute within one box still for good optics, but execute in parallel for runtime improvement. Opens door for templating and making many tasks. [GEODE-5282] Signed-off-by: Robert Houghton --- ci/pipelines/deploy_meta.sh | 5 +- ci/pipelines/geode-build/base.yml | 125 +++++++++++++++++- ci/pipelines/geode-build/deploy_pipeline.sh | 4 +- .../geode-build/test-stubs/distributed.yml | 27 ---- ci/pipelines/meta.yml | 9 +- ci/scripts/test-run.sh | 2 +- 6 files changed, 136 insertions(+), 36 deletions(-) delete mode 100644 ci/pipelines/geode-build/test-stubs/distributed.yml diff --git a/ci/pipelines/deploy_meta.sh b/ci/pipelines/deploy_meta.sh index 3f590bf1452b..fa99770adb9b 100755 --- a/ci/pipelines/deploy_meta.sh +++ b/ci/pipelines/deploy_meta.sh @@ -18,6 +18,9 @@ GEODE_BRANCH=$(git rev-parse --abbrev-ref HEAD) SANITIZED_GEODE_BRANCH=$(echo ${GEODE_BRANCH} | tr "/" "-") TARGET=geode +GEODE_FORK=${1:-apache} + +echo "Deploying pipline for ${GEODE_FORK}/${GEODE_BRANCH}" set -x -fly -t ${TARGET} set-pipeline -p meta-${SANITIZED_GEODE_BRANCH} -c meta.yml --var geode-build-branch=${GEODE_BRANCH} +fly -t ${TARGET} set-pipeline -p meta-${SANITIZED_GEODE_BRANCH} -c meta.yml --var geode-build-branch=${GEODE_BRANCH} --var geode-fork=${GEODE_FORK} diff --git a/ci/pipelines/geode-build/base.yml b/ci/pipelines/geode-build/base.yml index ff28af2611e2..d6e3d3f6c05f 100644 --- a/ci/pipelines/geode-build/base.yml +++ b/ci/pipelines/geode-build/base.yml @@ -38,7 +38,7 @@ resources: - name: geode type: git source: - uri: https://github.com/apache/geode.git + uri: (( concat "https://github.com/" metadata.geode-fork "/geode.git" )) branch: (( grab metadata.geode-build-branch )) ignore_paths: - ci/* @@ -46,7 +46,7 @@ resources: type: git source: depth: 1 - uri: https://github.com/apache/geode.git + uri: (( concat "https://github.com/" metadata.geode-fork "/geode.git" )) branch: (( grab metadata.geode-build-branch )) paths: - ci/* @@ -194,3 +194,124 @@ jobs: PUBLIC_BUCKET: ((!public-bucket)) run: path: geode-ci/ci/scripts/update-passing-ref.sh + +- name: DistributedTest + serial: true + public: true + plan: + - aggregate: + - get: geode + passed: [Build] + trigger: true + - get: geode-ci + - get: geode-build-version + passed: [Build] + - get: docker-geode-build-image + params: + rootfs: true + - aggregate: + - task: run-distributed-core + image: docker-geode-build-image + tags: [large] + privileged: true + timeout: 8h + config: + inputs: + - name: geode + - name: geode-ci + - name: docker-geode-build-image + - name: geode-build-version + platform: linux + outputs: + - name: built-geode + params: + MAINTENANCE_VERSION: (( grab metadata.geode-build-branch )) + SERVICE_ACCOUNT: ((!concourse-gcp-account)) + PUBLIC_BUCKET: ((!public-bucket)) + PARALLEL_DUNIT: true + DUNIT_PARALLEL_FORKS: 7 + CALL_STACK_TIMEOUT: 25200 + run: + args: + - geode-core:distributedTest + - distributedtestcore + path: geode-ci/ci/scripts/test-run.sh + on_failure: + aggregate: + - put: send-notification-email + params: + subject: built-geode/subject + body: built-geode/body + ensure: + aggregate: + - task: archive-results-core + image: docker-geode-build-image + config: + inputs: + - name: geode + - name: geode-ci + - name: geode-build-version + - name: built-geode + platform: linux + params: + MAINTENANCE_VERSION: (( grab metadata.geode-build-branch )) + SERVICE_ACCOUNT: ((!concourse-gcp-account)) + PUBLIC_BUCKET: ((!public-bucket)) + run: + args: + - geode-core:distributedTest + - distributedtestcore + path: geode-ci/ci/scripts/test-archive.sh + - task: run-distributed-everything-else + image: docker-geode-build-image + tags: [large] + privileged: true + timeout: 8h + config: + inputs: + - name: geode + - name: geode-ci + - name: docker-geode-build-image + - name: geode-build-version + platform: linux + outputs: + - name: built-geode + params: + MAINTENANCE_VERSION: (( grab metadata.geode-build-branch )) + SERVICE_ACCOUNT: ((!concourse-gcp-account)) + PUBLIC_BUCKET: ((!public-bucket)) + PARALLEL_DUNIT: true + DUNIT_PARALLEL_FORKS: 7 + CALL_STACK_TIMEOUT: 25200 + GRADLE_TASK_OPTIONS: -x geode-core:distributedTest + run: + args: + - distributedTest + - distributedtesteverythingelse + path: geode-ci/ci/scripts/test-run.sh + on_failure: + aggregate: + - put: send-notification-email + params: + subject: built-geode/subject + body: built-geode/body + ensure: + aggregate: + - task: archive-results-everythingelse + image: docker-geode-build-image + config: + inputs: + - name: geode + - name: geode-ci + - name: geode-build-version + - name: built-geode + platform: linux + params: + MAINTENANCE_VERSION: (( grab metadata.geode-build-branch )) + SERVICE_ACCOUNT: ((!concourse-gcp-account)) + PUBLIC_BUCKET: ((!public-bucket)) + run: + args: + - disitributedTest + - distributedtesteverythingelse + path: geode-ci/ci/scripts/test-archive.sh diff --git a/ci/pipelines/geode-build/deploy_pipeline.sh b/ci/pipelines/geode-build/deploy_pipeline.sh index 4b4effad8cd3..0e596d236b3f 100755 --- a/ci/pipelines/geode-build/deploy_pipeline.sh +++ b/ci/pipelines/geode-build/deploy_pipeline.sh @@ -60,7 +60,8 @@ for i in ${SCRIPTDIR}/test-stubs/*.yml; do echo "Merging ${i} into ${TMP_DIR}/${X}" ${SPRUCE} merge --prune metadata \ <(echo "metadata:"; \ - echo " geode-build-branch: ${GEODE_BRANCH}") \ + echo " geode-build-branch: ${GEODE_BRANCH}"; \ + echo " geode-fork: ${GEODE_FORK}") \ ${SCRIPTDIR}/test-template.yml \ ${i} > ${TMP_DIR}/${X} done @@ -70,6 +71,7 @@ ${SPRUCE} merge --prune metadata \ ${SCRIPTDIR}/base.yml \ <(echo "metadata:"; \ echo " geode-build-branch: ${GEODE_BRANCH}"; \ + echo " geode-fork: ${GEODE_FORK}"; \ echo " ") \ ${TMP_DIR}/*.yml > ${TMP_DIR}/final.yml diff --git a/ci/pipelines/geode-build/test-stubs/distributed.yml b/ci/pipelines/geode-build/test-stubs/distributed.yml deleted file mode 100644 index 93a8c6a1d036..000000000000 --- a/ci/pipelines/geode-build/test-stubs/distributed.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -metadata: - job: - name: DistributedTest - gradle_task: distributedTest - artifact_slug: distributedtestfiles - dunit: - parallel: true - forks: 7 - call_stack_timeout: 25200 - timeout: 8h - size: [large] diff --git a/ci/pipelines/meta.yml b/ci/pipelines/meta.yml index 1eb0f8d4db58..a7ee01774661 100644 --- a/ci/pipelines/meta.yml +++ b/ci/pipelines/meta.yml @@ -32,24 +32,24 @@ resources: - name: main username: ((!concourse-username)) password: ((!concourse-password)) -- name: geode-develop-pipeline +- name: geode-pipeline type: git source: - uri: https://github.com/apache/geode.git + uri: https://github.com/((!geode-fork))/geode.git branch: ((!geode-build-branch)) paths: - ci/* - name: geode-docker-images-pipeline type: git source: - uri: https://github.com/apache/geode.git + uri: https://github.com/((!geode-fork))/geode.git branch: ((!geode-build-branch)) paths: - ci/pipelines/docker-images.yml - name: geode-metrics-pipeline type: git source: - uri: https://github.com/apache/geode.git + uri: https://github.com/((!geode-fork))/geode.git branch: ((!geode-build-branch)) paths: - ci/pipelines/metrics.yml @@ -79,6 +79,7 @@ jobs: params: OUTPUT_DIRECTORY: results GEODE_BRANCH: ((!geode-build-branch)) + GEODE_FORK: ((!geode-fork)) CONCOURSE_USERNAME: ((!concourse-username)) CONCOURSE_PASSWORD: ((!concourse-password)) run: diff --git a/ci/scripts/test-run.sh b/ci/scripts/test-run.sh index dfde39750e67..ca4e82090e82 100755 --- a/ci/scripts/test-run.sh +++ b/ci/scripts/test-run.sh @@ -130,7 +130,7 @@ pushd ${GEODE_BUILD} set -x # ./gradlew --no-daemon -x javadoc -x spotlessCheck :geode-assembly:acceptanceTest --tests org.apache.geode.management.internal.cli.commands.PutCommandWithJsonTest ./gradlew ${PARALLEL_DUNIT} ${DUNIT_PARALLEL_FORKS} ${DUNIT_DOCKER_IMAGE} \ - --system-prop "java.io.tmpdir=${TMPDIR}" ${DEFAULT_GRADLE_TASK_OPTIONS} ${GRADLE_TASK} + --system-prop "java.io.tmpdir=${TMPDIR}" ${DEFAULT_GRADLE_TASK_OPTIONS} ${GRADLE_TASK_OPTIONS} ${GRADLE_TASK} export GRADLE_EXIT_STATUS=$? set +x popd