diff --git a/ci/pipelines/deploy_meta.sh b/ci/pipelines/deploy_meta.sh index 3f590bf1452b..fa99770adb9b 100755 --- a/ci/pipelines/deploy_meta.sh +++ b/ci/pipelines/deploy_meta.sh @@ -18,6 +18,9 @@ GEODE_BRANCH=$(git rev-parse --abbrev-ref HEAD) SANITIZED_GEODE_BRANCH=$(echo ${GEODE_BRANCH} | tr "/" "-") TARGET=geode +GEODE_FORK=${1:-apache} + +echo "Deploying pipline for ${GEODE_FORK}/${GEODE_BRANCH}" set -x -fly -t ${TARGET} set-pipeline -p meta-${SANITIZED_GEODE_BRANCH} -c meta.yml --var geode-build-branch=${GEODE_BRANCH} +fly -t ${TARGET} set-pipeline -p meta-${SANITIZED_GEODE_BRANCH} -c meta.yml --var geode-build-branch=${GEODE_BRANCH} --var geode-fork=${GEODE_FORK} diff --git a/ci/pipelines/geode-build/base.yml b/ci/pipelines/geode-build/base.yml index ff28af2611e2..d6e3d3f6c05f 100644 --- a/ci/pipelines/geode-build/base.yml +++ b/ci/pipelines/geode-build/base.yml @@ -38,7 +38,7 @@ resources: - name: geode type: git source: - uri: https://github.com/apache/geode.git + uri: (( concat "https://github.com/" metadata.geode-fork "/geode.git" )) branch: (( grab metadata.geode-build-branch )) ignore_paths: - ci/* @@ -46,7 +46,7 @@ resources: type: git source: depth: 1 - uri: https://github.com/apache/geode.git + uri: (( concat "https://github.com/" metadata.geode-fork "/geode.git" )) branch: (( grab metadata.geode-build-branch )) paths: - ci/* @@ -194,3 +194,124 @@ jobs: PUBLIC_BUCKET: ((!public-bucket)) run: path: geode-ci/ci/scripts/update-passing-ref.sh + +- name: DistributedTest + serial: true + public: true + plan: + - aggregate: + - get: geode + passed: [Build] + trigger: true + - get: geode-ci + - get: geode-build-version + passed: [Build] + - get: docker-geode-build-image + params: + rootfs: true + - aggregate: + - task: run-distributed-core + image: docker-geode-build-image + tags: [large] + privileged: true + timeout: 8h + config: + inputs: + - name: geode + - name: geode-ci + - name: docker-geode-build-image + - name: geode-build-version + platform: linux + outputs: + - name: built-geode + params: + MAINTENANCE_VERSION: (( grab metadata.geode-build-branch )) + SERVICE_ACCOUNT: ((!concourse-gcp-account)) + PUBLIC_BUCKET: ((!public-bucket)) + PARALLEL_DUNIT: true + DUNIT_PARALLEL_FORKS: 7 + CALL_STACK_TIMEOUT: 25200 + run: + args: + - geode-core:distributedTest + - distributedtestcore + path: geode-ci/ci/scripts/test-run.sh + on_failure: + aggregate: + - put: send-notification-email + params: + subject: built-geode/subject + body: built-geode/body + ensure: + aggregate: + - task: archive-results-core + image: docker-geode-build-image + config: + inputs: + - name: geode + - name: geode-ci + - name: geode-build-version + - name: built-geode + platform: linux + params: + MAINTENANCE_VERSION: (( grab metadata.geode-build-branch )) + SERVICE_ACCOUNT: ((!concourse-gcp-account)) + PUBLIC_BUCKET: ((!public-bucket)) + run: + args: + - geode-core:distributedTest + - distributedtestcore + path: geode-ci/ci/scripts/test-archive.sh + - task: run-distributed-everything-else + image: docker-geode-build-image + tags: [large] + privileged: true + timeout: 8h + config: + inputs: + - name: geode + - name: geode-ci + - name: docker-geode-build-image + - name: geode-build-version + platform: linux + outputs: + - name: built-geode + params: + MAINTENANCE_VERSION: (( grab metadata.geode-build-branch )) + SERVICE_ACCOUNT: ((!concourse-gcp-account)) + PUBLIC_BUCKET: ((!public-bucket)) + PARALLEL_DUNIT: true + DUNIT_PARALLEL_FORKS: 7 + CALL_STACK_TIMEOUT: 25200 + GRADLE_TASK_OPTIONS: -x geode-core:distributedTest + run: + args: + - distributedTest + - distributedtesteverythingelse + path: geode-ci/ci/scripts/test-run.sh + on_failure: + aggregate: + - put: send-notification-email + params: + subject: built-geode/subject + body: built-geode/body + ensure: + aggregate: + - task: archive-results-everythingelse + image: docker-geode-build-image + config: + inputs: + - name: geode + - name: geode-ci + - name: geode-build-version + - name: built-geode + platform: linux + params: + MAINTENANCE_VERSION: (( grab metadata.geode-build-branch )) + SERVICE_ACCOUNT: ((!concourse-gcp-account)) + PUBLIC_BUCKET: ((!public-bucket)) + run: + args: + - disitributedTest + - distributedtesteverythingelse + path: geode-ci/ci/scripts/test-archive.sh diff --git a/ci/pipelines/geode-build/deploy_pipeline.sh b/ci/pipelines/geode-build/deploy_pipeline.sh index 4b4effad8cd3..0e596d236b3f 100755 --- a/ci/pipelines/geode-build/deploy_pipeline.sh +++ b/ci/pipelines/geode-build/deploy_pipeline.sh @@ -60,7 +60,8 @@ for i in ${SCRIPTDIR}/test-stubs/*.yml; do echo "Merging ${i} into ${TMP_DIR}/${X}" ${SPRUCE} merge --prune metadata \ <(echo "metadata:"; \ - echo " geode-build-branch: ${GEODE_BRANCH}") \ + echo " geode-build-branch: ${GEODE_BRANCH}"; \ + echo " geode-fork: ${GEODE_FORK}") \ ${SCRIPTDIR}/test-template.yml \ ${i} > ${TMP_DIR}/${X} done @@ -70,6 +71,7 @@ ${SPRUCE} merge --prune metadata \ ${SCRIPTDIR}/base.yml \ <(echo "metadata:"; \ echo " geode-build-branch: ${GEODE_BRANCH}"; \ + echo " geode-fork: ${GEODE_FORK}"; \ echo " ") \ ${TMP_DIR}/*.yml > ${TMP_DIR}/final.yml diff --git a/ci/pipelines/geode-build/test-stubs/distributed.yml b/ci/pipelines/geode-build/test-stubs/distributed.yml deleted file mode 100644 index 93a8c6a1d036..000000000000 --- a/ci/pipelines/geode-build/test-stubs/distributed.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -metadata: - job: - name: DistributedTest - gradle_task: distributedTest - artifact_slug: distributedtestfiles - dunit: - parallel: true - forks: 7 - call_stack_timeout: 25200 - timeout: 8h - size: [large] diff --git a/ci/pipelines/meta.yml b/ci/pipelines/meta.yml index 1eb0f8d4db58..a7ee01774661 100644 --- a/ci/pipelines/meta.yml +++ b/ci/pipelines/meta.yml @@ -32,24 +32,24 @@ resources: - name: main username: ((!concourse-username)) password: ((!concourse-password)) -- name: geode-develop-pipeline +- name: geode-pipeline type: git source: - uri: https://github.com/apache/geode.git + uri: https://github.com/((!geode-fork))/geode.git branch: ((!geode-build-branch)) paths: - ci/* - name: geode-docker-images-pipeline type: git source: - uri: https://github.com/apache/geode.git + uri: https://github.com/((!geode-fork))/geode.git branch: ((!geode-build-branch)) paths: - ci/pipelines/docker-images.yml - name: geode-metrics-pipeline type: git source: - uri: https://github.com/apache/geode.git + uri: https://github.com/((!geode-fork))/geode.git branch: ((!geode-build-branch)) paths: - ci/pipelines/metrics.yml @@ -79,6 +79,7 @@ jobs: params: OUTPUT_DIRECTORY: results GEODE_BRANCH: ((!geode-build-branch)) + GEODE_FORK: ((!geode-fork)) CONCOURSE_USERNAME: ((!concourse-username)) CONCOURSE_PASSWORD: ((!concourse-password)) run: diff --git a/ci/scripts/test-run.sh b/ci/scripts/test-run.sh index dfde39750e67..ca4e82090e82 100755 --- a/ci/scripts/test-run.sh +++ b/ci/scripts/test-run.sh @@ -130,7 +130,7 @@ pushd ${GEODE_BUILD} set -x # ./gradlew --no-daemon -x javadoc -x spotlessCheck :geode-assembly:acceptanceTest --tests org.apache.geode.management.internal.cli.commands.PutCommandWithJsonTest ./gradlew ${PARALLEL_DUNIT} ${DUNIT_PARALLEL_FORKS} ${DUNIT_DOCKER_IMAGE} \ - --system-prop "java.io.tmpdir=${TMPDIR}" ${DEFAULT_GRADLE_TASK_OPTIONS} ${GRADLE_TASK} + --system-prop "java.io.tmpdir=${TMPDIR}" ${DEFAULT_GRADLE_TASK_OPTIONS} ${GRADLE_TASK_OPTIONS} ${GRADLE_TASK} export GRADLE_EXIT_STATUS=$? set +x popd