Skip to content
Permalink
Browse files
GEODE-8949: Versioning improvements (#754)
* Download latest Geode.
* Don't bake Geode into images.
* Trigger packer images.
* Configurable version for semver and geode resource.
* Adds version bump.
* Uses geode latest in source checks.
  • Loading branch information
pivotal-jbarrett committed Mar 2, 2021
1 parent 6c1cf0c commit 38fa9eef45da2a0743410d24c5852e8dd3a69494
Show file tree
Hide file tree
Showing 18 changed files with 126 additions and 96 deletions.
@@ -98,6 +98,7 @@ This Concourse pipeline YAML is rendered using `ytt`. Depends on output from `gi
* [Concourse](https://concourse-ci.org) v6.5.0+
* [`ytt`](https://get-ytt.io) v0.28.0+
* [`git`](https://git-scm.com) v2.25.2+
* [`yq`](https://github.com/mikefarah/yq) v4.6.0+ (optional for set-pipeline script)
* [`gcloud`](https://cloud.google.com/sdk/docs/install) SDK

## Layout
@@ -36,6 +36,8 @@ packages:
---
pipeline:
name: develop
version: 1.15.0
pre: build

github:
owner: apache
@@ -21,13 +21,19 @@
#@ "registry_image_resource",
#@ "semver_resource",
#@ "apache_directory_index_resource",
#@ "maven_resource",
#@ "build_resources",
#@ "build_jobs",
#@ "build_job_name",
#@ "check_source_job",
#@ "check_source_job_name",
#@ "update_pipeline_job",
#@ "update_pipeline_job_name")
#@ "update_pipeline_job_name",
#@ )
#@ load("functions.lib.yml",
#@ "initial_version",
#@ "version_pattern",
#@ )
#@ load("@ytt:data", "data")
#@ load("@ytt:template", "template")

@@ -36,17 +42,19 @@ resource_types:
- #@ resource_type("gci-resource", "smgoller/gci-resource")
- #@ resource_type("gcs-resource", "frodenas/gcs-resource")
- #@ resource_type("apache-directory-index-resource", "jakepivotal/apache-directory-index-resource")
- #@ resource_type("maven-resource", "jakepivotal/maven-resource")

resources:
- #@ semver_resource("version", "1.14.0-build.0")
- #@ semver_resource("version", initial_version())
- #@ gcr_image_resource("gcloud-image", "google.com/cloudsdktool/cloud-sdk", "alpine")
- #@ project_gcr_image_resource("task-image", "geode-native-task")
- #@ project_gcr_image_resource("clang-tools-image", "geode-native-clang-tools")
- #@ github_resource("source", data.values.github.owner, data.values.github.repository, data.values.repository.branch, ignore_paths=["ci/*", "packer/*"])
- #@ template.replace(build_resources(data.values.builds, data.values.configs))
- #@ registry_image_resource("ytt-image", "gerritk/ytt")
- #@ github_resource("ci-source", data.values.github.owner, data.values.github.repository, data.values.repository.branch, ["ci/*"])
- #@ apache_directory_index_resource("geode", "https://archive.apache.org/dist/geode/", "$VERSION", "apache-geode-$VERSION.tgz")
- #@ maven_resource("geode-latest", "https://maven.apachegeode-ci.info/snapshots", "org.apache.geode:apache-geode:tgz", version_pattern())
#! - #@ apache_directory_index_resource("geode", "https://archive.apache.org/dist/geode/", "$VERSION", "apache-geode-$VERSION.tgz")

jobs:
- #@ template.replace(build_jobs(data.values.builds, data.values.configs))
@@ -33,3 +33,13 @@
#@ def image_family_name(family):
#@ return (data.values.pipeline.name + "-" + family)[0:62]
#@ end

---
#@ def initial_version():
#@ return data.values.pipeline.version + ("-" + data.values.pipeline.pre + ".0" if data.values.pipeline.pre else "")
#@ end

---
#@ def version_pattern():
#@ return (data.values.pipeline.version + ("-" + data.values.pipeline.pre + "." if data.values.pipeline.pre else "")).replace(".", "\.") + ".*"
#@ end
@@ -80,7 +80,7 @@ remote_shell cmake -E chdir (@= path @) ctest -C ${CMAKE_CONFIG} -j(@= str(paral

(@ def clang_tidy_bash_task(): -@)
set -ueo pipefail
export GEODE_HOME=$(pwd)/geode/expanded/apache-geode-$(cat geode/version)
export GEODE_HOME=$(pwd)/geode
log=$(mktemp)
cd build
cmake ../source -DCMAKE_EXPORT_COMPILE_COMMANDS=ON >>${log} 2>&1 || (cat ${log}; exit 1)
@@ -90,7 +90,7 @@ run-clang-tidy -j $(nproc) -quiet

(@ def clang_format_bash_task(): -@)
set -ueo pipefail
export GEODE_HOME=$(pwd)/geode/expanded/apache-geode-$(cat geode/version)
export GEODE_HOME=$(pwd)/geode
log=$(mktemp)
cd build
cmake ../source -DCMAKE_EXPORT_COMPILE_COMMANDS=ON >>${log} 2>&1 || (cat ${log}; exit 1)
@@ -99,7 +99,7 @@ jq -r '.[].file' compile_commands.json | sort | uniq | grep -v $(pwd) | xargs cl

(@ def rat_check_bash_task(): -@)
set -ueo pipefail
export GEODE_HOME=$(pwd)/geode/expanded/apache-geode-$(cat geode/version)
export GEODE_HOME=$(pwd)/geode
log=$(mktemp)
cd build
cmake ../source -DUSE_RAT=ON >>${log} 2>&1 || (cat ${log}; exit 1)
@@ -112,6 +112,8 @@ cd source/packer
packer build -only=googlecompute \
-var-file=default.json \
-var pipeline=(@= gci_label_value(data.values.pipeline.name) @) \
-var version=(@= data.values.pipeline.version @) \
-var pre=(@= data.values.pipeline.pre @) \
-var owner=(@= gci_label_value(data.values.github.owner) @) \
-var repository=(@= gci_label_value(data.values.github.repository) @) \
-var branch=(@= gci_label_value(data.values.repository.branch) @) \
@@ -174,8 +176,9 @@ done
(@ def instance_delete_bash_task(): -@)
set -ueo pipefail
(@= google_variables() @)
(@= remote_functions() @)
(@= instance_variables() @)

instance_file=${INSTANCE_DIR}/instance.sh
instance_name=$(source ${instance_file} && echo -n ${name})

gcloud compute instances delete ${instance_name} \
@@ -200,10 +203,10 @@ git_rev=$(git rev-parse HEAD)
popd
version=$(cat version/number)
builddate=$(date "+%Y-%m-%d")
remote_upload source .
remote_shell cmake -E make_directory build
remote_shell cmake -E time cmake -E chdir build cmake ../source ${CMAKE_CONFIGURE_FLAGS} \
remote_shell cmake -E time cmake -E chdir build cmake ../source ${CMAKE_CONFIGURE_FLAGS} \
-DCMAKE_BUILD_TYPE=${CMAKE_CONFIG} \
-DGEODE_ROOT=../geode \
-DPRODUCT_VERSION=${version} \
-DPRODUCT_BUILDDATE=${builddate} \
-DPRODUCT_SOURCE_REVISION=${git_rev} \
@@ -220,3 +223,33 @@ for checksum in ${checksums}; do
remote_download ${checksum} package/
done
(@- end @)

(@ def upload_source_bash_task(): -@)
set -ueo pipefail
(@= remote_functions() @)
remote_upload source .
(@- end @)


(@ def upload_geode_bash_task(): -@)
set -ueo pipefail
(@= remote_functions() @)
geode_version=$(cat geode-latest/version)
geode_name="apache-geode-${geode_version}"
geode_artifact="${geode_name}.tgz"
remote_upload geode-latest/${geode_artifact} .
remote_shell cmake -E tar xvf ${geode_artifact}
remote_shell cmake -E rename ${geode_name} geode
(@- end @)

(@ def extract_geode_bash_task(): -@)
set -ueo pipefail
if (cmp -s geode-latest/version geode/version); then
exit 0
fi
geode_version=$(cat geode-latest/version)
geode_name="apache-geode-${geode_version}"
geode_artifact="${geode_name}.tgz"
tar xvf geode-latest/${geode_artifact} --directory=geode --strip-components=1
cp geode-latest/version geode/version
(@- end @)
@@ -33,6 +33,9 @@
#@ "instance_delete_bash_task",
#@ "download_build_bash_task",
#@ "build_bash_task",
#@ "upload_source_bash_task",
#@ "upload_geode_bash_task",
#@ "extract_geode_bash_task",
#@ )

#@ load("@ytt:overlay", "overlay")
@@ -116,6 +119,7 @@ source:
#@ }, "google-cloud")
#@ end

---
#@ def apache_directory_index_resource(name, directory, folder_pattern, file_pattern, expand=True):
#@ return resource(name, "apache-directory-index-resource", {
#@ "directory": directory,
@@ -125,6 +129,15 @@ source:
#@ }, "file-multiple")
#@ end

---
#@ def maven_resource(name, url, artifact, version=None):
#@ return resource(name, "maven-resource", {
#@ "url": url,
#@ "artifact": artifact,
#@ "version": version
#@ }, "archive")
#@ end

---
#@ def create_instance_task(build, config):
#@ return bash_task(
@@ -187,7 +200,16 @@ plan:
steps:
- get: gcloud-image
- get: #@ gci_resource_name(image_family_name(build.image_family))
- #@ create_instance_task(build, config)
- in_parallel:
fail_fast: true
steps:
- #@ create_instance_task(build, config)
- get: geode-latest
- in_parallel:
fail_fast: true
steps:
- #@ bash_task("upload-source", [{"name":"instance"},{"name":"source"}], [], upload_source_bash_task())
- #@ bash_task("upload-geode", [{"name":"instance"},{"name":"geode-latest"}], [], upload_geode_bash_task())
- #@ build_task(config.config, build.params)
- #@ remote_task("cpp-unit-tests", config.config, ctest_bash_task("build/cppcache/test", parallel=8), "30m", build.params)
- #@ remote_task("cpp-integration-tests", config.config, ctest_bash_task("build/cppcache/integration/test"), "1h", build.params)
@@ -224,7 +246,7 @@ plan:
- get: #@ gci_resource_name(build.source_image_family)
trigger: true
- get: packer-source
trigger: false
trigger: true
- #@ bash_task("packer-build", [{"name":"packer-source", "path":"source"}], [], packer_build_bash_task(build), "packer-image", attempts=5)
#@ end

@@ -405,7 +427,10 @@ plan:
trigger: true
- get: source
trigger: true
- get: geode
- do:
- get: task-image
- get: geode-latest
- #@ bash_task("extract-geode", [{"name":"geode-latest"}], [{"name":"geode"}], extract_geode_bash_task(), caches=[{"path":"geode"}])
- in_parallel:
steps:
- #@ bash_task("clang-tidy", [{"name":"source"}, {"name":"geode"}], [{"name":"build"}], clang_tidy_bash_task(), "clang-tools-image")
@@ -414,7 +439,7 @@ plan:
#@ end

---
#@ def bash_task(name, inputs, outputs, source, image="task-image", params={}, platform="linux", timeout=None, attempts=None):
#@ def bash_task(name, inputs, outputs, source, image="task-image", params={}, platform="linux", timeout=None, attempts=None, caches=None):
task: #@ name
#@ if timeout != None:
timeout: #@ timeout
@@ -427,6 +452,7 @@ config:
platform: #@ platform
inputs: #@ inputs
outputs: #@ outputs
caches: #@ caches
params:
_: #@ template.replace(params)
run:
@@ -140,6 +140,14 @@ jobs:
globs:
- "package-*-release-*/apache-geode-native-*"

#@overlay/append
- name: bump-minor-version
plan:
- put: version
params:
bump: minor
pre: #@ data.values.pipeline.pre

#@overlay/match-child-defaults missing_ok=True
groups:
#@overlay/match by="name"
@@ -149,7 +157,7 @@ groups:
- #@ packer_job_name(build)
- #@ docker_job_name("task-image")
- #@ docker_job_name("clang-tools-image")
#@overlay/match by="name"
#@overlay/match by="name"
- name: all
jobs:
#@ for build in data.values.builds:
@@ -163,4 +171,11 @@ groups:
#@overlay/append
- #@ version_source_job_name()
#@overlay/append
- bump-minor-version
#@overlay/append
- github-pre-release
#@overlay/match by="name"
- name: version
jobs:
- #@ version_source_job_name()
- bump-minor-version
@@ -25,6 +25,8 @@ Options:
Parameter Description Default
--target Fly target. "default"
--branch Branch to build. Current checked out branch.
--version Version of Geode. 1.14.0
--pre Version pre release tag. "" | Empty
--pipeline Name of pipeline to set. Based on repository owner name and branch.
--github-owner GitHub owner for repository. Current tracking branch repository owner.
--github-repository GitHub repository name. Current tracking branch repository name.
@@ -67,6 +69,7 @@ done

ytt=${ytt:-ytt}
fly=${fly:-fly}
yq=${yq:-yq}

target=${target:-default}
output=${output:-$(mktemp -d)}
@@ -84,6 +87,16 @@ fi
pipeline=${pipeline:-${github_owner}-${branch}}
pipeline=${pipeline//[^[:word:]-]/-}
if (which ${yq} >/dev/null); then
version=${version:-$(bash -c "${yq} \"\$@\"" yq -N e '.pipeline.version | select(.) ' - < base/base.yml)}
pre=${pre:-$(bash -c "${yq} \"\$@\"" yq -N e '.pipeline.pre | select(.) ' - < base/base.yml)}
fi
pre=${pre:-"build"}
if [ "${pre}" == "none" ]; then
pre=""
fi
google_project=${google_project:-$(gcloud config get-value project)}
google_zone=${google_zone:-'$(curl "http://metadata.google.internal/computeMetadata/v1/instance/zone" -H "Metadata-Flavor: Google" -s | cut -d / -f 4)'}
google_storage_bucket=${google_storage_bucket:-${google_project}-concourse}
@@ -100,6 +113,8 @@ for variant in ${variants}; do
--file base \
--file ${variant} \
--data-value "pipeline.name=${pipeline}" \
--data-value "pipeline.version=${version}" \
--data-value "pipeline.pre=${pre}" \
--data-value "pipeline.variant=${variant}" \
--data-value "repository.branch=${branch}" \
--data-value "github.owner=${github_owner}" \
@@ -121,8 +121,7 @@
"rhel/disable-selinux.sh",
"rhel/7/install-devtoolset.sh",
"rhel/install-jdk-11.sh",
"linux/install-cmake.sh",
"linux/install-geode.sh"
"linux/install-cmake.sh"
]
},
{
@@ -120,8 +120,7 @@
"scripts": [
"rhel/disable-selinux.sh",
"rhel/install-jdk-11.sh",
"linux/install-cmake.sh",
"linux/install-geode.sh"
"linux/install-cmake.sh"
]
},
{
@@ -100,7 +100,6 @@
"execute_command": "{{.Vars}} sudo -E -S bash '{{.Path}}'",
"scripts": [
"linux/install-cmake.sh",
"linux/install-geode.sh",
"ubuntu/install-clang-format.sh",
"ubuntu/install-jdk-11.sh"
]
@@ -100,7 +100,6 @@
"execute_command": "{{.Vars}} sudo -E -S bash '{{.Path}}'",
"scripts": [
"linux/install-cmake.sh",
"linux/install-geode.sh",
"ubuntu/install-clang-format.sh",
"ubuntu/install-jdk-11.sh"
]
@@ -101,7 +101,6 @@
"scripts": [
"ubuntu/install-clang-format.sh",
"linux/install-cmake.sh",
"linux/install-geode.sh",
"ubuntu/install-jdk-11.sh"
]
},
@@ -95,12 +95,6 @@
"choco install nuget.commandline -confirm"
]
},
{
"type": "powershell",
"scripts": [
"windows/install-geode.ps1"
]
},
{
"type": "file",
"source": "windows/aws/init-ssh.ps1",

0 comments on commit 38fa9ee

Please sign in to comment.