update verify pipeline and omnibus build/test to use containers

.buildkite-platform.json

@@ -0,0 +1,4 @@
+{
+    "chef_foundation": "0.1.24",
+    "omnibus_toolchain": "3.0.0"
+}

.buildkite/build-test-omnibus.sh

@@ -0,0 +1,112 @@
+if [[ $BUILDKITE_ORGANIZATION_SLUG == "chef-oss" ]]; then
+  echo "- block: Build & Test Omnibus Packages"
+  echo "  prompt: Continue to run omnibus package build and tests for applicable platforms?"
+fi
+
+FILTER="${OMNIBUS_FILTER:=*}"
+
+platforms=("amazon-2:centos-7" "centos-6:centos-6" "centos-7:centos-7" "centos-8:centos-8" "rhel-9:rhel-9" "debian-9:debian-9" "debian-10:debian-9" "debian-11:debian-9" "ubuntu-1604:ubuntu-1604" "ubuntu-1804:ubuntu-1604" "ubuntu-2004:ubuntu-1604" "ubuntu-2204:ubuntu-1604" "sles-15:sles-15" "windows-2019:windows-2019")
+
+omnibus_build_platforms=()
+omnibus_test_platforms=()
+
+# build build array and test array based on filter
+for platform in ${platforms[@]}; do
+    case ${platform%:*} in
+        $FILTER)
+            omnibus_build_platforms[${#omnibus_build_platforms[@]}]=${platform#*:}
+            omnibus_test_platforms[${#omnibus_test_platforms[@]}]=$platform
+            ;;
+    esac
+done
+
+# remove duplicates from build array
+omnibus_build_platforms=($(printf "%s\n" "${omnibus_build_platforms[@]}" | sort -u | tr '\n' ' '))
+
+for platform in ${omnibus_build_platforms[@]}; do
+  if [[ $platform != *"windows"* ]]; then
+    echo "- label: \":hammer_and_wrench::docker: $platform\""
+    echo "  retry:"
+    echo "    automatic:"
+    echo "      limit: 1"
+    echo "  key: build-$platform"
+    echo "  agents:"
+    echo "    queue: default-privileged"
+    echo "  plugins:"
+    echo "  - docker#v3.5.0:"
+    echo "      image: chefes/omnibus-toolchain-$platform:$OMNIBUS_TOOLCHAIN_VERSION"
+    echo "      privileged: true"
+    echo "      propagate-environment: true"
+    echo "      environment:"
+    echo "        - RPM_SIGNING_KEY"
+    echo "        - CHEF_FOUNDATION_VERSION"
+    echo "  commands:"
+    echo "    - ./.expeditor/scripts/omnibus_chef_build.sh"
+    echo "  timeout_in_minutes: 60"
+  else 
+    echo "- label: \":hammer_and_wrench::windows: $platform\""
+    echo "  retry:"
+    echo "    automatic:"
+    echo "      limit: 1"
+    echo "  key: build-$platform"
+    echo "  agents:"
+    echo "    queue: default-$platform-privileged"
+    echo "  plugins:"
+    echo "  - docker#v3.5.0:"
+    echo "      image: chefes/omnibus-toolchain-$platform:$OMNIBUS_TOOLCHAIN_VERSION"
+    echo "      shell:"
+    echo "      - powershell"
+    echo "      - \"-Command\""
+    echo "      propagate-environment: true"
+    echo "      environment:"
+    echo "        - CHEF_FOUNDATION_VERSION"
+    echo "        - BUILDKITE_AGENT_ACCESS_TOKEN"
+    echo "        - AWS_ACCESS_KEY_ID"
+    echo "        - AWS_SECRET_ACCESS_KEY"
+    echo "        - AWS_SESSION_TOKEN"
+    echo "      volumes:"
+    echo '        - "c:\\buildkite-agent:c:\\buildkite-agent"'
+    echo "  commands:"
+    echo "    - ./.expeditor/scripts/omnibus_chef_build.ps1"
+    echo "  timeout_in_minutes: 60"
+  fi
+done
+
+echo "- wait: ~"
+
+for platform in ${omnibus_test_platforms[@]}; do
+  if [[ $platform != *"windows"* ]]; then
+    echo "- env:"
+    echo "    OMNIBUS_BUILDER_KEY: build-${platform#*:}"
+    echo "  label: \":mag::docker: ${platform%:*}\""
+    echo "  retry:"
+    echo "    automatic:"
+    echo "      limit: 1"
+    echo "  agents:"
+    echo "    queue: default-privileged"
+    echo "  plugins:"
+    echo "  - docker#v3.5.0:"
+    echo "      image: chefes/omnibus-toolchain-${platform%:*}:$OMNIBUS_TOOLCHAIN_VERSION"
+    echo "      privileged: true"
+    echo "      propagate-environment: true"
+    echo "  commands:"
+    echo "    - ./.expeditor/scripts/download_built_omnibus_pkgs.sh"
+    echo "    - omnibus/omnibus-test.sh"
+    echo "  timeout_in_minutes: 60"
+  else
+    echo "- env:"
+    echo "    OMNIBUS_BUILDER_KEY: build-windows-2019"
+    echo "  key: test-windows-2019"
+    echo '  label: ":mag::windows: windows-2019"'
+    echo "  retry:"
+    echo "    automatic:"
+    echo "      limit: 1"
+    echo "  agents:"
+    echo "    queue: default-windows-2019-privileged"
+    echo "  commands:"
+    echo "    - ./.expeditor/scripts/download_built_omnibus_pkgs.ps1"
+    echo "    - ./omnibus/omnibus-test.ps1"
+    echo "  timeout_in_minutes: 60"
+  fi
+done
+

.buildkite/hooks/pre-command

@@ -7,6 +7,27 @@ set -eu
 
 docker ps || true
 
+# Get chef foundation version from the json file
+CHEF_FOUNDATION_VERSION=$(cat .buildkite-platform.json | jq -r '.chef_foundation')
+export CHEF_FOUNDATION_VERSION
+echo $CHEF_FOUNDATION_VERSION
+
+OMNIBUS_TOOLCHAIN_VERSION=$(cat .buildkite-platform.json | jq -r '.omnibus_toolchain')
+export OMNIBUS_TOOLCHAIN_VERSION
+echo $OMNIBUS_TOOLCHAIN_VERSION
+
+if [ $BUILDKITE_STEP_KEY == "build-windows-2019" ] && [ $BUILDKITE_ORGANIZATION_SLUG == "chef" ]
+then
+  TOKEN=$(curl -sX PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
+  RESPONSE=$(curl -sH "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/iam/security-credentials/default-windows-2019-privileged-chef-Role)
+  AWS_ACCESS_KEY_ID=$(echo $RESPONSE | jq -r '.AccessKeyId')
+  export AWS_ACCESS_KEY_ID
+  AWS_SECRET_ACCESS_KEY=$(echo $RESPONSE | jq -r '.SecretAccessKey')
+  export AWS_SECRET_ACCESS_KEY
+  AWS_SESSION_TOKEN=$(echo $RESPONSE | jq -r '.Token')
+  export AWS_SESSION_TOKEN
+fi
+
 # We've now seen cases where origin/main on the build hosts can get
 # out of date. This causes us to build components unnecessarily.
 # Fetching it here hopefully will prevent this situation.
@@ -35,3 +56,10 @@ if [[ "$BUILDKITE_BRANCH" != "main" ]]; then
       "Couldn't rebase onto main ([${main}](${github}${main})), building PR HEAD ([${pr_head}](${github}${pr_head}))."
   fi
 fi
+
+# Only execute if on RHEL/CentOS/SLES
+if [[ "$BUILDKITE_LABEL" =~ rhel|sles|centos ]] && [[ $BUILDKITE_ORGANIZATION_SLUG != "chef-oss" ]]; then
+  export VAULT_ADDR="https://vault.ps.chef.co"
+  export VAULT_TOKEN="$(vault login -method=aws -path=aws/private-cd -token-only header_value=vault.ps.chef.co role=ci)"
+  export RPM_SIGNING_KEY="$(vault kv get -field packages_at_chef_io account/static/packages/signing_certs)"
+fi

.buildkite/verify.adhoc.pipeline.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# exit immediately on failure, or if an undefined variable is used
+set -eu
+
+echo "---"
+echo "steps:"
+echo ""
+
+# include build and test omnibus pipeline
+DIR="${BASH_SOURCE%/*}"
+if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
+source "$DIR/build-test-omnibus.sh"

.buildkite/verify.pipeline.sh

@@ -0,0 +1,173 @@
+#!/bin/bash
+
+# exit immediately on failure, or if an undefined variable is used
+set -eu
+
+echo "---"
+echo "steps:"
+echo ""
+
+test_platforms=("centos-6" "centos-7" "centos-8" "rhel-9" "debian-9" "ubuntu-1604" "sles-15")
+
+for platform in ${test_platforms[@]}; do
+  echo "- label: \"{{matrix}} $platform :ruby:\""
+  echo "  retry:"
+  echo "    automatic:"
+  echo "      limit: 1"
+  echo "  agents:"
+  echo "    queue: default-privileged"
+  echo "  matrix:"
+  echo "    - \"Unit\""
+  echo "    - \"Integration\""
+  echo "    - \"Functional\""
+  echo "  plugins:"
+  echo "  - docker#v3.5.0:"
+  echo "      image: chefes/omnibus-toolchain-${platform#*:}:$OMNIBUS_TOOLCHAIN_VERSION"
+  echo "      privileged: true"
+  echo "      environment:"
+  echo "        - CHEF_FOUNDATION_VERSION"
+  echo "      propagate-environment: true"
+  echo "  commands:"
+  echo "    - .expeditor/scripts/prep_and_run_tests.sh {{matrix}}"
+  echo "  timeout_in_minutes: 60"
+done
+
+win_test_platforms=("windows-2019:windows-2019")
+
+for platform in ${win_test_platforms[@]}; do
+  echo "- label: \"{{matrix}} ${platform#*:} :windows:\""
+  echo "  retry:"
+  echo "    automatic:"
+  echo "      limit: 1"
+  echo "  agents:"
+  echo "    queue: default-${platform%:*}-privileged"
+  echo "  matrix:"
+  echo "    - \"Unit\""
+  echo "    - \"Integration\""
+  echo "  plugins:"
+  echo "  - docker#v3.5.0:"
+  echo "      image: chefes/omnibus-toolchain-${platform#*:}:$OMNIBUS_TOOLCHAIN_VERSION"
+  echo "      shell:"
+  echo "      - powershell"
+  echo "      - \"-Command\""
+  echo "      environment:"
+  echo "        - CHEF_FOUNDATION_VERSION"
+  echo "      propagate-environment: true"
+  echo "  commands:"
+  echo "    - .\.expeditor\scripts\prep_and_run_tests.ps1 {{matrix}}"
+  echo "  timeout_in_minutes: 60"
+
+done
+
+for platform in ${win_test_platforms[@]}; do
+  echo "- label: \"Functional ${platform#*:} :windows:\""
+  echo "  retry:"
+  echo "    automatic:"
+  echo "      limit: 1"
+  echo "  commands:"
+  echo "    - .\.expeditor\scripts\prep_and_run_tests.ps1 Functional"
+  echo "  agents:"
+  echo "    queue: single-use-windows-2019-privileged"
+  echo "  env:"
+  echo "  - CHEF_FOUNDATION_VERSION"
+  echo "    - .\.expeditor\scripts\prep_and_run_tests.ps1 {{matrix}}"
+  echo "  timeout_in_minutes: 60"
+done
+
+external_gems=("chef-zero" "cheffish" "chefspec" "knife-windows" "berkshelf")
+
+for gem in ${external_gems[@]}; do
+  echo "- label: \"$gem gem :ruby:\""
+  echo "  retry:"
+  echo "    automatic:"
+  echo "      limit: 1"
+  echo "  agents:"
+  echo "    queue: default"
+  echo "  plugins:"
+  echo "  - docker#v3.5.0:"
+  echo "      image: chefes/omnibus-toolchain-ubuntu-1804:$OMNIBUS_TOOLCHAIN_VERSION"
+  echo "      environment:"
+  echo "        - CHEF_FOUNDATION_VERSION"
+  if [ $gem == "chef-zero" ] 
+  then
+    echo "        - PEDANT_OPTS=--skip-oc_id"
+    echo "        - CHEF_FS=true"
+  fi
+  echo "      propagate-environment: true"
+  echo "  - chef/cache#v1.5.0:"
+  echo "      s3_bucket: core-buildkite-cache-chef-oss-prod"
+  echo "      cached_folders:"
+  echo "      - vendor"
+  echo "  timeout_in_minutes: 60"
+  echo "  commands:"
+  echo "    - .expeditor/scripts/bk_container_prep.sh"
+  if [ $gem == "berkshelf" ]
+  then
+    echo "    - export PATH=\"/opt/chef/bin:/usr/local/sbin:/usr/sbin:/sbin:${PATH}\""
+    echo "    - apt-get update -y"
+    # cspell:disable-next-line
+    echo "    - apt-get install -y graphviz"
+    echo "    - bundle config set --local without omnibus_package"
+  else
+    echo "    - export PATH=\"/opt/chef/bin:${PATH}\""
+    echo "    - bundle config set --local without omnibus_package"
+    echo "    - bundle config set --local path 'vendor/bundle'"
+  fi
+  echo "    - bundle install --jobs=3 --retry=3"
+  case $gem in 
+    "chef-zero")
+      echo "    - bundle exec tasks/bin/run_external_test chef/chef-zero main rake pedant"
+      ;;
+    "cheffish")
+      echo "    - bundle exec tasks/bin/run_external_test chef/cheffish main rake spec"
+      ;;
+    "chefspec")
+      echo "    - bundle exec tasks/bin/run_external_test chefspec/chefspec main rake"
+      ;;
+    "knife-windows")
+      echo "    - bundle exec tasks/bin/run_external_test chef/knife-windows main rake spec"
+      ;;
+    "berkshelf")
+      echo "    - bundle exec tasks/bin/run_external_test chef/berkshelf main rake"
+      ;;
+    *)
+      echo -e "\n Gem $gem is not valid\n" >&2
+      exit 1
+      ;;
+  esac
+done
+
+habitat_plans=("linux" "linux-kernel2" "windows")
+
+for plan in ${habitat_plans[@]}; do
+  echo "- label: \":habicat: $plan plan\""
+  echo "  retry:"
+  echo "    automatic:"
+  echo "      limit: 1"
+  echo "  agents:"
+  if [ $plan == "windows" ]
+  then
+    echo "    queue: single-use-windows-2019-privileged"
+  else
+    echo "    queue: single-use-privileged"
+  fi
+  echo "  plugins:"
+  echo "  - chef/cache#v1.5.0:"
+  echo "      s3_bucket: core-buildkite-cache-chef-oss-prod"
+  echo "      cached_folders:"
+  echo "      - vendor"
+  echo "  timeout_in_minutes: 60"
+  echo "  commands:"
+  if [ $plan == "windows" ]
+  then
+    echo "    - ./.expeditor/scripts/verify-plan.ps1"
+  else
+    echo "    - sudo ./.expeditor/scripts/install-hab.sh 'x86_64-$plan'"
+    echo "    - sudo ./.expeditor/scripts/verify-plan.sh"
+  fi
+done
+
+# include build and test omnibus pipeline
+DIR="${BASH_SOURCE%/*}"
+if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
+source "$DIR/build-test-omnibus.sh"

.expeditor/config.yml

@@ -30,6 +30,24 @@ pipelines:
       public: true
       env:
         - IGNORE_ARTIFACTORY_RUBY_PROXY: true # Artifactory is throwing 500's when downloading some gems.
+  - verify/release:
+      definition: .expeditor/verify.pipeline.yml
+      env:
+        - IGNORE_CACHE: true # caching causes constant build failures
+        - IGNORE_ARTIFACTORY_RUBY_PROXY: true
+  - verify/adhoc:
+      definition: .expeditor/verify.adhoc.pipeline.yml
+      env:
+        - ADHOC: true
+        - IGNORE_CACHE: true # caching causes constant build failures
+        - IGNORE_ARTIFACTORY_RUBY_PROXY: true # Artifactory is throwing 500's when downloading some gems.
+  - verify/adhoc-canary:
+      canary: true
+      definition: .expeditor/verify.adhoc.pipeline.yml
+      env:
+        - ADHOC: true
+        - IGNORE_CACHE: true # caching causes constant build failures
+        - IGNORE_ARTIFACTORY_RUBY_PROXY: true # Artifactory is throwing 500's when downloading some gems.
   - docker/build:
       definition: .expeditor/docker-build.pipeline.yml
       trigger: default
@@ -130,6 +148,11 @@ subscriptions:
             - "Expeditor: Skip Omnibus"
             - "Expeditor: Skip All"
           only_if: built_in:bump_version
+      - trigger_pipeline:verify/release:
+          ignore_labels:
+            - "Expeditor: Skip Omnibus"
+            - "Expeditor: Skip All"
+          only_if: built_in:bump_version
 
   # the habitat chain
   - workload: buildkite_hab_build_group_published:{{agent_id}}:*