From c08b3d8c604ef42d05ef6cd2dee3bdea99c3d796 Mon Sep 17 00:00:00 2001 From: Jeremiah Snapp Date: Wed, 19 Jan 2022 20:24:47 -0500 Subject: [PATCH] Make it easy to reproduce verify pipeline builds locally This makes it easy to reproduce failing builds as well as provide an environment for debugging, training and exploring the omnibus build process. Signed-off-by: Jeremiah Snapp --- .gitignore | 1 + Dockerfile | 13 ------------- README.md | 28 +++++++++++++++++++++++++--- Rakefile | 6 +++--- docker-compose.yml | 15 +++++++++++++++ test/Gemfile | 8 +++++++- test/generate_steps.rb | 2 +- test/omnibus-build.sh | 20 ++++++++++++++++++++ 8 files changed, 72 insertions(+), 21 deletions(-) create mode 100644 docker-compose.yml create mode 100644 test/omnibus-build.sh diff --git a/.gitignore b/.gitignore index 2b87b19c2..0a87bf0b9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .bundle Gemfile.lock pkg/* +test/pkg/* /binstubs /bin diff --git a/Dockerfile b/Dockerfile index 76699dc4f..29832e29d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,3 @@ FROM chefes/releng-base -ARG SOFTWARE -ARG VERSION - -COPY ./ /omnibus-software -COPY ./test /test - -WORKDIR /test - RUN curl -L https://omnitruck.chef.io/install.sh | bash -s -- -P omnibus-toolchain - -RUN . /opt/omnibus-toolchain/bin/load-omnibus-toolchain \ - && DEBUG=1 bundle config set --local without development \ - && DEBUG=1 bundle install \ - && bundle exec omnibus build test \ No newline at end of file diff --git a/README.md b/README.md index 633db8fd4..ebffb8f9a 100644 --- a/README.md +++ b/README.md @@ -30,14 +30,36 @@ This repository is versioned and tagged using the `YY.MM.BUILD` to allow folks t For information on contributing to this project please see our [Contributing Documentation](https://github.com/chef/chef/blob/main/CONTRIBUTING.md) -### Testing via Docker +### Testing On Ubuntu 18.04 via Docker -We provide a sample Dockerfile you can use to ensure that your software definitions are able to compile on Ubuntu 18.04. +#### Interactive Testing + +If you want to enter an interactive shell in a dockerized omnibus build environment on Ubuntu 18.04 you can run the following command. ``` -bundle exec rake test_build () +docker-compose run --rm -e SOFTWARE= -e VERSION= builder ``` +Now you should be in a shell in the container ready to explore, modify or build. + +You can start the build by running the following command in your shell. + +``` +bundle exec omnibus build test +``` + +The container will automatically be destroyed when you exit it without requiring any further cleanup. + +#### Non-Interactive Testing + +If you only want to run a build without an interactive shell you can set the `CI` environment variable as you see in the following command. + +``` +docker-compose run --rm -e SOFTWARE= -e VERSION= -e CI=true builder +``` + +The container will automatically be destroyed when you exit it without requiring any further cleanup. + ## License & Copyright - Copyright:: Copyright (c) 2012-2021 Chef Software, Inc. diff --git a/Rakefile b/Rakefile index c9adcd5d2..c2630f0b5 100644 --- a/Rakefile +++ b/Rakefile @@ -27,9 +27,9 @@ task :test_build do raise "\nERROR: You must specify a software name\n\n" if software.nil? - command = "docker build --build-arg SOFTWARE=#{software}" - command += " --build-arg VERSION=#{version}" unless version == "default" - command += " --tag localhost:5000/omnibus-#{software}:#{version} ." + command = "docker-compose run --rm -e CI=true -e SOFTWARE=#{software}" + command += " -e VERSION=#{version}" unless version == "default" + command += " builder" sh command end diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..507e0254d --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,15 @@ +services: + builder: + build: . + volumes: + - ./:/omnibus-software + - ./test:/test + working_dir: /test + command: bash --init-file omnibus-build.sh + + # Multiple concurrent "docker-compose run" commands on the same system in the CI pipeline caused intermittent failures + # while creating the network. + # Using "network_mode: bridge" forces compose to use docker's legacy default bridge network instead. + # We considered namespacing the network using COMPOSE_PROJECT_NAME=$BUILDKITE_JOB_ID but using the legacy bridge network + # means we can avoid worrying about any impact and management overhead that multiple networks might create. + network_mode: bridge diff --git a/test/Gemfile b/test/Gemfile index 22a3c02b0..7f1edf58d 100644 --- a/test/Gemfile +++ b/test/Gemfile @@ -1,4 +1,10 @@ source "https://rubygems.org" gem "omnibus", github: "chef/omnibus", branch: "main" -gem "omnibus-software", path: "/omnibus-software" \ No newline at end of file +gem "omnibus-software", path: "/omnibus-software" + +group :development do + gem "pry" + gem "pry-byebug" + gem "pry-stack_explorer" +end diff --git a/test/generate_steps.rb b/test/generate_steps.rb index 5f862b329..890395d02 100755 --- a/test/generate_steps.rb +++ b/test/generate_steps.rb @@ -41,7 +41,7 @@ def method_missing(m, *args, &block) $versions.compact.uniq.each do |version| puts <<~EOH - label: "test-build (#{software} #{version})" - command: docker build --build-arg SOFTWARE=#{software} --build-arg VERSION=#{version} . + command: docker-compose run --rm -e SOFTWARE=#{software} -e VERSION=#{version} -e CI builder timeout_in_minutes: 30 expeditor: executor: diff --git a/test/omnibus-build.sh b/test/omnibus-build.sh new file mode 100644 index 000000000..e7bc1c4a7 --- /dev/null +++ b/test/omnibus-build.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# shellcheck disable=SC1090 +. ~/.bashrc + +# shellcheck disable=SC1091 +. /opt/omnibus-toolchain/bin/load-omnibus-toolchain + +if [[ $CI == true ]]; then + DEBUG=1 bundle config set --local without development +fi + +rm -f Gemfile.lock + +DEBUG=1 bundle install + +if [[ $CI == true ]]; then + bundle exec omnibus build test + exit $? +fi