From e7f8d3b12ca63ebe96722203475d1792cdab5cfd Mon Sep 17 00:00:00 2001 From: Schneems Date: Thu, 21 Mar 2024 08:54:54 -0500 Subject: [PATCH] Add AMD (x86_64) and ARM (aarch64) on Heroku-24 AMD/x86_64 is the architecture used for all prior base images. Heroku-24 (Ubuntu 24.04) base image is provided with support for ARM/aarch64 (think m1 Mac or graviton AWS server) support. This PR does several things: - Introduces support for heroku-24 base image - Build both arm64 and amd64 architecture binaries --- .github/workflows/build_ruby.yml | 40 ++++++++++++++++++++++ .github/workflows/ci.yml | 2 +- Rakefile | 20 +++++++++-- build.rb | 1 + dockerfiles/Dockerfile.heroku-24 | 17 +++++++++ lib/build_script.rb | 40 ++++++++++++++++++---- lib/docker_command.rb | 12 +++++-- rubies/heroku-24/ruby-3.2.3.sh | 5 +++ spec/unit/build_script_spec.rb | 59 ++++++++++++++++++++++++++++++++ spec/unit/docker_command_spec.rb | 40 +++++++++++++++++++--- 10 files changed, 221 insertions(+), 15 deletions(-) create mode 100644 dockerfiles/Dockerfile.heroku-24 create mode 100755 rubies/heroku-24/ruby-3.2.3.sh diff --git a/.github/workflows/build_ruby.yml b/.github/workflows/build_ruby.yml index 60b3ffd..6692f77 100644 --- a/.github/workflows/build_ruby.yml +++ b/.github/workflows/build_ruby.yml @@ -78,3 +78,43 @@ jobs: run: aws s3 sync ./builds "s3://${S3_BUCKET}" - name: Output Rubygems version run: bundle exec rake "rubygems_version[${{inputs.ruby_version}},$STACK]" + + build-and-upload-heroku-24: + if: (!startsWith(inputs.ruby_version, '3.0')) # https://bugs.ruby-lang.org/issues/18658 + runs-on: pub-hk-ubuntu-22.04-xlarge + env: + STACK: "heroku-24" + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up Ruby + uses: ruby/setup-ruby@ec02537da5712d66d4d50a0f33b7eb52773b5ed1 + with: + ruby-version: '3.2' + - name: Install dependencies + run: bundle install + - name: Output CHANGELOG + run: bundle exec rake "changelog[${{inputs.ruby_version}}]" + - name: Build Docker image + run: bundle exec rake "generate_image[$STACK]" + - name: Generate Ruby Dockerfile + run: bundle exec rake "new[${{inputs.ruby_version}},$STACK]" + - name: Build and package Ruby runtime + run: bash "rubies/$STACK/ruby-${{inputs.ruby_version}}.sh" + - name: Build and package Ruby runtime amd64 + run: | + export DOCKER_DEFAULT_PLATFORM=linux/amd64 + bash "rubies/$STACK/ruby-${{inputs.ruby_version}}.sh" + - name: Build and package Ruby runtime arm64 + run: | + export DOCKER_DEFAULT_PLATFORM=linux/arm64 + bash "rubies/$STACK/ruby-${{inputs.ruby_version}}.sh" + - name: Upload Ruby runtime archive to S3 dry run + if: (inputs.dry_run) + run: aws s3 sync ./builds "s3://${S3_BUCKET}" --dryrun + - name: Upload Ruby runtime archive to S3 production + if: (!inputs.dry_run) + run: aws s3 sync ./builds "s3://${S3_BUCKET}" + - name: Output Rubygems version + run: bundle exec rake "rubygems_version[${{inputs.ruby_version}},$STACK]" + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1452820..b372e6b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,7 +35,7 @@ jobs: runs-on: pub-hk-ubuntu-22.04-xlarge strategy: matrix: - stack: ["heroku-20", "heroku-22"] + stack: ["heroku-20", "heroku-22", "heroku-24"] version: ["3.1.4"] steps: - name: Checkout diff --git a/Rakefile b/Rakefile index 8996475..2e89edb 100644 --- a/Rakefile +++ b/Rakefile @@ -79,8 +79,24 @@ end desc "Build docker image for stack" task :generate_image, [:stack] do |t, args| require "fileutils" - FileUtils.cp("dockerfiles/Dockerfile.#{args[:stack]}", "Dockerfile") - system("docker build -t hone/ruby-builder:#{args[:stack]} .") + stack = args[:stack] + FileUtils.cp("dockerfiles/Dockerfile.#{stack}", "Dockerfile") + image = "hone/ruby-builder:#{stack}" + arguments = ["-t #{image}"] + + # rubocop:disable Lint/EmptyWhen + case stack + when "heroku-24" + arguments.push("--platform='linux/amd64,linux/arm64'") + when "heroku-20", "heroku-22" + else + raise "Unknown stack: #{stack}" + end + # rubocop:enable Lint/EmptyWhen + + command = "docker build #{arguments.join(" ")} ." + puts "Running: `#{command}`" + system(command) FileUtils.rm("Dockerfile") end diff --git a/build.rb b/build.rb index e19b538..fc162a4 100644 --- a/build.rb +++ b/build.rb @@ -7,6 +7,7 @@ run_build_script( stack: ENV.fetch("STACK"), + architecture: get_architecture, ruby_version: ENV.fetch("VERSION"), workspace_dir: ARGV[0], output_dir: ARGV[1], diff --git a/dockerfiles/Dockerfile.heroku-24 b/dockerfiles/Dockerfile.heroku-24 new file mode 100644 index 0000000..778926b --- /dev/null +++ b/dockerfiles/Dockerfile.heroku-24 @@ -0,0 +1,17 @@ +FROM heroku/heroku:24-build + +USER root + +# setup workspace +RUN rm -rf /tmp/workspace +RUN mkdir -p /tmp/workspace + +RUN apt-get update -y; apt-get install ruby -y +RUN curl https://sh.rustup.rs -sSf | sh -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + +# output dir is mounted + +ADD build.rb /tmp/build.rb +COPY lib/ /tmp/lib/ +CMD ["ruby", "/tmp/build.rb", "/tmp/workspace", "/tmp/output", "/tmp/cache"] diff --git a/lib/build_script.rb b/lib/build_script.rb index a9d072a..ad73463 100644 --- a/lib/build_script.rb +++ b/lib/build_script.rb @@ -25,14 +25,13 @@ def run!(cmd) # Build logic in a method def run_build_script( - io: $stdout, + architecture:, io: $stdout, workspace_dir: ARGV[0], output_dir: ARGV[1], cache_dir: ARGV[2], stack: ENV.fetch("STACK"), ruby_version: ENV.fetch("STACK") ) - parts = VersionParts.new(ruby_version) ruby_version = RubyVersion.new(ruby_version) @@ -74,10 +73,12 @@ def run_build_script( dir: ruby_binary_dir.join("bin") ) - destination = Pathname(output_dir) - .join(stack) - .tap(&:mkpath) - .join(ruby_version.tar_file_name_output) + destination = stack_architecture_tar_file_name( + stack: stack, + output_dir: output_dir, + architecture: architecture, + tar_file_name_output: ruby_version.tar_file_name_output + ) io.puts "Writing #{destination}" tar_dir( @@ -88,6 +89,20 @@ def run_build_script( end end +# Returns a Pathname to the destination tar file +# +# The directory structure corresponds to the S3 directory structure directly +def stack_architecture_tar_file_name(stack:, output_dir:, tar_file_name_output:, architecture:) + output_stack_dir = Pathname(output_dir).join(stack) + + case stack + when "heroku-24" + output_stack_dir.join(architecture) + else + output_stack_dir + end.tap(&:mkpath).join(tar_file_name_output) +end + # Runs a command on the command line and streams the results def pipe(command, io: $stdout) output = "" @@ -199,3 +214,16 @@ def fix_binstubs_in_dir(dir:, io: $stdout) end end end + +def get_architecture(system_output: `arch`, success: $?.success?) + raise "Error running `arch`: #{system_output}" unless success + + case system_output.strip + when "x86_64" + "amd64" + when "aarch64" + "arm64" + else + raise "Unknown architecture: #{system_output}" + end +end diff --git a/lib/docker_command.rb b/lib/docker_command.rb index 95159ae..f802371 100644 --- a/lib/docker_command.rb +++ b/lib/docker_command.rb @@ -1,5 +1,13 @@ +require "build_script" + module DockerCommand - def self.gem_version_from_tar(ruby_version:, stack:) - "docker run -v $(pwd)/builds/#{stack}:/tmp/output hone/ruby-builder:#{stack} bash -c \"mkdir /tmp/unzipped && tar xzf /tmp/output/#{ruby_version.tar_file_name_output} -C /tmp/unzipped && echo 'Rubygems version is: ' && /tmp/unzipped/bin/gem -v\"" + def self.gem_version_from_tar(ruby_version:, stack:, system_output: `docker run --rm hone/ruby-builder:#{stack} arch`, success: $?.success?) + ruby_tar_file = stack_architecture_tar_file_name( + stack: stack, + output_dir: "/tmp/output", + architecture: get_architecture(system_output: system_output, success: success), + tar_file_name_output: ruby_version.tar_file_name_output + ) + "docker run -v $(pwd)/builds:/tmp/output hone/ruby-builder:#{stack} bash -c \"mkdir /tmp/unzipped && tar xzf #{ruby_tar_file} -C /tmp/unzipped && echo 'Rubygems version is: ' && /tmp/unzipped/bin/gem -v\"" end end diff --git a/rubies/heroku-24/ruby-3.2.3.sh b/rubies/heroku-24/ruby-3.2.3.sh new file mode 100755 index 0000000..6c0317d --- /dev/null +++ b/rubies/heroku-24/ruby-3.2.3.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +source `dirname $0`/../common.sh + +docker run -v $OUTPUT_DIR:/tmp/output -v $CACHE_DIR:/tmp/cache -e VERSION=3.2.3 -e STACK=heroku-24 hone/ruby-builder:heroku-24 diff --git a/spec/unit/build_script_spec.rb b/spec/unit/build_script_spec.rb index bfb0fd9..1c4d888 100644 --- a/spec/unit/build_script_spec.rb +++ b/spec/unit/build_script_spec.rb @@ -68,4 +68,63 @@ expect(unzip_dir.entries.map(&:to_s)).to include("bar.txt") end end + + it "gets system architecture" do + expect(get_architecture(system_output: "x86_64", success: true)).to eq("amd64") + expect(get_architecture(system_output: "aarch64", success: true)).to eq("arm64") + + expect { + get_architecture( + system_output: "No such command", + success: false + ) + }.to raise_error("Error running `arch`: No such command") + + expect { + get_architecture( + system_output: "lol", + success: true + ) + }.to raise_error("Unknown architecture: lol") + end + + it "is stack and architecture aware when writing tar files for heroku-24" do + actual = stack_architecture_tar_file_name( + stack: "heroku-24", + output_dir: "/tmp/output", + architecture: "amd64", + tar_file_name_output: "ruby-3.1.2.tgz" + ) + + expect(actual).to eq(Pathname("/tmp").join("output").join("heroku-24").join("amd64").join("ruby-3.1.2.tgz")) + + actual = stack_architecture_tar_file_name( + stack: "heroku-24", + output_dir: "/tmp/output", + architecture: "arm64", + tar_file_name_output: "ruby-3.1.2.tgz" + ) + + expect(actual).to eq(Pathname("/tmp").join("output").join("heroku-24").join("arm64").join("ruby-3.1.2.tgz")) + end + + it "is architecture agnostic when writing tar files for heroku-22 and heroku-20" do + actual = stack_architecture_tar_file_name( + stack: "heroku-20", + output_dir: "/tmp/output", + architecture: "amd64", + tar_file_name_output: "ruby-3.1.2.tgz" + ) + + expect(actual).to eq(Pathname("/tmp").join("output").join("heroku-20").join("ruby-3.1.2.tgz")) + + actual = stack_architecture_tar_file_name( + stack: "heroku-22", + output_dir: "/tmp/output", + architecture: "arm64", + tar_file_name_output: "ruby-3.1.2.tgz" + ) + + expect(actual).to eq(Pathname("/tmp").join("output").join("heroku-22").join("ruby-3.1.2.tgz")) + end end diff --git a/spec/unit/docker_command_spec.rb b/spec/unit/docker_command_spec.rb index c23ccf1..d0de78e 100644 --- a/spec/unit/docker_command_spec.rb +++ b/spec/unit/docker_command_spec.rb @@ -3,14 +3,46 @@ describe DockerCommand do it "Generates docker command for outputting rubygems versions" do - actual = DockerCommand.gem_version_from_tar(ruby_version: RubyVersion.new("3.1.4"), stack: "heroku-22") - expected = %{docker run -v $(pwd)/builds/heroku-22:/tmp/output hone/ruby-builder:heroku-22 bash -c "mkdir /tmp/unzipped && tar xzf /tmp/output/ruby-3.1.4.tgz -C /tmp/unzipped && echo 'Rubygems version is: ' && /tmp/unzipped/bin/gem -v"} + actual = DockerCommand.gem_version_from_tar( + ruby_version: RubyVersion.new("3.1.4"), + stack: "heroku-22", + system_output: "x86_64", + success: true + ) + expected = %{docker run -v $(pwd)/builds:/tmp/output hone/ruby-builder:heroku-22 bash -c "mkdir /tmp/unzipped && tar xzf /tmp/output/heroku-22/ruby-3.1.4.tgz -C /tmp/unzipped && echo 'Rubygems version is: ' && /tmp/unzipped/bin/gem -v"} + expect(actual).to eq(expected) + end + + it "Works with amd and arm" do + actual = DockerCommand.gem_version_from_tar( + ruby_version: RubyVersion.new("3.1.4"), + stack: "heroku-24", + system_output: "x86_64", + success: true + ) + + expected = %{docker run -v $(pwd)/builds:/tmp/output hone/ruby-builder:heroku-24 bash -c "mkdir /tmp/unzipped && tar xzf /tmp/output/heroku-24/amd64/ruby-3.1.4.tgz -C /tmp/unzipped && echo 'Rubygems version is: ' && /tmp/unzipped/bin/gem -v"} + expect(actual).to eq(expected) + + actual = DockerCommand.gem_version_from_tar( + ruby_version: RubyVersion.new("3.1.4"), + stack: "heroku-24", + system_output: "aarch64", + success: true + ) + + expected = %{docker run -v $(pwd)/builds:/tmp/output hone/ruby-builder:heroku-24 bash -c "mkdir /tmp/unzipped && tar xzf /tmp/output/heroku-24/arm64/ruby-3.1.4.tgz -C /tmp/unzipped && echo 'Rubygems version is: ' && /tmp/unzipped/bin/gem -v"} expect(actual).to eq(expected) end it "works with preview releases" do - actual = DockerCommand.gem_version_from_tar(ruby_version: RubyVersion.new("3.3.0-preview2"), stack: "heroku-22") - expected = %{docker run -v $(pwd)/builds/heroku-22:/tmp/output hone/ruby-builder:heroku-22 bash -c "mkdir /tmp/unzipped && tar xzf /tmp/output/ruby-3.3.0.preview2.tgz -C /tmp/unzipped && echo 'Rubygems version is: ' && /tmp/unzipped/bin/gem -v"} + actual = DockerCommand.gem_version_from_tar( + ruby_version: RubyVersion.new("3.3.0-preview2"), + stack: "heroku-22", + system_output: "x86_64", + success: true + ) + expected = %{docker run -v $(pwd)/builds:/tmp/output hone/ruby-builder:heroku-22 bash -c "mkdir /tmp/unzipped && tar xzf /tmp/output/heroku-22/ruby-3.3.0.preview2.tgz -C /tmp/unzipped && echo 'Rubygems version is: ' && /tmp/unzipped/bin/gem -v"} expect(actual).to eq(expected) end end