Add AMD (x86_64) and ARM (aarch64) on Heroku-24

AMD/x86_64 is the architecture used for all prior base images. Heroku-24 (Ubuntu 24.04) base image is provided with support for ARM/aarch64 (think m1 Mac or graviton AWS server) support. This PR does several things:

- Introduces support for heroku-24 base image
- Build both arm64 and amd64 architecture binaries

.github/workflows/build_ruby.yml

@@ -78,3 +78,43 @@ jobs:
         run: aws s3 sync ./builds "s3://${S3_BUCKET}"
       - name: Output Rubygems version
         run: bundle exec rake "rubygems_version[${{inputs.ruby_version}},$STACK]"
+
+  build-and-upload-heroku-24:
+    if: (!startsWith(inputs.ruby_version, '3.0')) # https://bugs.ruby-lang.org/issues/18658
+    runs-on: pub-hk-ubuntu-22.04-xlarge
+    env:
+      STACK: "heroku-24"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@ec02537da5712d66d4d50a0f33b7eb52773b5ed1
+        with:
+          ruby-version: '3.2'
+      - name: Install dependencies
+        run: bundle install
+      - name: Output CHANGELOG
+        run: bundle exec rake "changelog[${{inputs.ruby_version}}]"
+      - name: Build Docker image
+        run: bundle exec rake "generate_image[$STACK]"
+      - name: Generate Ruby Dockerfile
+        run: bundle exec rake "new[${{inputs.ruby_version}},$STACK]"
+      - name: Build and package Ruby runtime
+        run: bash "rubies/$STACK/ruby-${{inputs.ruby_version}}.sh"
+      - name: Build and package Ruby runtime amd64
+        run: |
+          export DOCKER_DEFAULT_PLATFORM=linux/amd64
+          bash "rubies/$STACK/ruby-${{inputs.ruby_version}}.sh"
+      - name: Build and package Ruby runtime arm64
+        run: |
+          export DOCKER_DEFAULT_PLATFORM=linux/arm64
+          bash "rubies/$STACK/ruby-${{inputs.ruby_version}}.sh"
+      - name: Upload Ruby runtime archive to S3 dry run
+        if: (inputs.dry_run)
+        run: aws s3 sync ./builds "s3://${S3_BUCKET}" --dryrun
+      - name: Upload Ruby runtime archive to S3 production
+        if: (!inputs.dry_run)
+        run: aws s3 sync ./builds "s3://${S3_BUCKET}"
+      - name: Output Rubygems version
+        run: bundle exec rake "rubygems_version[${{inputs.ruby_version}},$STACK]"
+

.github/workflows/ci.yml

@@ -35,7 +35,7 @@ jobs:
     runs-on: pub-hk-ubuntu-22.04-xlarge
     strategy:
       matrix:
-        stack: ["heroku-20", "heroku-22"]
+        stack: ["heroku-20", "heroku-22", "heroku-24"]
         version: ["3.1.4"]
     steps:
       - name: Checkout

Rakefile

@@ -79,8 +79,24 @@ end
 desc "Build docker image for stack"
 task :generate_image, [:stack] do |t, args|
   require "fileutils"
-  FileUtils.cp("dockerfiles/Dockerfile.#{args[:stack]}", "Dockerfile")
-  system("docker build -t hone/ruby-builder:#{args[:stack]} .")
+  stack = args[:stack]
+  FileUtils.cp("dockerfiles/Dockerfile.#{stack}", "Dockerfile")
+  image = "hone/ruby-builder:#{stack}"
+  arguments = ["-t #{image}"]
+
+  # rubocop:disable Lint/EmptyWhen
+  case stack
+  when "heroku-24"
+    arguments.push("--platform='linux/amd64,linux/arm64'")
+  when "heroku-20", "heroku-22"
+  else
+    raise "Unknown stack: #{stack}"
+  end
+  # rubocop:enable Lint/EmptyWhen
+
+  command = "docker build #{arguments.join(" ")} ."
+  puts "Running: `#{command}`"
+  system(command)
   FileUtils.rm("Dockerfile")
 end
 

build.rb

@@ -7,6 +7,7 @@
 
 run_build_script(
   stack: ENV.fetch("STACK"),
+  architecture: get_architecture,
   ruby_version: ENV.fetch("VERSION"),
   workspace_dir: ARGV[0],
   output_dir: ARGV[1],

dockerfiles/Dockerfile.heroku-24

@@ -0,0 +1,17 @@
+FROM heroku/heroku:24-build
+
+USER root
+
+# setup workspace
+RUN rm -rf /tmp/workspace
+RUN mkdir -p /tmp/workspace
+
+RUN apt-get update -y; apt-get install ruby -y
+RUN curl https://sh.rustup.rs -sSf | sh -s -- -y
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+# output dir is mounted
+
+ADD build.rb /tmp/build.rb
+COPY lib/ /tmp/lib/
+CMD ["ruby", "/tmp/build.rb", "/tmp/workspace", "/tmp/output", "/tmp/cache"]

lib/build_script.rb

@@ -25,14 +25,13 @@ def run!(cmd)
 
 # Build logic in a method
 def run_build_script(
-  io: $stdout,
+  architecture:, io: $stdout,
   workspace_dir: ARGV[0],
   output_dir: ARGV[1],
   cache_dir: ARGV[2],
   stack: ENV.fetch("STACK"),
   ruby_version: ENV.fetch("STACK")
 )
-
   parts = VersionParts.new(ruby_version)
   ruby_version = RubyVersion.new(ruby_version)
 
@@ -74,10 +73,12 @@ def run_build_script(
       dir: ruby_binary_dir.join("bin")
     )
 
-    destination = Pathname(output_dir)
-      .join(stack)
-      .tap(&:mkpath)
-      .join(ruby_version.tar_file_name_output)
+    destination = stack_architecture_tar_file_name(
+      stack: stack,
+      output_dir: output_dir,
+      architecture: architecture,
+      tar_file_name_output: ruby_version.tar_file_name_output
+    )
 
     io.puts "Writing #{destination}"
     tar_dir(
@@ -88,6 +89,20 @@ def run_build_script(
   end
 end
 
+# Returns a Pathname to the destination tar file
+#
+# The directory structure corresponds to the S3 directory structure directly
+def stack_architecture_tar_file_name(stack:, output_dir:, tar_file_name_output:, architecture:)
+  output_stack_dir = Pathname(output_dir).join(stack)
+
+  case stack
+  when "heroku-24"
+    output_stack_dir.join(architecture)
+  else
+    output_stack_dir
+  end.tap(&:mkpath).join(tar_file_name_output)
+end
+
 # Runs a command on the command line and streams the results
 def pipe(command, io: $stdout)
   output = ""
@@ -199,3 +214,16 @@ def fix_binstubs_in_dir(dir:, io: $stdout)
     end
   end
 end
+
+def get_architecture(system_output: `arch`, success: $?.success?)
+  raise "Error running `arch`: #{system_output}" unless success
+
+  case system_output.strip
+  when "x86_64"
+    "amd64"
+  when "aarch64"
+    "arm64"
+  else
+    raise "Unknown architecture: #{system_output}"
+  end
+end

lib/docker_command.rb

@@ -1,5 +1,13 @@
+require "build_script"
+
 module DockerCommand
-  def self.gem_version_from_tar(ruby_version:, stack:)
-    "docker run -v $(pwd)/builds/#{stack}:/tmp/output hone/ruby-builder:#{stack} bash -c \"mkdir /tmp/unzipped && tar xzf /tmp/output/#{ruby_version.tar_file_name_output} -C /tmp/unzipped && echo 'Rubygems version is: ' && /tmp/unzipped/bin/gem -v\""
+  def self.gem_version_from_tar(ruby_version:, stack:, system_output: `docker run --rm hone/ruby-builder:#{stack} arch`, success: $?.success?)
+    ruby_tar_file = stack_architecture_tar_file_name(
+      stack: stack,
+      output_dir: "/tmp/output",
+      architecture: get_architecture(system_output: system_output, success: success),
+      tar_file_name_output: ruby_version.tar_file_name_output
+    )
+    "docker run -v $(pwd)/builds:/tmp/output hone/ruby-builder:#{stack} bash -c \"mkdir /tmp/unzipped && tar xzf #{ruby_tar_file} -C /tmp/unzipped && echo 'Rubygems version is: ' && /tmp/unzipped/bin/gem -v\""
   end
 end

rubies/heroku-24/ruby-3.2.3.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+source `dirname $0`/../common.sh
+
+docker run -v $OUTPUT_DIR:/tmp/output -v $CACHE_DIR:/tmp/cache -e VERSION=3.2.3  -e STACK=heroku-24 hone/ruby-builder:heroku-24

spec/unit/build_script_spec.rb

@@ -68,4 +68,63 @@
       expect(unzip_dir.entries.map(&:to_s)).to include("bar.txt")
     end
   end
+
+  it "gets system architecture" do
+    expect(get_architecture(system_output: "x86_64", success: true)).to eq("amd64")
+    expect(get_architecture(system_output: "aarch64", success: true)).to eq("arm64")
+
+    expect {
+      get_architecture(
+        system_output: "No such command",
+        success: false
+      )
+    }.to raise_error("Error running `arch`: No such command")
+
+    expect {
+      get_architecture(
+        system_output: "lol",
+        success: true
+      )
+    }.to raise_error("Unknown architecture: lol")
+  end
+
+  it "is stack and architecture aware when writing tar files for heroku-24" do
+    actual = stack_architecture_tar_file_name(
+      stack: "heroku-24",
+      output_dir: "/tmp/output",
+      architecture: "amd64",
+      tar_file_name_output: "ruby-3.1.2.tgz"
+    )
+
+    expect(actual).to eq(Pathname("/tmp").join("output").join("heroku-24").join("amd64").join("ruby-3.1.2.tgz"))
+
+    actual = stack_architecture_tar_file_name(
+      stack: "heroku-24",
+      output_dir: "/tmp/output",
+      architecture: "arm64",
+      tar_file_name_output: "ruby-3.1.2.tgz"
+    )
+
+    expect(actual).to eq(Pathname("/tmp").join("output").join("heroku-24").join("arm64").join("ruby-3.1.2.tgz"))
+  end
+
+  it "is architecture agnostic when writing tar files for heroku-22 and heroku-20" do
+    actual = stack_architecture_tar_file_name(
+      stack: "heroku-20",
+      output_dir: "/tmp/output",
+      architecture: "amd64",
+      tar_file_name_output: "ruby-3.1.2.tgz"
+    )
+
+    expect(actual).to eq(Pathname("/tmp").join("output").join("heroku-20").join("ruby-3.1.2.tgz"))
+
+    actual = stack_architecture_tar_file_name(
+      stack: "heroku-22",
+      output_dir: "/tmp/output",
+      architecture: "arm64",
+      tar_file_name_output: "ruby-3.1.2.tgz"
+    )
+
+    expect(actual).to eq(Pathname("/tmp").join("output").join("heroku-22").join("ruby-3.1.2.tgz"))
+  end
 end

spec/unit/docker_command_spec.rb

@@ -3,14 +3,46 @@
 
 describe DockerCommand do
   it "Generates docker command for outputting rubygems versions" do
-    actual = DockerCommand.gem_version_from_tar(ruby_version: RubyVersion.new("3.1.4"), stack: "heroku-22")
-    expected = %{docker run -v $(pwd)/builds/heroku-22:/tmp/output hone/ruby-builder:heroku-22 bash -c "mkdir /tmp/unzipped && tar xzf /tmp/output/ruby-3.1.4.tgz -C /tmp/unzipped && echo 'Rubygems version is: ' && /tmp/unzipped/bin/gem -v"}
+    actual = DockerCommand.gem_version_from_tar(
+      ruby_version: RubyVersion.new("3.1.4"),
+      stack: "heroku-22",
+      system_output: "x86_64",
+      success: true
+    )
+    expected = %{docker run -v $(pwd)/builds:/tmp/output hone/ruby-builder:heroku-22 bash -c "mkdir /tmp/unzipped && tar xzf /tmp/output/heroku-22/ruby-3.1.4.tgz -C /tmp/unzipped && echo 'Rubygems version is: ' && /tmp/unzipped/bin/gem -v"}
+    expect(actual).to eq(expected)
+  end
+
+  it "Works with amd and arm" do
+    actual = DockerCommand.gem_version_from_tar(
+      ruby_version: RubyVersion.new("3.1.4"),
+      stack: "heroku-24",
+      system_output: "x86_64",
+      success: true
+    )
+
+    expected = %{docker run -v $(pwd)/builds:/tmp/output hone/ruby-builder:heroku-24 bash -c "mkdir /tmp/unzipped && tar xzf /tmp/output/heroku-24/amd64/ruby-3.1.4.tgz -C /tmp/unzipped && echo 'Rubygems version is: ' && /tmp/unzipped/bin/gem -v"}
+    expect(actual).to eq(expected)
+
+    actual = DockerCommand.gem_version_from_tar(
+      ruby_version: RubyVersion.new("3.1.4"),
+      stack: "heroku-24",
+      system_output: "aarch64",
+      success: true
+    )
+
+    expected = %{docker run -v $(pwd)/builds:/tmp/output hone/ruby-builder:heroku-24 bash -c "mkdir /tmp/unzipped && tar xzf /tmp/output/heroku-24/arm64/ruby-3.1.4.tgz -C /tmp/unzipped && echo 'Rubygems version is: ' && /tmp/unzipped/bin/gem -v"}
     expect(actual).to eq(expected)
   end
 
   it "works with preview releases" do
-    actual = DockerCommand.gem_version_from_tar(ruby_version: RubyVersion.new("3.3.0-preview2"), stack: "heroku-22")
-    expected = %{docker run -v $(pwd)/builds/heroku-22:/tmp/output hone/ruby-builder:heroku-22 bash -c "mkdir /tmp/unzipped && tar xzf /tmp/output/ruby-3.3.0.preview2.tgz -C /tmp/unzipped && echo 'Rubygems version is: ' && /tmp/unzipped/bin/gem -v"}
+    actual = DockerCommand.gem_version_from_tar(
+      ruby_version: RubyVersion.new("3.3.0-preview2"),
+      stack: "heroku-22",
+      system_output: "x86_64",
+      success: true
+    )
+    expected = %{docker run -v $(pwd)/builds:/tmp/output hone/ruby-builder:heroku-22 bash -c "mkdir /tmp/unzipped && tar xzf /tmp/output/heroku-22/ruby-3.3.0.preview2.tgz -C /tmp/unzipped && echo 'Rubygems version is: ' && /tmp/unzipped/bin/gem -v"}
     expect(actual).to eq(expected)
   end
 end