diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index c6506bdd1..e0c330be9 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -112,17 +112,24 @@ jobs:
           load: true
           cache-from: type=gha
           cache-to: type=gha,mode=max
-      - uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: "3.2"
-          bundler-cache: true
-      - run: ./bin/setup
-      - run: bundle exec rake compile
+      - name: Setup builder container
+        run: |
+          docker run -dit -v "$GITHUB_WORKSPACE:/home/me/build" -w /home/me/build \
+            -e "ENABLE_GITHUB_ACTIONS_MARKUP=true" \
+            -e "RUBYWASM_UID=$(id -u)" -e "RUBYWASM_GID=$(id -g)" \
+            -e "GNUMAKEFLAGS=-j$((1 + $(nproc --all)))" \
+            --name builder \
+            ${{ steps.builder-image.outputs.imageid }} /bin/sh
+          echo 'docker exec -u me builder "$@"' > ./build-exec
+          chmod +x ./build-exec
+      - run: ./build-exec bundle config set --local without check
+      - run: ./build-exec ./bin/setup
+      - run: ./build-exec bundle exec rake compile
       - name: Pre-release configuration
-        run: bundle exec rake ${{ matrix.entry.prerelease }}[${{ inputs.prerel_name }}]
+        run: ./build-exec bundle exec rake ${{ matrix.entry.prerelease }}[${{ inputs.prerel_name }}]
         if: ${{ inputs.prerel_name != '' && matrix.entry.prerelease != '' }}
       - name: rake ${{ matrix.entry.task }}
-        run: docker run -v "$GITHUB_WORKSPACE:/home/me/build" -w /home/me/build -e "ENABLE_GITHUB_ACTIONS_MARKUP=true" -e "RUBYWASM_UID=$(id -u)" -e "RUBYWASM_GID=$(id -g)" -e "GNUMAKEFLAGS=-j$((1 + $(nproc --all)))" ${{ steps.builder-image.outputs.imageid }} rake --verbose ${{ matrix.entry.task }}
+        run: ./build-exec rake --verbose ${{ matrix.entry.task }}
       - name: rake ${{ matrix.entry.test }}
         run: rake ${{ matrix.entry.test }}
         if: ${{ matrix.entry.test != '' }}
diff --git a/.gitignore b/.gitignore
index a342a4509..49fde32b6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,3 +14,5 @@ compile_commands.json
 html
 
 build_manifest.json
+
+vendor/bundle
diff --git a/Gemfile b/Gemfile
index 5cb6599bc..a3ef79e8b 100644
--- a/Gemfile
+++ b/Gemfile
@@ -8,7 +8,10 @@ group :development do
   gem "rake"
   gem "rake-compiler"
   gem "rb_sys", "~> 0.9.63"
-  gem "syntax_tree", "~> 3.5"
   gem "webrick"
+end
+
+group :check do
+  gem "syntax_tree", "~> 3.5"
   gem "steep"
 end
diff --git a/bin/setup b/bin/setup
index ffa93c6bc..b98faf9db 100755
--- a/bin/setup
+++ b/bin/setup
@@ -3,5 +3,8 @@ set -euo pipefail
 IFS=$'\n\t'
 set -vx
 
-bundle install
+root="$(cd "$(dirname "$0")/.." && pwd)"
+
+env BUNDLE_GEMFILE="$root/Gemfile" bundle install
+env BUNDLE_GEMFILE="$root/packages/npm-packages/ruby-wasm-wasi/Gemfile" bundle install
 npm ci
diff --git a/builders/wasm32-unknown-wasi/Dockerfile b/builders/wasm32-unknown-wasi/Dockerfile
index 13b06cd1f..a02cd36a7 100644
--- a/builders/wasm32-unknown-wasi/Dockerfile
+++ b/builders/wasm32-unknown-wasi/Dockerfile
@@ -5,13 +5,15 @@ ARG NODE_MAJOR_VERSION=20
 
 RUN set -eux; \
   apt-get update; \
-  apt-get install ruby ruby-bundler bison make autoconf git curl build-essential libyaml-dev zlib1g-dev gosu -y; \
   apt-get install -y ca-certificates curl gnupg; \
   mkdir -p /etc/apt/keyrings; \
   curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg; \
   echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR_VERSION.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list; \
   apt-get update; \
-  apt-get install nodejs -y; \
+  apt-get install ruby ruby-dev ruby-bundler nodejs \
+    bison make autoconf git curl build-essential \
+    libyaml-dev zlib1g-dev gosu \
+    libclang-13-dev -y; \
   apt-get clean; \
   rm -r /var/lib/apt/lists/*
 
@@ -21,6 +23,21 @@ RUN set -eux pipefail; \
   unzip wasi-preset-args-x86_64-unknown-linux-gnu.zip; \
   mv wasi-preset-args /usr/local/bin/wasi-preset-args
 
+ENV RUSTUP_HOME=/usr/local/rustup \
+    CARGO_HOME=/usr/local/cargo \
+    PATH=/usr/local/cargo/bin:$PATH \
+    RUST_VERSION=1.74
+
+RUN set -eux pipefail; \
+  curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
+    sh -s -- -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION; \
+  chmod -R a+w $RUSTUP_HOME $CARGO_HOME
+
+ENV BUNDLE_PATH=/usr/local/gems
+RUN set -eux; \
+  mkdir -p $BUNDLE_PATH; \
+  chmod -R 777 $BUNDLE_PATH
+
 COPY entrypoint.sh /usr/local/bin/entrypoint.sh
 RUN chmod +x /usr/local/bin/entrypoint.sh
 ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
diff --git a/lib/ruby_wasm/packager/core.rb b/lib/ruby_wasm/packager/core.rb
index 9c77f6410..83dfdc9d8 100644
--- a/lib/ruby_wasm/packager/core.rb
+++ b/lib/ruby_wasm/packager/core.rb
@@ -89,7 +89,7 @@ def derive_build
       return @build if @build
       __skip__ =
         build ||= RubyWasm::Build.new(name, **@packager.full_build_options)
-      build.crossruby.user_exts = user_exts
+      build.crossruby.user_exts = user_exts(build)
       build.crossruby.debugflags = %w[-g]
       build.crossruby.wasmoptflags = %w[-O3 -g]
       build.crossruby.ldflags = %w[
@@ -104,7 +104,7 @@ def derive_build
       build
     end
 
-    def user_exts
+    def user_exts(build)
       @user_exts ||=
         specs_with_extensions.flat_map do |spec, exts|
           exts.map do |ext|
@@ -113,7 +113,7 @@ def user_exts
             ext_relative_path = File.join(spec.full_name, ext_feature)
             RubyWasm::CrossRubyExtProduct.new(
               ext_srcdir,
-              @build.toolchain,
+              build.toolchain,
               ext_relative_path: ext_relative_path
             )
           end
diff --git a/packages/npm-packages/ruby-wasm-wasi/tools/pack-ruby-wasm.sh b/packages/npm-packages/ruby-wasm-wasi/tools/pack-ruby-wasm.sh
index 342f3f0a3..66f3d0ba2 100755
--- a/packages/npm-packages/ruby-wasm-wasi/tools/pack-ruby-wasm.sh
+++ b/packages/npm-packages/ruby-wasm-wasi/tools/pack-ruby-wasm.sh
@@ -18,9 +18,12 @@ package_dir="$(cd "$(dirname "$0")/.." && pwd)"
 mkdir -p "$dist_dir"
 
 # Cache rubies in the package dir
-export RUBY_WASM_ROOT="$package_dir/../../../"
+export RUBY_WASM_ROOT="$package_dir/../../.."
+export BUNDLE_GEMFILE="$package_dir/Gemfile"
 cd "$package_dir"
 
+echo "$0: Entering $package_dir"
+
 rbwasm_options="--ruby-version $ruby_version --target wasm32-unknown-wasi --build-profile full"
 bundle exec rbwasm build ${rbwasm_options[@]} --no-stdlib -o "$dist_dir/ruby.wasm"
 "$WASMOPT" --strip-debug "$dist_dir/ruby.wasm" -o "$dist_dir/ruby.wasm"
diff --git a/sig/ruby_wasm/packager.rbs b/sig/ruby_wasm/packager.rbs
index f0220c78b..999d147f3 100644
--- a/sig/ruby_wasm/packager.rbs
+++ b/sig/ruby_wasm/packager.rbs
@@ -55,7 +55,7 @@ class RubyWasm::Packager
       @build: RubyWasm::Build
       def derive_build: () -> RubyWasm::Build
       @user_exts: Array[RubyWasm::CrossRubyExtProduct]?
-      def user_exts: () -> Array[RubyWasm::CrossRubyExtProduct]
+      def user_exts: (RubyWasm::Build) -> Array[RubyWasm::CrossRubyExtProduct]
 
       def name: () -> string
     end
@@ -88,4 +88,4 @@ class RubyWasm::Packager
     def ruby_version: () -> String
     def rubyarchdir: () -> String
   end
-end
\ No newline at end of file
+end