diff --git a/kokoro/release/ruby/linux/ruby/ruby_build.sh b/kokoro/release/ruby/linux/ruby/ruby_build.sh
index 95f1dea906b..f8ae9623f25 100755
--- a/kokoro/release/ruby/linux/ruby/ruby_build.sh
+++ b/kokoro/release/ruby/linux/ruby/ruby_build.sh
@@ -12,7 +12,7 @@ fi
 umask 0022
 pushd ruby
 gem install bundler -v 2.1.4
-bundle install && bundle exec rake gem:native
+bundle update && bundle exec rake gem:native
 ls pkg
 mv pkg/* $ARTIFACT_DIR
 popd
diff --git a/kokoro/release/ruby/macos/ruby/ruby_build.sh b/kokoro/release/ruby/macos/ruby/ruby_build.sh
index 9fc42b13eb4..7f6c18fe8e8 100755
--- a/kokoro/release/ruby/macos/ruby/ruby_build.sh
+++ b/kokoro/release/ruby/macos/ruby/ruby_build.sh
@@ -11,7 +11,7 @@ fi
 
 umask 0022
 pushd ruby
-bundle install && bundle exec rake gem:native
+bundle update && bundle exec rake gem:native
 ls pkg
 mv pkg/* $ARTIFACT_DIR
 popd
diff --git a/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh b/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh
index 7ff1ce56966..880c23390cf 100755
--- a/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh
+++ b/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh
@@ -63,7 +63,7 @@ set +x
 rvm use 2.5.0
 set -x
 ruby --version | grep 'ruby 2.5.0'
-for v in 2.6.0 2.5.1 ; do
+for v in 2.6.0 2.5.1 2.4.0 2.3.0; do
   ccache -c
   rake -f "$CROSS_RUBY" cross-ruby VERSION="$v" HOST=x86_64-darwin11 MAKE="$MAKE"
 done
diff --git a/ruby/Rakefile b/ruby/Rakefile
index 53241fd515b..2aa7743e20d 100644
--- a/ruby/Rakefile
+++ b/ruby/Rakefile
@@ -73,7 +73,7 @@ else
     ['x86-mingw32', 'x64-mingw32', 'x86_64-linux', 'x86-linux'].each do |plat|
       RakeCompilerDock.sh <<-"EOT", platform: plat
         bundle && \
-        IN_DOCKER=true rake native:#{plat} pkg/#{spec.full_name}-#{plat}.gem RUBY_CC_VERSION=2.7.0:2.6.0:2.5.0
+        IN_DOCKER=true rake native:#{plat} pkg/#{spec.full_name}-#{plat}.gem RUBY_CC_VERSION=2.7.0:2.6.0:2.5.0:2.4.0:2.3.0
       EOT
     end
   end
@@ -81,7 +81,7 @@ else
   if RUBY_PLATFORM =~ /darwin/
     task 'gem:native' do
       system "rake genproto"
-      system "rake cross native gem RUBY_CC_VERSION=2.7.0:2.6.0:2.5.1"
+      system "rake cross native gem RUBY_CC_VERSION=2.7.0:2.6.0:2.5.1:2.4.0:2.3.0"
     end
   else
     task 'gem:native' => [:genproto, 'gem:windows']