diff --git a/Dockerfile b/Dockerfile
index 2b41996..9169bc7 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -24,8 +24,12 @@ RUN apt-get update && apt-get install -y \
 # For an example of php80-rc5 near complete, using pickle instead of pear/pecl, look to:
 # https://github.com/stronk7/moodle-php-apache/tree/8.0-buster-pickle-version
 
-# Setup the required extensions.
+# Generate all the UTF-8 locales.
 ARG DEBIAN_FRONTEND=noninteractive
+ADD root/tmp/setup/locales-gen.sh /tmp/setup/locales-gen.sh
+RUN /tmp/setup/locales-gen.sh
+
+# Setup the required extensions.
 ADD root/tmp/setup/php-extensions.sh /tmp/setup/php-extensions.sh
 RUN /tmp/setup/php-extensions.sh
 
diff --git a/root/tmp/setup/locales-gen.sh b/root/tmp/setup/locales-gen.sh
new file mode 100755
index 0000000..6cebe6a
--- /dev/null
+++ b/root/tmp/setup/locales-gen.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -e
+
+echo "Installing apt dependencies"
+
+# Packages for installing locales.
+RUNTIME_LOCALES="locales"
+
+apt-get update
+apt-get install -y --no-install-recommends apt-transport-https \
+    $RUNTIME_LOCALES
+
+echo "Installing UTF-8 locales"
+
+# Generate the locales configuration for all possible UTF-8 locales.
+grep UTF-8 /usr/share/i18n/SUPPORTED > /etc/locale.gen
+locale-gen
+
+# Keep our image size down..
+apt-get autoremove -y
+apt-get clean
+rm -rf /var/lib/apt/lists/*
diff --git a/root/tmp/setup/php-extensions.sh b/root/tmp/setup/php-extensions.sh
index a79f863..3819c37 100755
--- a/root/tmp/setup/php-extensions.sh
+++ b/root/tmp/setup/php-extensions.sh
@@ -17,7 +17,7 @@ PACKAGES_MYMARIA="libmariadb3"
 
 # Packages for other Moodle runtime dependenices.
 PACKAGES_RUNTIME="ghostscript libaio1 libcurl4 libgss3 libicu67 libmcrypt-dev libxml2 libxslt1.1 \
-  libzip-dev locales sassc unzip zip"
+  libzip-dev sassc unzip zip"
 
 # Packages for Memcached.
 PACKAGES_MEMCACHED="libmemcached11 libmemcachedutil2"
@@ -34,11 +34,6 @@ apt-get install -y --no-install-recommends apt-transport-https \
     $PACKAGES_MEMCACHED \
     $PACKAGES_LDAP
 
-# Generate the locales configuration for all possible UTF-8 locales.
-echo 'Generating locales..'
-grep UTF-8 /usr/share/i18n/SUPPORTED > /etc/locale.gen
-locale-gen
-
 echo "Installing php extensions"
 
 # ZIP