diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl
index 9d661201..2bf2ce1a 100644
--- a/Dockerfile.tmpl
+++ b/Dockerfile.tmpl
@@ -44,6 +44,7 @@ RUN uv pip install --system --no-cache "pycuda"
 
 # Use a fixed apt-get repo to stop intermittent failures due to flaky httpredir connections,
 # as described by Lionel Chan at http://stackoverflow.com/a/37426929/5881346
+RUN /tmp/clean-layer.sh
 RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list && \
     apt-get update --allow-releaseinfo-change && \
     # Needed by lightGBM (GPU build)
diff --git a/clean-layer.sh b/clean-layer.sh
index 9a50e7bf..3a69c68b 100755
--- a/clean-layer.sh
+++ b/clean-layer.sh
@@ -17,4 +17,12 @@ apt-get clean
 # Ensures the current working directory won't be deleted
 cd /usr/local/src/
 # Delete source files used for building binaries
-rm -rf /usr/local/src/*
\ No newline at end of file
+rm -rf /usr/local/src/*
+
+# More clean up to save disk space
+rm -rf /var/lib/apt/lists/*
+find /tmp -mindepth 1 ! -name 'clean-layer.sh' -delete 2>/dev/null || true
+rm -rf /root/.cache
+rm -rf /var/cache/apt/archives/*
+find /usr -name '*.pyc' -delete 2>/dev/null || true
+find /usr -name '__pycache__' -type d -exec rm -rf {} + 2>/dev/null || true
\ No newline at end of file