diff --git a/ci/get-ocp-repo.sh b/ci/get-ocp-repo.sh
index dc795daef..4fb06fa75 100755
--- a/ci/get-ocp-repo.sh
+++ b/ci/get-ocp-repo.sh
@@ -73,7 +73,7 @@ cosa_workdir=
 ocp_manifest=
 output_dir=
 rc=0
-options=$(getopt --options h --longoptions help,cosa-workdir:,ocp-layer:,output-dir:,cleanup,create-gpg-keys -- "$@") || rc=$?
+options=$(getopt --options h --longoptions help,cosa-workdir:,ocp-layer:,output-dir:,cleanup -- "$@") || rc=$?
 [ $rc -eq 0 ] || print_usage_and_exit
 eval set -- "$options"
 while [ $# -ne 0 ]; do
@@ -83,7 +83,6 @@ while [ $# -ne 0 ]; do
         --ocp-layer) ocp_manifest=$2; shift;;
         --output-dir) output_dir=$2; shift;;
         --cleanup) cleanup_repos; exit 0;;
-        --create-gpg-keys) create_gpg_keys; exit 0;;
         --) break;;
         *) echo "$0: invalid argument: $1" >&2; exit 1;;
     esac
diff --git a/ci/prow-entrypoint.sh b/ci/prow-entrypoint.sh
index 294d01372..5ed7c2148 100755
--- a/ci/prow-entrypoint.sh
+++ b/ci/prow-entrypoint.sh
@@ -74,14 +74,16 @@ cosa_build() {
     cosa fetch
     # Only build the ostree image by default
     cosa build ostree
-    # Build extensions container
+}
+
+cosa_build_extensions() {
     cosa buildextend-extensions-container
 }
 
 # Build QEMU image and run all kola tests
 kola_test_qemu() {
     cosa buildextend-qemu
-    cosa kola run --parallel 2 --output-dir ${ARTIFACT_DIR:-/tmp}/kola --rerun --allow-rerun-success tags=needs-internet
+    cosa kola run --parallel 2 --output-dir ${ARTIFACT_DIR:-/tmp}/kola --rerun --allow-rerun-success tags=needs-internet "$@"
 }
 
 # Build metal, metal4k & live images and run kola tests
@@ -270,21 +272,19 @@ main() {
             cosa_init "$2"
             prepare_repos
             ;;
-        "build" | "init-and-build-default")  # TODO: change prow job to use init-and-build-default
-            cosa_init "ocp-rhel-9.6"
-            cosa_build
-            ;;
         # this is called by cosa's CI
         "rhcos-cosa-prow-pr-ci")
             setup_user
             cosa_init "ocp-rhel-9.6"
             cosa_build
+            cosa_build_extensions
             kola_test_qemu
             ;;
         "rhcos-9-build-test-qemu")
             setup_user
             cosa_init "ocp-rhel-9.6"
             cosa_build
+            cosa_build_extensions
             kola_test_qemu
             ;;
         "rhcos-9-build-test-metal")
@@ -301,13 +301,13 @@ main() {
             ;;
         "scos-9-build-test-qemu")
             setup_user
-            cosa_init "okd-c9s"
+            cosa_init "c9s"
             cosa_build
-            kola_test_qemu
+            kola_test_qemu --tag '!openshift'
             ;;
         "scos-9-build-test-metal")
             setup_user
-            cosa_init "okd-c9s"
+            cosa_init "c9s"
             cosa_build
             kola_test_metal
             ;;
diff --git a/extensions/Dockerfile b/extensions/Dockerfile
index 6d3168deb..b78273bca 100644
--- a/extensions/Dockerfile
+++ b/extensions/Dockerfile
@@ -6,10 +6,10 @@ FROM registry.ci.openshift.org/rhcos-devel/rhel-coreos:latest as os
 RUN mkdir /os
 WORKDIR /os
 ADD . .
-ARG COSA
-ARG VARIANT
-RUN if [[ -z "$COSA" ]] ; then ci/get-ocp-repo.sh --ocp-layer packages-openshift.yaml; else ci/get-ocp-repo.sh --create-gpg-keys; fi
-RUN if [[ -n "${VARIANT}" ]]; then MANIFEST="manifest-${VARIANT}.yaml"; EXTENSIONS="extensions-${VARIANT}.yaml"; else MANIFEST="manifest.yaml"; EXTENSIONS="extensions.yaml"; fi && rpm-ostree compose extensions --rootfs=/ --output-dir=/usr/share/rpm-ostree/extensions/ ./"${MANIFEST}" ./"${EXTENSIONS}"
+ARG OPENSHIFT_CI=0
+ARG VARIANT=""
+RUN if [ "${OPENSHIFT_CI}" != 0 ]; then ci/get-ocp-repo.sh --ocp-layer packages-openshift.yaml; fi
+RUN --mount=type=secret,id=yumrepos,target=/os/secret.repo if [[ -n "${VARIANT}" ]]; then MANIFEST="manifest-${VARIANT}.yaml"; EXTENSIONS="extensions-${VARIANT}.yaml"; else MANIFEST="manifest.yaml"; EXTENSIONS="extensions.yaml"; fi && rpm-ostree compose extensions --rootfs=/ --output-dir=/usr/share/rpm-ostree/extensions/ ./"${MANIFEST}" ./"${EXTENSIONS}"
 
 ## Creates the repo metadata for the extensions.
 ## This uses Fedora as a lowest-common-denominator because it will work on
@@ -21,16 +21,18 @@ RUN rm -f /etc/yum.repos.d/*.repo \
 RUN dnf install -y createrepo_c
 RUN createrepo_c /usr/share/rpm-ostree/extensions/
 
-# Generate extensions.json for meta.json, written to a bind-mounted path during the build.
+# Generate extensions.json for meta.json.
 # Use dnf repoquery to print 'name: version,' for each RPM
 # sed to remove the comma from the last RPM
-RUN sh -c 'echo "{" > /tmp/extensions.json && \
-dnf repoquery --repofrompath=extensions,/usr/share/rpm-ostree/extensions/ \
+RUN (echo "{" && \
+(dnf repoquery --repofrompath=extensions,/usr/share/rpm-ostree/extensions/ \
   --quiet --disablerepo=* --enablerepo=extensions \
   --queryformat "\"%{name}\": \"%{evr}.%{arch}\"," | \
-sed "$ s/,$//" >> /tmp/extensions.json && \
-echo "}" >> /tmp/extensions.json'
+sed "$ s/,$//") && echo "}") >> /usr/share/rpm-ostree/extensions.json
 
 ## Final container that has the extensions repo dir
 FROM registry.access.redhat.com/ubi9/ubi:latest
 COPY --from=builder /usr/share/rpm-ostree/extensions/ /usr/share/rpm-ostree/extensions/
+# Make this the last layer, this is similar to the metalayer trick in the node
+# image, but this one is specific to rpm-ostree extensions.
+COPY --from=builder /usr/share/rpm-ostree/extensions.json /usr/share/rpm-ostree/extensions.json
diff --git a/scripts/generate-metadata b/scripts/generate-metadata
index 9aa26de91..bf463e2d3 100755
--- a/scripts/generate-metadata
+++ b/scripts/generate-metadata
@@ -42,6 +42,10 @@ def get_rpmdb_pkglist():
     rpmdb = []
     for line in out.splitlines():
         n, e, v, r, a = line.split()
+        if n == 'gpg-pubkey':
+            # those aren't real packages, it's just how rpm represents imported
+            # GPG keys
+            continue
         # canonicalize none to 0 to match rpm-ostree semantics
         if e == '(none)':
             e = '0'