Use docker registry (#580)

* Implement Dockerfile and restructure * .github: update workflows to use docker registry * implement utilities to use for local development * add output folder This fixes permission issues when docker makes the container and creating any file with a runner will fail. * small fixes
ChimeraOS · Apr 14, 2023 · 381026f · 381026f
1 parent dc92422
commit 381026f
Show file tree

Hide file tree

Showing 18 changed files with 254 additions and 190 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -0,0 +1,36 @@
+name: Build docker container
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+on:
+  workflow_dispatch:
+  workflow_call:
+  schedule:
+    - cron: '0 1 * * *'
+
+jobs:
+  build:
+    name: Build base docker image
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
diff --git a/.github/workflows/builder.yml b/.github/workflows/builder.yml
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -1,13 +1,45 @@
 name: System image build
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
 on:
   push:
     branches:
-    - 'master'
+      - master
   pull_request:
     branches:
-    - 'master'
+      - 'master'
+
 jobs:
+  changes:
+    runs-on: ubuntu-latest
+    name: Changes check
+    # Required permissions
+    permissions:
+      pull-requests: read
+    outputs:
+      require-rebuild: ${{ steps.filter.outputs.require-rebuild }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            require-rebuild:
+              - 'Dockerfile'
+              - 'manifest'
+              - 'pkgs/**'
+
+  build-docker-image:
+    name: Build and publish docker container
+    needs: changes
+    if: ${{ needs.changes.outputs.require-rebuild == 'true' }}
+    uses: ./.github/workflows/build.yml
+
   list-pkgbuilds:
+    name: List AUR packages
     runs-on: ubuntu-latest
     outputs:
       aur-pkgs: ${{ steps.set-aur-pkgs.outputs.matrix }}
@@ -17,36 +49,11 @@ jobs:
         run: source ./manifest ; echo "matrix=$(echo ${AUR_PACKAGES} | jq -R -s -c 'split(" ")[:-1]')" >> $GITHUB_OUTPUT
         shell: bash
 
-  build-environment:
-    name: Build environment
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Cache builder image
-        id: builder-cache
-        uses: actions/cache@v3
-        with:
-          key: ${{ hashFiles('pkgs/**/PKGBUILD')}}
-          path: /tmp/builder.tar
-      - if: ${{ steps.builder-cache.outputs.cache-hit != 'true' }}
-        uses: docker/setup-buildx-action@v2
-      - if: ${{ steps.builder-cache.outputs.cache-hit != 'true' }}
-        name: build builder image
-        uses: docker/build-push-action@v3
-        with:
-          context: .
-          tags: image-builder:latest
-          outputs: type=docker,dest=/tmp/builder.tar
-      - if: ${{ steps.builder-cache.outputs.cache-hit != 'true' }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: image-builder
-          path: /tmp/builder.tar
-
   aur-pkgbuild:
     needs:
+     - build-docker-image
      - list-pkgbuilds
-     - build-environment
+    if: ${{ !cancelled() && (success() || failure() || needs.build-docker-image.result == 'skipped') }}
     name: Build AUR package
     runs-on: ubuntu-latest
     strategy:
@@ -55,34 +62,27 @@ jobs:
         package: ${{ fromJson(needs.list-pkgbuilds.outputs.aur-pkgs) }}
     steps:
       - uses: actions/checkout@v3
-      - name: Use cached builder image
-        id: builder-cache
-        uses: actions/cache@v3
-        with:
-          key: ${{ hashFiles('pkgs/**/PKGBUILD')}}
-          path: /tmp/builder.tar
-      - if: ${{ steps.builder-cache.outputs.cache-hit != 'true' }} 
-        name: Download builder
-        uses: actions/download-artifact@v3
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v4
         with:
-          name: image-builder
-          path: /tmp
-      - name: Build package
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+      - name: Build packages
         run: |
-          docker load --input /tmp/builder.tar
-          docker run --rm -v $(pwd):/workdir -v $(pwd)/output:/output --entrypoint=/workdir/build-package.sh --privileged=true image-builder:latest ${{ matrix.package }}
-      - run: ls -ahl output/*
+          docker pull ${{ steps.meta.outputs.tags }}
+          docker run --rm -v $(pwd):/workdir --entrypoint=/workdir/aur-pkgs/build-aur-package.sh ${{ steps.meta.outputs.tags }} ${{ matrix.package }}
       - name: Upload Package Archives
         uses: actions/upload-artifact@v3
         with:
-            name: aur-packages
-            path: output/*.pkg.tar*
+            name: AUR-packages
+            path: aur-pkgs/*.pkg.tar*
 
   build:
-    needs: 
-     - build-environment
+    needs:
+     - build-docker-image
      - aur-pkgbuild
-    name: System image build
+    if: ${{ !cancelled() && (success() || failure() || needs.build-docker-image.result == 'skipped') }}
+    name: Build ChimeraOS image
     runs-on: ubuntu-latest
     permissions:
       contents: write
@@ -93,27 +93,27 @@ jobs:
       image_filename: ${{ steps.build_image.outputs.image_filename }}
     steps:
       - uses: actions/checkout@v3
-      - name: Use cached builder image
-        id: builder-cache
-        uses: actions/cache@v3
-        with:
-          key: ${{ hashFiles('pkgs/**/PKGBUILD')}}
-          path: /tmp/builder.tar
-      - if: ${{ steps.builder-cache.outputs.cache-hit != 'true' }} 
-        name: Download builder
-        uses: actions/download-artifact@v3
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v4
         with:
-          name: image-builder
-          path: /tmp
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
       - uses: actions/download-artifact@v3
         with:
-            name: aur-packages
+            name: AUR-packages
             path: aur-pkgs/
-      - run: docker load --input /tmp/builder.tar
       - name: Build system image
         id: build_image
-        run: docker run --rm -v $(pwd):/workdir -v $(pwd)/output:/output -v $GITHUB_OUTPUT:$GITHUB_OUTPUT -e "GITHUB_OUTPUT=$GITHUB_OUTPUT" --privileged=true image-builder:latest $(echo ${GITHUB_SHA} | cut -c1-7)
-
+        run: |
+          docker pull ${{ steps.meta.outputs.tags }}
+          docker run -u root --rm --entrypoint=/workdir/build-image.sh -v $(pwd):/workdir -v $(pwd)/output:/output -v $GITHUB_OUTPUT:$GITHUB_OUTPUT -e "GITHUB_OUTPUT=$GITHUB_OUTPUT" --privileged=true ${{ steps.meta.outputs.tags }} $(echo ${GITHUB_SHA} | cut -c1-7)
+          echo -e "$(docker inspect --format='{{index .RepoDigests 0}}' ${{ steps.meta.outputs.tags }})" > output/container.txt
+      - if: github.ref != 'refs/heads/master'
+        name: Create artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: Output
+          path: output/
       - if: github.ref == 'refs/heads/master'
         name: Create release
         id: create_release
@@ -129,3 +129,4 @@ jobs:
             output/${{ steps.build_image.outputs.image_filename }}
             output/build_info.txt
             output/sha256sum.txt
+            output/container.txt
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,6 @@
 *.SRCINFO
 *.img.tar.xz
 output/*
+aur-pkgs/*
+!aur-pkgs/build-aur-packages.sh
+!aur-pkgs/build-aur-package.sh
diff --git a/Dockerfile b/Dockerfile
@@ -1,19 +1,47 @@
 FROM archlinux:base-devel
 LABEL contributor="shadowapex@gmail.com"
+RUN sed -i '/ParallelDownloads/s/^#//g' /etc/pacman.conf && \
+    echo -e "[multilib]\nInclude = /etc/pacman.d/mirrorlist\n" >> /etc/pacman.conf && \
+    echo -e "keyserver-options auto-key-retrieve" >> /etc/pacman.d/gnupg/gpg.conf && \
+    pacman --noconfirm -Syyuu && \
+    pacman --noconfirm -S \
+    arch-install-scripts \
+    btrfs-progs \
+    fmt \
+    xcb-util-wm \
+    wget \
+    pyalpm \
+    python-build \
+    python-installer \
+    python-markdown-it-py \
+    python-setuptools \
+    python-wheel \
+    sudo \
+    reflector \
+    && \
+    pacman --noconfirm -S --needed git && \
+    echo "%wheel ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers && \
+    useradd build -G wheel -m && \
+    su - build -c "git clone https://aur.archlinux.org/pikaur.git /tmp/pikaur" && \
+    su - build -c "cd /tmp/pikaur && makepkg -f" && \
+    pacman --noconfirm -U /tmp/pikaur/pikaur-*.pkg.tar.zst
 
-# Freeze if required
-COPY freeze.sh own-pkg.sh manifest /
-COPY pkgs /pkgs
+# Add a fake systemd-run script to workaround pikaur requirement.
+RUN echo -e "#!/bin/bash\nif [[ \"$1\" == \"--version\" ]]; then echo 'fake 244 version'; fi\nmkdir -p /var/cache/pikaur\n" >> /usr/bin/systemd-run && \
+    chmod +x /usr/bin/systemd-run
 
-RUN chmod +x freeze.sh && ./freeze.sh
-# Build pikaur packages as the 'build' user
-ENV BUILD_USER "build"
+COPY manifest /manifest
+# Freeze packages and overwrite with overrides when needed
+RUN source /manifest; if [ -n "${ARCHIVE_DATE}" ]; then echo "Server=https://archive.archlinux.org/repos/${ARCHIVE_DATE}/\$repo/os/\$arch" > /etc/pacman.d/mirrorlist; else reflector --verbose --latest 10 --protocol https --sort rate --save /etc/pacman.d/mirrorlist; fi && \
+    pacman --noconfirm -Syyuu; if [ -n "${PACKAGE_OVERRIDES}" ]; then wget --directory-prefix=/tmp/extra_pkgs ${PACKAGE_OVERRIDES}; pacman --noconfirm -U --overwrite '*' /tmp/extra_pkgs/*; fi
 
+USER build
+ENV BUILD_USER "build"
 ENV GNUPGHOME  "/etc/pacman.d/gnupg"
-RUN chmod +x own-pkg.sh && ./own-pkg.sh
-
 # Built image will be moved here. This should be a host mount to get the output.
 ENV OUTPUT_DIR /output
 
-WORKDIR /workdir
-ENTRYPOINT ["/workdir/build.sh"]
+COPY pkgs /packages
+RUN /packages/build-packages.sh
+
+WORKDIR /workdir
diff --git a/aur-pkgs/build-aur-package.sh b/aur-pkgs/build-aur-package.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e
+set -x
+
+source manifest;
+
+sudo mkdir -p /workdir/aur-pkgs
+sudo chown build:build /workdir/aur-pkgs
+
+PIKAUR_CMD="PKGDEST=/workdir/aur-pkgs pikaur --noconfirm -Sw ${1}"
+PIKAUR_RUN=(bash -c "${PIKAUR_CMD}")
+"${PIKAUR_RUN[@]}"
+# remove any epoch (:) in name, replace with -- since not allowed in artifacts
+find /workdir/aur-pkgs/*.pkg.tar* -type f -name '*:*' -execdir bash -c 'mv "$1" "${1//:/--}"' bash {} \;
diff --git a/aur-pkgs/build-aur-packages.sh b/aur-pkgs/build-aur-packages.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -e
+set -x
+
+source manifest;
+
+sudo mkdir -p /workdir/aur-pkgs
+sudo chown build:build /workdir/aur-pkgs
+
+PIKAUR_CMD="PKGDEST=/workdir/aur-pkgs pikaur --noconfirm -Sw ${AUR_PACKAGES}"
+PIKAUR_RUN=(bash -c "${PIKAUR_CMD}")
+"${PIKAUR_RUN[@]}"
diff --git a/build.sh → build-image.sh b/build.sh → build-image.sh
@@ -58,7 +58,7 @@ mkdir ${BUILD_PATH}/extra_pkgs
 
 cp -rv aur-pkgs/*.pkg.tar* ${BUILD_PATH}/extra_pkgs
 # Own packages already exist in docker container
-cp -rv /pkgs/**/*.pkg.tar* ${BUILD_PATH}/own_pkgs
+cp -rv /pkgs/*.pkg.tar* ${BUILD_PATH}/own_pkgs
 
 if [ -n "${PACKAGE_OVERRIDES}" ]; then
 	cp -rv /tmp/extra_pkgs/*.pkg.tar* ${BUILD_PATH}/extra_pkgs

diff --git a/build-package.sh b/build-package.sh