[wip] github-actions: use cache-image in devshell workflow

.github/workflows/dbld-images.yml

@@ -0,0 +1,67 @@
+name: Compile dbld-images
+
+on:
+  pull_request:
+  push:
+    paths:
+      - .github/workflows/dbld-images.yml
+      # List is from dbld/rules cache-image-% "WATCHED_FILES"
+      - "dbld/**"
+      - "packaging/rhel/syslog-ng.spec"
+      - "packaging/debian/control"
+  schedule:
+    - cron: '00 03 * * *'
+  workflow_dispatch:
+    inputs:
+      upload_images:
+        description: Should we upload the images into GitHub Packages? (true/false)
+        required: false
+        default: "false"
+
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+    env:
+      CONTAINER_REGISTRY: "ghcr.io/${{ github.repository_owner }}"
+
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v2
+
+      - name: Build the images
+        run: dbld/rules images -j $(nproc)
+
+      - name: Should we upload the images?
+        run: |
+          if [ "${{ github.event.inputs.upload_images }}" = "true" ] || \
+             ( \
+              [ "${{ github.repository_owner }}" = "syslog-ng" ] && \
+              [ "${{ github.ref }}" = "refs/heads/master" ] && \
+              [ "${{ github.event_name }}" = "push" ] \
+            )
+          then
+            UPLOAD_IMAGES_INTERNAL="true"
+          else
+            UPLOAD_IMAGES_INTERNAL="false"
+          fi
+
+          . .github/workflows/gh-tools.sh
+          gh_export UPLOAD_IMAGES_INTERNAL
+
+      - name: Log in to the Container registry
+        if: env.UPLOAD_IMAGES_INTERNAL == 'true'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Push the images
+        if: env.UPLOAD_IMAGES_INTERNAL == 'true'
+        run: |
+          IMAGES=$(docker image ls --filter "reference=${{ env.CONTAINER_REGISTRY }}/dbld-*:latest" --format '{{.Repository}}:{{.Tag}}')
+          for IMAGE in $IMAGES; do
+            echo "Pushing image: $IMAGE"
+            docker push $IMAGE
+          done

.github/workflows/devshell.yml

@@ -1,16 +1,77 @@
 name: CI @ devshell
 
 on:
-  pull_request:
+  pull_request_target:
   push:
   schedule:
     - cron: '00 21 * * *'
 
 jobs:
+  cache-image-devshell:
+    runs-on: ubuntu-latest
+
+    outputs:
+      docker-image: ${{ steps.determine-actual-image.outputs.docker-image }}
+
+    env:
+      CACHED_IMAGE_NAME: ghcr.io/syslog-ng/dbld-devshell:latest
+
+    steps:
+      - name: Checkout syslog-ng source
+        uses: actions/checkout@v2
+
+      - name: Determine cached image ID
+        run: |
+          ./dbld/rules pull-image-devshell || true # remove the || true
+          CACHED_IMAGE_ID=$(docker images -q ${CACHED_IMAGE_NAME})
+
+          . .github/workflows/gh-tools.sh
+          gh_export CACHED_IMAGE_ID
+
+      - name: cache-image-devshell
+        run: |
+          ./dbld/rules cache-image-devshell
+
+      - name: Determine actual image
+        id: determine-actual-image
+        run: |
+          CURRENT_IMAGE_ID=$(docker images -q ${CACHED_IMAGE_NAME})
+          if [[ "${CURRENT_IMAGE_ID}" == "${CACHED_IMAGE_ID}" ]]
+          then
+            DOCKER_IMAGE=${CACHED_IMAGE_NAME}
+            USE_CACHED_IMAGE=true
+          else
+            DOCKER_IMAGE=ghcr.io/${{ github.actor }}/dbld-devshell:ci-${{ github.run_id }}
+            USE_CACHED_IMAGE=false
+            docker tag ${CACHED_IMAGE_NAME} ${DOCKER_IMAGE}
+          fi
+
+          echo "Using image: ${DOCKER_IMAGE}"
+
+          . .github/workflows/gh-tools.sh
+          gh_export USE_CACHED_IMAGE
+          echo "::set-output name=docker-image::${DOCKER_IMAGE}"
+
+      - name: Log in to the Container registry
+        if: env.USE_CACHED_IMAGE == 'false'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Upload the temporary docker image
+        if: env.USE_CACHED_IMAGE == 'false'
+        run: |
+          docker push ${{ steps.determine-actual-image.outputs.docker-image }}
+
   general:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
+
+    needs: cache-image-devshell
+
     container:
-      image: balabit/syslog-ng-devshell:latest
+      image: ${{ needs.cache-image-devshell.outputs.docker-image }}
       options: --privileged --ulimit core=-1
 
     strategy:
@@ -144,9 +205,12 @@ jobs:
           path: ${{ env.COREFILES_DIR }}
 
   distcheck:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
+
+    needs: cache-image-devshell
+
     container:
-      image: balabit/syslog-ng-devshell:latest
+      image: ${{ needs.cache-image-devshell.outputs.docker-image }}
 
     steps:
       - name: Checkout syslog-ng source
@@ -184,9 +248,12 @@ jobs:
           exec_prop_check "make distcheck -j 3 V=1"
 
   style-check:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
+
+    needs: cache-image-devshell
+
     container:
-      image: balabit/syslog-ng-devshell:latest
+      image: ${{ needs.cache-image-devshell.outputs.docker-image }}
 
     steps:
       - name: Checkout syslog-ng source
@@ -235,9 +302,12 @@ jobs:
           path: light-style-problems.diff
 
   copyright-check:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
+
+    needs: cache-image-devshell
+
     container:
-      image: balabit/syslog-ng-devshell:latest
+      image: ${{ needs.cache-image-devshell.outputs.docker-image }}
       env:
         COPYRIGHTVERBOSITY: 1
 
@@ -256,9 +326,12 @@ jobs:
           path: copyright-run.log
 
   commits-check:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
+
+    needs: cache-image-devshell
+
     container:
-      image: balabit/syslog-ng-devshell:latest
+      image: ${{ needs.cache-image-devshell.outputs.docker-image }}
 
     steps:
       - name: Checkout syslog-ng source

dbld/README.md

@@ -17,7 +17,7 @@ command (without parameters it will run without any side effect), or read
 the source code on [GitHub](rules).
 
 Almost every `dbld/rules` command runs in a Docker container.  You can use
-the pre-built containers from [DockerHub](https://hub.docker.com/u/balabit/)
+the pre-built containers from [GitHub](https://github.com/syslog-ng?tab=packages&repo_name=syslog-ng)
 or build your own images with the `dbld/rules image-<os>` command.
 
 The source code and build products are mounted externally in the following locations:

dbld/images/devshell.dockerfile

@@ -1,4 +1,5 @@
-FROM balabit/syslog-ng-tarball:latest
+ARG CONTAINER_REGISTRY
+FROM $CONTAINER_REGISTRY/dbld-tarball:latest
 
 ARG ARG_IMAGE_PLATFORM
 ARG COMMIT

dbld/images/kira.dockerfile

@@ -1,4 +1,5 @@
-FROM balabit/syslog-ng-ubuntu-focal
+ARG CONTAINER_REGISTRY
+FROM $CONTAINER_REGISTRY/dbld-ubuntu-focal
 LABEL maintainer="Andras Mitzki <andras.mitzki@balabit.com>, Laszlo Szemere <laszlo.szemere@balabit.com>, Balazs Scheidler <balazs.scheidler@oneidentity.com>"
 
 ARG ARG_IMAGE_PLATFORM

dbld/images/tarball.dockerfile

@@ -1,4 +1,5 @@
-FROM balabit/syslog-ng-debian-testing:latest
+ARG CONTAINER_REGISTRY
+FROM $CONTAINER_REGISTRY/dbld-debian-testing:latest
 
 ARG ARG_IMAGE_PLATFORM
 ARG COMMIT

dbld/rules

@@ -19,6 +19,7 @@ TARBALL_IMAGE ?= tarball
 DEFAULT_DEB_IMAGE=ubuntu-bionic
 DEFAULT_RPM_IMAGE=centos-7
 DOCKER=docker
+CONTAINER_REGISTRY ?= ghcr.io/syslog-ng
 MODE ?= snapshot
 VERSION ?= $(shell MODE=${MODE} scripts/version.sh)
 DOCKER_RUN_ARGS=-e USER_NAME_ON_HOST=$(shell whoami)	\
@@ -49,7 +50,7 @@ GIT_RELEASE_TAG=syslog-ng-$(VERSION)
 CONFIGURE_OPTS=--enable-debug --enable-manpages --with-python=2 --prefix=/install $(CONFIGURE_ADD)
 DBLD_RULES=$(MAKE) --no-print-directory -f $(DBLD_DIR)/rules
 
-DOCKER_SHELL=$(DOCKER) run $(DOCKER_RUN_ARGS) --rm -ti balabit/syslog-ng-$* /dbld/shell
+DOCKER_SHELL=$(DOCKER) run $(DOCKER_RUN_ARGS) --rm -ti ${CONTAINER_REGISTRY}/dbld-$* /dbld/shell
 
 -include dbld/rules.conf
 
@@ -93,10 +94,10 @@ help:
 
 bootstrap: bootstrap-$(DEFAULT_IMAGE)
 bootstrap-%: setup
-	$(DOCKER) run $(DOCKER_RUN_ARGS) --rm  -i  balabit/syslog-ng-$* /dbld/bootstrap
+	$(DOCKER) run $(DOCKER_RUN_ARGS) --rm  -i  ${CONTAINER_REGISTRY}/dbld-$* /dbld/bootstrap
 
 make-%: setup
-	$(DOCKER) run $(DOCKER_RUN_ARGS) --rm -i  balabit/syslog-ng-$(DEFAULT_IMAGE) /dbld/make $(MAKE_ARGS) $*
+	$(DOCKER) run $(DOCKER_RUN_ARGS) --rm -i  ${CONTAINER_REGISTRY}/dbld-$(DEFAULT_IMAGE) /dbld/make $(MAKE_ARGS) $*
 
 tarball-from-root: setup
 	@if [ -f $(ROOT_DIR)/../$(TARBALL_BASENAME) ] && [ ! -f $(TARBALL) ]; then \
@@ -118,7 +119,7 @@ tarball-%: tarball-from-root
 		fi; \
 		echo "Git status follows:" && \
 		( $(GIT) status || echo "Git not found..." ) && \
-		$(DOCKER) run $(DOCKER_RUN_ARGS) --rm -i balabit/syslog-ng-$* /dbld/tarball; \
+		$(DOCKER) run $(DOCKER_RUN_ARGS) --rm -i ${CONTAINER_REGISTRY}/dbld-$* /dbld/tarball; \
 	else \
 		echo "Tarball $(TARBALL) is up to date (except files excluded by $(DBLD_DIR)/tarball-changes.ignore)"; \
 	fi
@@ -137,20 +138,20 @@ pkg-tarball: pkg-tarball-$(TARBALL_IMAGE)
 #
 pkg-tarball-%: tarball-%
 	@if ! tar --strip-components=1 --show-transformed-names  -tvf $(TARBALL) | grep  ' debian/rules$$' > /dev/null; then \
-		$(DOCKER) run $(DOCKER_RUN_ARGS) --rm -i balabit/syslog-ng-$* /dbld/pkg-tarball; \
+		$(DOCKER) run $(DOCKER_RUN_ARGS) --rm -i ${CONTAINER_REGISTRY}/dbld-$* /dbld/pkg-tarball; \
 	fi
 
 package: package-$(DEFAULT_IMAGE)
 package-%: pkg-tarball
-	$(DOCKER) run $(DOCKER_RUN_ARGS) --rm -i  balabit/syslog-ng-$* /dbld/package
+	$(DOCKER) run $(DOCKER_RUN_ARGS) --rm -i  ${CONTAINER_REGISTRY}/dbld-$* /dbld/package
 
 deb: deb-$(DEFAULT_DEB_IMAGE)
 deb-%: pkg-tarball
-	$(DOCKER) run $(DOCKER_RUN_ARGS) --rm -i  balabit/syslog-ng-$* /dbld/deb
+	$(DOCKER) run $(DOCKER_RUN_ARGS) --rm -i  ${CONTAINER_REGISTRY}/dbld-$* /dbld/deb
 
 rpm: rpm-$(DEFAULT_RPM_IMAGE)
 rpm-%: pkg-tarball
-	$(DOCKER) run $(DOCKER_RUN_ARGS) --rm -i  balabit/syslog-ng-$* /dbld/rpm
+	$(DOCKER) run $(DOCKER_RUN_ARGS) --rm -i  ${CONTAINER_REGISTRY}/dbld-$* /dbld/rpm
 
 validate-tree-clean:
 	@if ! $(GIT) diff-index --quiet HEAD; then \
@@ -165,10 +166,10 @@ validate-version-format:
 	fi
 
 prepare-release: setup validate-tree-clean validate-version-format
-	$(DOCKER) run $(DOCKER_RUN_ARGS) --rm -i  balabit/syslog-ng-$(TARBALL_IMAGE) /dbld/prepare-release $(VERSION)
+	$(DOCKER) run $(DOCKER_RUN_ARGS) --rm -i  ${CONTAINER_REGISTRY}/dbld-$(TARBALL_IMAGE) /dbld/prepare-release $(VERSION)
 
 validate-release: validate-tree-clean validate-version-format
-	$(DOCKER) run $(DOCKER_RUN_ARGS) --rm -i  balabit/syslog-ng-$(TARBALL_IMAGE) /dbld/validate-release-version $(VERSION)
+	$(DOCKER) run $(DOCKER_RUN_ARGS) --rm -i  ${CONTAINER_REGISTRY}/dbld-$(TARBALL_IMAGE) /dbld/validate-release-version $(VERSION)
 
 	@if $(GIT) rev-parse --verify -q "$(GIT_RELEASE_TAG)" > /dev/null; then \
 		echo "Your git tree already has $(GIT_RELEASE_TAG), this might indicate a duplicate release, please remove that first."; \
@@ -198,7 +199,7 @@ clean:
 run: run-$(DEFAULT_IMAGE)
 run: RUN_COMMAND=echo Specify RUN_COMMAND to do something sensible here
 run-%: setup
-	$(DOCKER) run $(DOCKER_RUN_ARGS) --rm -ti balabit/syslog-ng-$* bash -c "$(RUN_COMMAND)"
+	$(DOCKER) run $(DOCKER_RUN_ARGS) --rm -ti ${CONTAINER_REGISTRY}/dbld-$* bash -c "$(RUN_COMMAND)"
 
 shell: shell-$(DEFAULT_IMAGE)
 shell-%: setup
@@ -212,19 +213,19 @@ image-devshell: image-tarball
 image-tarball: image-debian-testing
 image-%:
 	$(DBLD_DIR)/prepare-image-build $* && \
-        $(DOCKER) build $(DOCKER_BUILD_ARGS) --build-arg=ARG_IMAGE_PLATFORM=$* --build-arg=COMMIT=$$($(GIT) rev-parse --short HEAD || echo "") --network=host -t balabit/syslog-ng-$* -f $(DBLD_DIR)/images/$*.dockerfile $(DBLD_DIR)
+        $(DOCKER) build $(DOCKER_BUILD_ARGS) --build-arg=ARG_IMAGE_PLATFORM=$* --build-arg=COMMIT=$$($(GIT) rev-parse --short HEAD || echo "") --build-arg=CONTAINER_REGISTRY=${CONTAINER_REGISTRY} --network=host -t ${CONTAINER_REGISTRY}/dbld-$* -f $(DBLD_DIR)/images/$*.dockerfile $(DBLD_DIR)
 
 pull-images: $(foreach image,$(BUILDER_IMAGES), pull-image-$(image))
 pull-image: pull-image-$(DEFAULT_IMAGE)
 pull-image-%:
-	$(DOCKER) pull balabit/syslog-ng-$*
+	$(DOCKER) pull ${CONTAINER_REGISTRY}/dbld-$*
 
 cache-image-%:
-	@IMAGE=balabit/syslog-ng-$*:latest;										\
+	@IMAGE=${CONTAINER_REGISTRY}/dbld-$*:latest;										\
 	IMAGE_ID=$$(docker images -q $$IMAGE | head -1);								\
 	WATCHED_FILES="dbld packaging/rhel/syslog-ng.spec packaging/debian/control";					\
 	if [ "$$IMAGE_ID" = "" ]; then 											\
-		$(DBLD_RULES) pull-image-$*;										\
+		$(DBLD_RULES) pull-image-$* || echo "Image pull failed, continuing...";					\
 	fi;														\
 	IMAGE_ID=$$(docker images -q $$IMAGE | head -1);								\
 	if [ "$$IMAGE_ID" != "" ]; then 										\
@@ -248,7 +249,7 @@ cache-image-%:
 exec: exec-$(DEFAULT_IMAGE)
 exec: EXEC_COMMAND=echo Specify EXEC_COMMAND to do something sensible here
 exec-%: setup
-	@container=`$(DOCKER) ps | grep syslog-ng-$* | head -1 | cut -d ' ' -f1`; \
+	@container=`$(DOCKER) ps | grep dbld-$* | head -1 | cut -d ' ' -f1`; \
 	$(DOCKER) exec -ti  $$container $(EXEC_COMMAND)
 
 login: login-$(DEFAULT_IMAGE)

news/developer-note-3782.md

@@ -0,0 +1,5 @@
+`dbld`: move dbld image cache from DockerHub to GitHub
+
+In 2021, GitHub introduced the GitHub Packages service. Among other
+repositories - it provides a standard Docker registry. DBLD can use
+such a regsistry, to avoid unnecessary rebuilding of the images.