Rework the Makefile so its not racy (#105)

## Description Reworks/overhauls the Makefile setup so that we have proper deps and targets, while minimizing repetition. Also merges ci.yaml and push.yaml into just one file/workflow ## Why is this needed I tried building hook locally and was confused as to how to do it and what it was going to do. Once I opened the Makefile I could not let it be. I merged ci.yaml and push.yaml because push.yaml was potentially overwriting the 0.0 tags for the container images in quay.io if a push to a non-main branch occurred. This would cause anyone building hook locally to unintended/possibly buggy code. ## How Has This Been Tested? I've run so many `make` invocations its not even funny. I've built the arm and amd targets for image and the containers. I've run `make dist` and have the expected tarball. I've also run `make run` and have booted a qemu vm with the built files. Everything good so far. Also ran through github actions a bunch. ## How are existing users impacted? What migration steps/scripts do we need? Able to build hook locally, more easily/reliably. Won't get unexpected containers when building hook. ## Checklist: I have: - [ ] updated the documentation and/or roadmap (if required) - [ ] added unit or e2e tests - [ ] provided instructions on how to upgrade
tinkerbell · May 24, 2022 · deb3a19 · deb3a19
2 parents 90e373a + 7798824
commit deb3a19
Show file tree

Hide file tree

Showing 24 changed files with 282 additions and 389 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,2 @@
+*
+!shell.nix
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -1,46 +1,43 @@
-name: For each PR
+name: For each PR and Push
 on:
   pull_request:
     paths-ignore:
       - kernel/**
+  push:
+    paths-ignore:
+      - kernel/**
 jobs:
   validation:
     runs-on: ubuntu-20.04
-    services:
-      registry:
-        image: registry:2
-        ports:
-          - 5000:5000
     steps:
+      - name: Setup Dynamic Env
+        run: |
+          echo "MAKEFLAGS=-j$(nproc)" | tee $GITHUB_ENV
+
       - name: Checkout code
         uses: actions/checkout@v3
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
 
       - name: Set up Docker Buildx
-        id: buildx
         uses: docker/setup-buildx-action@v2
-        with:
-          driver-opts: network=host
 
-      - name: Build and push bootkit
-        uses: docker/build-push-action@v3
+      - name: Login to quay.io
+        if: github.ref == 'refs/heads/main'
+        uses: docker/login-action@v1
         with:
-          context: ./bootkit/
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: localhost:5000/tinkerbell/hook-bootkit:latest
+          registry: quay.io
+          username: ${{ secrets.QUAY_USERNAME }}
+          password: ${{ secrets.QUAY_PASSWORD }}
 
-      - name: Build and push tink-docker
-        uses: docker/build-push-action@v3
-        with:
-          context: ./tink-docker/
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: localhost:5000/tinkerbell/hook-docker:latest
+      - name: Figure Out Commit Short ID
+        id: commitid
+        run: |
+          echo ::set-output name=short::$(git rev-parse --short HEAD)
 
-      - uses: cachix/install-nix-action@v17
+      - name: Install nix
+        uses: cachix/install-nix-action@v17
         with:
           nix_path: nixpkgs=channel:nixos-unstable
 
@@ -50,14 +47,20 @@ jobs:
       - name: Run formatters and linters
         run: nix-shell --run .github/workflows/formatters-and-linters.sh
 
-      # Replace hook-{bootkit,docker} but not hook-kernel
-      - run: sed -E -i 's,quay.io/tinkerbell/hook-(bootkit|docker),localhost:5000/tinkerbell/hook-\1,g' hook.in.yaml
+      - name: Build Hook Tarballs
+        run: nix-shell --run 'make TAG=${{steps.commitid.outputs.short}} dist'
 
-      - name: Build
-        run: ./hack/ci-build.sh
+      - name: Publish Hook
+        if: github.ref == 'refs/heads/main'
+        run: |
+          # Build and push the container images
+          nix-shell --run 'make TAG=${{steps.commitid.outputs.short}} push'
+          nix-shell --run 'make TAG=latest push'
+          # Build and push the linuxkit images
+          nix-shell --run 'make TAG=${{steps.commitid.outputs.short}} deploy'
+          nix-shell --run 'make TAG=latest deploy'
 
-      # TODO: add artifacts for the built images
       - uses: actions/upload-artifact@v3
         with:
-          name: hook-${{ github.sha }}.tar.gz
-          path: hook-${{ github.sha }}.tar.gz
+          name: hook-${{steps.commitid.outputs.short}}.tar.gz
+          path: out/${{steps.commitid.outputs.short}}/rel/hook-${{steps.commitid.outputs.short}}.tar.gz
diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml
diff --git a/.gitignore b/.gitignore
@@ -1,9 +1,10 @@
 bin/
-bootkit/local/
+dbg/
 dist/
 .env
-hook-*.tar.gz
-hook.yaml
+hook-*
+!/hook-bootkit/
+!/hook-docker/
+hook*.*.yaml
 out/
 *.swp
-tink-docker/local/
diff --git a/Makefile b/Makefile
@@ -1,131 +1,37 @@
-ORG ?= quay.io/tinkerbell
-ARCH := $(shell uname -m)
-
-ifeq ($(strip $(TAG)),)
-  # ^ guards against TAG being defined but empty string which makes `TAG ?=` not work
-  TAG := latest
+# set the ORG
+### !!NOTE!!
+# If this is changed then a fresh output dir is required (`git clean -fxd` or just `rm -rf out`)
+# Handling this better shows some of make's suckiness compared to newer build tools (redo, tup ...) where the command lines to tools invoked isn't tracked by make
+ORG := quay.io/tinkerbell
+# makes sure there's no trailing / so we can just add them in the recipes which looks nicer
+ORG := $(shell echo "${ORG}" | sed 's|/*$$||')
+
+# The following `ifeq` are the equivalent of FOO ?= except that they work correctly if FOO is set but empty
+ifeq ($(strip $(LINUXKIT_CONFIG)),)
+  LINUXKIT_CONFIG := hook.yaml
 endif
-default: bootkitBuild tink-dockerBuild image
 
-dev: dev-bootkitBuild dev-tink-dockerBuild
-ifeq ($(ARCH),x86_64)
-dev: dev-image-amd64
-endif
-ifeq ($(ARCH),aarch64)
-dev: dev-image-arm64
+ifeq ($(strip $(TAG)),)
+  TAG := sha-$(shell git rev-parse --short HEAD)
 endif
+T := $(strip $(TAG))
 
-# This option is for running docker manifest command
-export DOCKER_CLI_EXPERIMENTAL := enabled
-
-LINUXKIT_CONFIG ?= hook.in.yaml
-.PHONY: hook.yaml
-hook.yaml: $(LINUXKIT_CONFIG)
-	sed '/quay.io/ s|:latest|:$(TAG)|' $^ > $@.tmp
-	mv $@.tmp $@
-
-image-amd64: hook.yaml
-	mkdir -p out
-	linuxkit build -docker -pull -format kernel+initrd -name hook-x86_64 -dir out hook.yaml
-
-image-arm64: hook.yaml
-	mkdir -p out
-	linuxkit build -docker -pull -arch arm64 -format kernel+initrd -name hook-aarch64 -dir out hook.yaml
-
-dev-image-amd64: hook.yaml
-	mkdir -p out
-	linuxkit build -docker -format kernel+initrd -name hook-x86_64 -dir out hook.yaml
-
-dev-image-arm64: hook.yaml
-	mkdir -p out
-	linuxkit build -docker -arch arm64 -format kernel+initrd -name hook-aarch64 -dir out hook.yaml
-
-image: image-amd64 image-arm64
-
-debug-image-amd64:
-	mkdir -p out/amd64
-	linuxkit build --docker -format kernel+initrd -name debug-x86_64 -dir out hook_debug.yaml
+help: ## Print this help
+	@grep --no-filename -E '^[[:alnum:]_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sed 's/:.*## /·/' | sort | column -t -s '·' -c $$(tput cols)
 
-debug-image-arm64:
-	mkdir -p out/arm64
-	linuxkit build --docker -arch arm64 -format kernel+initrd -name debug-aarch64 -dir out hook_debug.yaml
+include rules.mk
+include lint.mk
 
-debug-image: debug-image-amd64 debug-image-arm64
+all: containers images ## Build release mode boot files and container images for all supported architectures
 
-run-amd64:
-	sudo ~/go/bin/linuxkit run qemu --mem 2048 out/hook-x86_64
+dev: image-dbg-$(ARCH) ## Build debug mode boot files and container images for currently running architecture
 
-run-arm64:
-	sudo ~/go/bin/linuxkit run qemu --mem 2048 out/hook-aarch64
+images: ## Build release mode boot files for all supported architectures
 
-run:
-	sudo ~/go/bin/linuxkit run qemu --mem 2048 out/hook-${ARCH}
+containers: hook-bootkit hook-docker ## Build container images
 
-dev-bootkitBuild:
-	cd bootkit; docker buildx build --load -t $(ORG)/hook-bootkit:$(TAG) .
-
-bootkitBuild:
-	cd bootkit; docker buildx build --platform linux/amd64,linux/arm64 --push -t $(ORG)/hook-bootkit:$(TAG) .
-
-dev-tink-dockerBuild:
-	cd tink-docker; docker buildx build --load -t $(ORG)/hook-docker:$(TAG) .
-
-tink-dockerBuild:
-	cd tink-docker; docker buildx build --platform linux/amd64,linux/arm64 --push -t $(ORG)/hook-docker:$(TAG) .
-
-dev-convert:
-	rm -rf ./convert
-	mkdir ./convert
-	cp out/hook-${ARCH}-initrd.img ./convert/initrd.gz
-	cd convert/; gunzip ./initrd.gz; cpio -idv < initrd; rm initrd; find . -print0 | cpio --null -ov --format=newc > ../initramfs-${ARCH}; gzip ../initramfs-${ARCH}
-
-.PHONY: convert
-convert:
-	for a in x86_64 aarch64; do \
-		rm -rf ./convert; \
-		mkdir ./convert; \
-		cp out/hook-$$a-initrd.img ./convert/initrd.gz; \
-		cd convert/; gunzip ./initrd.gz; cpio -idv < initrd; rm initrd; find . -print0 | cpio --null -ov --format=newc > ../initramfs-$$a; gzip ../initramfs-$$a; cd ../;\
-	done
-
-dist: default convert
-	rm -rf ./dist ./convert
-	mkdir ./dist
-	for a in x86_64 aarch64; do \
-		mv ./initramfs-$$a.gz ./dist/initramfs-$$a; \
-		mv ./out/hook-$$a-kernel ./dist/vmlinuz-$$a; \
-	done
-	rm -rf out
-	cd ./dist && tar -czvf ../hook-${TAG}.tar.gz ./*
-
-dist-existing-images: image convert
-	rm -rf ./dist ./convert
-	mkdir ./dist
-	for a in x86_64 aarch64; do \
-		mv ./initramfs-$$a.gz ./dist/initramfs-$$a; \
-		mv ./out/hook-$$a-kernel ./dist/vmlinuz-$$a; \
-	done
-	rm -rf out
-	cd ./dist && tar -czvf ../hook-${TAG}.tar.gz ./*
-
-
-dev-dist: dev dev-convert
-	rm -rf ./dist ./convert
-	mkdir ./dist
-	mv ./initramfs-${ARCH}.gz ./dist/initramfs-${ARCH}
-	mv ./out/hook-${ARCH}-kernel ./dist/vmlinuz-${ARCH}
-	rm -rf out
-	cd ./dist && tar -czvf ../hook-${TAG}.tar.gz ./*
-
-deploy: dist
-ifeq ($(shell git rev-parse --abbrev-ref HEAD),main)
-	s3cmd sync ./hook-${TAG}.tar.gz s3://s.gianarb.it/hook/${TAG}.tar.gz
-	s3cmd cp s3://s.gianarb.it/hook/hook-${TAG}.tar.gz s3://s.gianarb.it/hook/hook-main.tar.gz
-endif
+debug: ## Build debug mode boot files and container images for all supported architectures
 
-.PHONY: clean
-clean:
-	rm ./hook-${TAG}.tar.gz
-	rm -rf dist/ out/ tink-docker/local/ bootkit/local/
+push: push-hook-bootkit push-hook-docker ## Push container images to registry
 
--include lint.mk
+run: run-$(ARCH) ## Boot system using qemu