ci: split image builds to speed up jobs (#1807)

* ci: split image builds to speed up jobs

* simplify makefile

.github/workflows/build-x86-image.yaml

@@ -21,8 +21,8 @@ env:
   KIND_VERSION: v0.14.0
 
 jobs:
-  build:
-    name: Build x86
+  build-kube-ovn:
+    name: Build kube-ovn
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3
@@ -59,35 +59,60 @@ jobs:
       - name: Build
         run: |
           go mod tidy
-          make release
-          make tar
+          make lint
+          make image-kube-ovn
+          make tar-kube-ovn
 
-      - name: Upload kube-ovn image to artifact
+      - name: Upload images to artifact
         uses: actions/upload-artifact@v3
         with:
           name: kube-ovn
           path: kube-ovn.tar
 
-      - name: Upload vpc-nat-gateway image to artifact
+  build-vpc-nat-gateway:
+    name: Build vpc-nat-gateway
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+      - uses: docker/setup-buildx-action@v2
+
+      - name: Build
+        run: |
+          make image-vpc-nat-gateway
+          make tar-vpc-nat-gateway
+
+      - name: Upload image to artifact
         uses: actions/upload-artifact@v3
         with:
           name: vpc-nat-gateway
           path: vpc-nat-gateway.tar
 
+  build-centos-compile:
+    name: Build centos-compile
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+      - uses: docker/setup-buildx-action@v2
+
+      - name: Build
+        run: |
+          make image-centos-compile
+          make tar-centos-compile
+
       - name: Upload centos7-compile image to artifact
         uses: actions/upload-artifact@v3
         with:
           name: centos7-compile
           path: centos7-compile.tar
 
-#      - name: Upload centos8-compile image to artifact
-#        uses: actions/upload-artifact@v3
-#        with:
-#          name: centos8-compile
-#          path: centos8-compile.tar
+      # - name: Upload centos8-compile image to artifact
+      #   uses: actions/upload-artifact@v3
+      #   with:
+      #     name: centos8-compile
+      #     path: centos8-compile.tar
 
   single-e2e:
-    needs: build
+    needs: build-kube-ovn
     name: 1-master-e2e
     runs-on: ubuntu-20.04
     timeout-minutes: 30
@@ -158,7 +183,7 @@ jobs:
           sh dist/images/cleanup.sh
 
   single-iptables-e2e:
-    needs: build
+    needs: build-kube-ovn
     name: 1-master-iptables-e2e
     runs-on: ubuntu-20.04
     timeout-minutes: 30
@@ -229,7 +254,7 @@ jobs:
           sh dist/images/cleanup.sh
 
   single-vlan-e2e:
-    needs: build
+    needs: build-kube-ovn
     name: 1-master-vlan-e2e
     runs-on: ubuntu-20.04
     timeout-minutes: 30
@@ -300,7 +325,7 @@ jobs:
           sh dist/images/cleanup.sh
 
   single-underlay-e2e-single-nic:
-    needs: build
+    needs: build-kube-ovn
     name: 1-master-underlay-e2e-single-nic
     runs-on: ubuntu-20.04
     timeout-minutes: 30
@@ -387,7 +412,7 @@ jobs:
           '
 
   single-node-e2e:
-    needs: build
+    needs: build-kube-ovn
     name: 1-node-e2e
     runs-on: ubuntu-20.04
     timeout-minutes: 30
@@ -442,7 +467,7 @@ jobs:
           max_attempts: 3
           shell: bash
           command: |
-            sudo make kind-install-single
+            sudo make kind-install
 
       - name: Run E2E
         run: |
@@ -458,7 +483,7 @@ jobs:
           sh dist/images/cleanup.sh
 
   ha-e2e:
-    needs: build
+    needs: build-kube-ovn
     name: 3-master-e2e
     runs-on: ubuntu-20.04
     timeout-minutes: 30
@@ -533,7 +558,7 @@ jobs:
           sh dist/images/cleanup.sh
 
   ipv6-e2e:
-    needs: build
+    needs: build-kube-ovn
     name: ipv6-e2e
     runs-on: ubuntu-20.04
     timeout-minutes: 30
@@ -604,7 +629,7 @@ jobs:
           sh dist/images/cleanup.sh
 
   ipv6-vlan-e2e:
-    needs: build
+    needs: build-kube-ovn
     name: ipv6-vlan-e2e
     runs-on: ubuntu-20.04
     timeout-minutes: 30
@@ -675,7 +700,7 @@ jobs:
           sh dist/images/cleanup.sh
 
   ipv6-underlay-e2e-single-nic:
-    needs: build
+    needs: build-kube-ovn
     name: ipv6-underlay-e2e-single-nic
     runs-on: ubuntu-20.04
     timeout-minutes: 30
@@ -762,7 +787,7 @@ jobs:
           '
 
   dual-stack-e2e:
-    needs: build
+    needs: build-kube-ovn
     name: dual-stack-e2e
     runs-on: ubuntu-20.04
     timeout-minutes: 30
@@ -833,7 +858,7 @@ jobs:
           sh dist/images/cleanup.sh
 
   dual-stack-underlay-e2e-single-nic:
-    needs: build
+    needs: build-kube-ovn
     name: dual-stack-underlay-e2e-single-nic
     runs-on: ubuntu-20.04
     timeout-minutes: 30
@@ -920,7 +945,7 @@ jobs:
           '
 
   dual-stack-underlay-logical-gw-e2e:
-    needs: build
+    needs: build-kube-ovn
     name: dual-stack-underlay-logical-gateway-e2e
     runs-on: ubuntu-20.04
     timeout-minutes: 30
@@ -1007,7 +1032,7 @@ jobs:
           '
 
   no-lb-e2e:
-    needs: build
+    needs: build-kube-ovn
     name: disable-loadbalancer-e2e
     runs-on: ubuntu-20.04
     timeout-minutes: 30
@@ -1078,7 +1103,7 @@ jobs:
           sh dist/images/cleanup.sh
 
   no-lb-iptables-e2e:
-    needs: build
+    needs: build-kube-ovn
     name: disable-loadbalancer-iptables-e2e
     runs-on: ubuntu-20.04
     timeout-minutes: 30
@@ -1149,7 +1174,7 @@ jobs:
           sh dist/images/cleanup.sh
 
   no-np-e2e:
-    needs: build
+    needs: build-kube-ovn
     name: disable-network-policy-e2e
     runs-on: ubuntu-20.04
     timeout-minutes: 30
@@ -1220,7 +1245,9 @@ jobs:
           sh dist/images/cleanup.sh
 
   multus-e2e:
-    needs: build
+    needs:
+      - build-kube-ovn
+      - build-vpc-nat-gateway
     name: multus-e2e
     runs-on: ubuntu-20.04
     timeout-minutes: 30
@@ -1295,7 +1322,7 @@ jobs:
           make e2e-multus
 
   ovnic-e2e:
-    needs: build
+    needs: build-kube-ovn
     name: 1-master-ovnic-e2e
     runs-on: ubuntu-20.04
     timeout-minutes: 30
@@ -1367,7 +1394,7 @@ jobs:
           sh dist/images/cleanup.sh
 
   cilium-e2e:
-    needs: build
+    needs: build-kube-ovn
     name: 1-master-cilium-e2e
     runs-on: ubuntu-20.04
     timeout-minutes: 30
@@ -1443,7 +1470,7 @@ jobs:
           sudo sh dist/images/cleanup.sh
 
   installation-compatibility-e2e:
-    needs: build
+    needs: build-kube-ovn
     name: installation-compatibility-e2e
     runs-on: ubuntu-20.04
     timeout-minutes: 10
@@ -1485,6 +1512,7 @@ jobs:
 
   push:
     needs:
+      - build-centos-compile
       - single-e2e
       - single-iptables-e2e
       - single-vlan-e2e
@@ -1524,10 +1552,10 @@ jobs:
         with:
           name: centos7-compile
 
-#      - name: Download centos8-compile image
-#        uses: actions/download-artifact@v3
-#        with:
-#          name: centos8-compile
+      # - name: Download centos8-compile image
+      #   uses: actions/download-artifact@v3
+      #   with:
+      #     name: centos8-compile
 
       - name: Load Image
         run: |

Makefile

@@ -38,19 +38,9 @@ build-go-arm:
 	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-cmd -ldflags $(GOLDFLAGS) -v ./cmd
 	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-webhook -ldflags $(GOLDFLAGS) -v ./cmd/webhook
 
-.PHONY: build-bin
-build-bin:
-	docker run --rm -e GOOS=linux -e GOCACHE=/tmp -e GOARCH=$(ARCH) -e GOPROXY=https://goproxy.cn \
-		-u $(shell id -u):$(shell id -g) \
-		-v $(CURDIR):/go/src/github.com/kubeovn/kube-ovn:ro \
-		-v $(CURDIR)/dist:/go/src/github.com/kubeovn/kube-ovn/dist/ \
-		golang:$(GO_VERSION) /bin/bash -c '\
-		cd /go/src/github.com/kubeovn/kube-ovn && \
-		make build-go '
-
-.PHONY: build-dev-images
-build-dev-images: build-bin
-	docker build -t $(REGISTRY)/kube-ovn:$(DEV_TAG) --build-arg ARCH=amd64 -f dist/images/Dockerfile dist/images/
+.PHONY: build-dev
+build-dev: build-go
+	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/kube-ovn:$(DEV_TAG) -o type=docker -f dist/images/Dockerfile dist/images/
 
 .PHONY: build-dpdk
 build-dpdk:
@@ -69,14 +59,23 @@ base-amd64-dpdk:
 base-arm64:
 	docker buildx build --platform linux/arm64 --build-arg ARCH=arm64 -t $(REGISTRY)/kube-ovn-base:$(RELEASE_TAG)-arm64 -o type=docker -f dist/images/Dockerfile.base dist/images/
 
-.PHONY: release
-release: lint build-go
+.PHONY: image-kube-ovn
+image-kube-ovn: build-go
 	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG) -o type=docker -f dist/images/Dockerfile dist/images/
 	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG)-no-avx512 -o type=docker -f dist/images/Dockerfile.no-avx512 dist/images/
 	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG)-dpdk -o type=docker -f dist/images/Dockerfile.dpdk dist/images/
+
+.PHONY: image-vpc-nat-gateway
+image-vpc-nat-gateway:
 	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/vpc-nat-gateway:$(RELEASE_TAG) -o type=docker -f dist/images/vpcnatgateway/Dockerfile dist/images/vpcnatgateway
+
+.PHONY: image-centos-compile
+image-centos-compile:
 	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/centos7-compile:$(RELEASE_TAG) -o type=docker -f dist/images/compile/centos7/Dockerfile fastpath/
-#	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/centos8-compile:$(RELEASE_TAG) -o type=docker -f dist/images/compile/centos8/Dockerfile fastpath/
+	# docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/centos8-compile:$(RELEASE_TAG) -o type=docker -f dist/images/compile/centos8/Dockerfile fastpath/
+
+.PHONY: release
+release: lint image-kube-ovn image-vpc-nat-gateway image-centos-compile
 
 .PHONY: release-arm
 release-arm: build-go-arm
@@ -91,12 +90,21 @@ push-dev:
 push-release: release
 	docker push $(REGISTRY)/kube-ovn:$(RELEASE_TAG)
 
-.PHONY: tar
-tar:
+.PHONY: tar-kube-ovn
+tar-kube-ovn:
 	docker save $(REGISTRY)/kube-ovn:$(RELEASE_TAG) $(REGISTRY)/kube-ovn:$(RELEASE_TAG)-no-avx512 -o kube-ovn.tar
+
+.PHONY: tar-vpc-nat-gateway
+tar-vpc-nat-gateway:
 	docker save $(REGISTRY)/vpc-nat-gateway:$(RELEASE_TAG) -o vpc-nat-gateway.tar
+
+.PHONY: tar-centos-compile
+tar-centos-compile:
 	docker save $(REGISTRY)/centos7-compile:$(RELEASE_TAG) -o centos7-compile.tar
-#	docker save $(REGISTRY)/centos8-compile:$(RELEASE_TAG) -o centos8-compile.tar
+	# docker save $(REGISTRY)/centos8-compile:$(RELEASE_TAG) -o centos8-compile.tar
+
+.PHONY: tar
+tar: tar-kube-ovn tar-vpc-nat-gateway tar-centos-compile
 
 .PHONY: base-tar-amd64
 base-tar-amd64:
@@ -212,14 +220,9 @@ kind-install-underlay: kind-load-image kind-untaint-control-plane
 	ENABLE_SSL=true ENABLE_VLAN=true VLAN_NIC=eth0 bash install-underlay.sh
 	kubectl describe no
 
-.PHONY: kind-install-single
-kind-install-single: kind-load-image
-	ENABLE_SSL=true dist/images/install.sh
-	kubectl describe no
-
 .PHONY: kind-install-ipv6
-kind-install-ipv6: kind-load-image kind-untaint-control-plane
-	ENABLE_SSL=true IPV6=true dist/images/install.sh
+kind-install-ipv6:
+	IPV6=true $(MAKE) kind-install
 
 .PHONY: kind-install-underlay-ipv6
 kind-install-underlay-ipv6: kind-load-image kind-untaint-control-plane
@@ -234,9 +237,8 @@ kind-install-underlay-ipv6: kind-load-image kind-untaint-control-plane
 	ENABLE_SSL=true IPV6=true ENABLE_VLAN=true VLAN_NIC=eth0 bash install-underlay.sh
 
 .PHONY: kind-install-dual
-kind-install-dual: kind-load-image kind-untaint-control-plane
-	ENABLE_SSL=true DUAL_STACK=true dist/images/install.sh
-	kubectl describe no
+kind-install-dual:
+	DUAL_STACK=true $(MAKE) kind-install
 
 .PHONY: kind-install-underlay-dual
 kind-install-underlay-dual: kind-load-image kind-untaint-control-plane
@@ -278,7 +280,6 @@ kind-install-multus: kind-load-image kind-untaint-control-plane
 	kubectl apply -f "$(MULTUS_YAML)"
 	kubectl -n kube-system rollout status ds kube-multus-ds
 	kubectl apply -f yamls/lb-svc-attachment.yaml
-	kind load docker-image --name kube-ovn $(REGISTRY)/kube-ovn:$(RELEASE_TAG)
 	kind load docker-image --name kube-ovn $(VPC_NAT_GW_IMG)
 	ENABLE_SSL=true ENABLE_LB_SVC=true CNI_CONFIG_PRIORITY=10 dist/images/install.sh
 	kubectl describe no
@@ -300,7 +301,6 @@ kind-install-ic:
 	sleep 6
 	kubectl -n kube-system get pods | grep ovs-ovn | awk '{print $$1}' | xargs kubectl -n kube-system delete pod
 
-
 .PHONY: kind-install-cilium
 kind-install-cilium: kind-load-image kind-untaint-control-plane
 	$(eval KUBERNETES_SERVICE_HOST = $(shell kubectl get nodes kube-ovn-control-plane -o jsonpath='{.status.addresses[0].address}'))