move builder step of Makefile to compile step of build.yaml

Signed-off-by: jtcheng <jtcheng@alauda.io>
alauda · Apr 15, 2024 · d38e753 · d38e753
1 parent 1474dce
commit d38e753
Show file tree

Hide file tree

Showing 5 changed files with 138 additions and 49 deletions.
diff --git a/.build/build2.yaml → .build/build.v2.yaml b/.build/build2.yaml → .build/build.v2.yaml
@@ -124,8 +124,9 @@ spec:
     ##########
     # compile amd64 and arm64
     #########
-    - name: compile-amd64
+    - name: compile-amd64-harbor
       timeout: 2.5h
+      retries: 3
       taskRef:
         kind: Task
         name: docker-in-docker
@@ -137,20 +138,64 @@ spec:
       params:
         - name: command
           value: |
+            set -ex
+            
             BUILDPATH=$(workspaces.source.path)
             VERSIONTAG=2.6.4-$(build.git.lastCommit.shortID)-amd64
             BASEIMAGETAG=${VERSIONTAG}
             BASEIMAGENAMESPACE=build-harbor.alauda.cn/devops
             IMAGENAMESPACE=build-harbor.alauda.cn/devops
             REGISTRY_SRC_TAG=v2.8.0
             
+            # compile harbor
             BUILDPATH="$(workspaces.source.path)" make compile
+    - name: compile-amd64-thirdparty
+      timeout: 2.5h
+      retries: 3
+      taskRef:
+        kind: Task
+        name: docker-in-docker
+      runAfter:
+        - patch-amd64
+      workspaces:
+        - name: source
+          workspace: source
+      params:
+        - name: command
+          value: |
+            set -ex
+            
+            BUILDPATH=$(workspaces.source.path)
+            VERSIONTAG=2.6.4-$(build.git.lastCommit.shortID)-amd64
+            BASEIMAGETAG=${VERSIONTAG}
+            BASEIMAGENAMESPACE=build-harbor.alauda.cn/devops
+            IMAGENAMESPACE=build-harbor.alauda.cn/devops
+            REGISTRY_SRC_TAG=v2.8.0
             
             # compile registry
-            cd ./make/photon/registry && ./builder ${REGISTRY_SRC_TAG}
+            cd ${BUILDPATH}/make/photon/registry && ./builder ${REGISTRY_SRC_TAG} && cd -
+
+            # compile trivy
+            TRIVYADAPTERVERSION=v0.30.7
+            mkdir -p ${BUILDPATH}/make/photon/trivy-adapter/binary
+            cd ${BUILDPATH}/make/photon/trivy-adapter && ./builder.sh ${TRIVYADAPTERVERSION} && cd -
+
+            # compile chartserver
+            GOBUILDIMAGE=golang:1.19.4
+            CHART_SERVER_CODE_BASE=https://github.com/alauda/chartmuseum.git
+            CHARTMUSEUM_SRC_TAG=v0.14.1
+            CHART_SERVER_MAIN_PATH=cmd/chartmuseum
+            CHART_SERVER_BIN_NAME=chartm
+            cd ${BUILDPATH}/make/photon/chartserver && ./builder ${GOBUILDIMAGE} ${CHART_SERVER_CODE_BASE} ${CHARTMUSEUM_SRC_TAG} ${CHART_SERVER_MAIN_PATH} ${CHART_SERVER_BIN_NAME} && cd -
+
+            # compile notary
+            NOTARYVERSION=v0.6.1
+            NOTARYMIGRATEVERSION=v4.11.0
+            cd ${BUILDPATH}/make/photon/notary && ./builder ${NOTARYVERSION} ${NOTARYMIGRATEVERSION} && cd -
 
-    - name: compile-arm64
+    - name: compile-arm64-harbor
       timeout: 2.5h
+      retries: 3
       taskRef:
         kind: Task
         name: docker-in-docker
@@ -162,17 +207,60 @@ spec:
       params:
         - name: command
           value: |
+            set -ex
+            
             BUILDPATH=$(workspaces.source.path)
             VERSIONTAG=2.6.4-$(build.git.lastCommit.shortID)-arm64
             BASEIMAGETAG=${VERSIONTAG}
             BASEIMAGENAMESPACE=build-harbor.alauda.cn/devops
             IMAGENAMESPACE=build-harbor.alauda.cn/devops
             REGISTRY_SRC_TAG=v2.8.0
             
+            # compile harbor
             BUILDPATH="$(workspaces.source.path)" make compile
-
+    - name: compile-arm64-thirdparty
+      timeout: 2.5h
+      retries: 3
+      taskRef:
+        kind: Task
+        name: docker-in-docker
+      runAfter:
+        - patch-arm64
+      workspaces:
+        - name: source
+          workspace: source-arm64
+      params:
+        - name: command
+          value: |
+            set -ex
+            
+            BUILDPATH=$(workspaces.source.path)
+            VERSIONTAG=2.6.4-$(build.git.lastCommit.shortID)-arm64
+            BASEIMAGETAG=${VERSIONTAG}
+            BASEIMAGENAMESPACE=build-harbor.alauda.cn/devops
+            IMAGENAMESPACE=build-harbor.alauda.cn/devops
+            REGISTRY_SRC_TAG=v2.8.0
+            
             # compile registry
-            cd ./make/photon/registry && ./builder ${REGISTRY_SRC_TAG}
+            cd ${BUILDPATH}/make/photon/registry && ./builder ${REGISTRY_SRC_TAG} && cd -
+
+            # compile trivy
+            TRIVYADAPTERVERSION=v0.30.7
+            mkdir -p ${BUILDPATH}/make/photon/trivy-adapter/binary
+            cd ${BUILDPATH}/make/photon/trivy-adapter && ./builder.sh ${TRIVYADAPTERVERSION} && cd -
+
+            # compile chartserver
+            GOBUILDIMAGE=golang:1.19.4
+            CHART_SERVER_CODE_BASE=https://github.com/alauda/chartmuseum.git
+            CHARTMUSEUM_SRC_TAG=v0.14.1
+            CHART_SERVER_MAIN_PATH=cmd/chartmuseum
+            CHART_SERVER_BIN_NAME=chartm
+            cd ${BUILDPATH}/make/photon/chartserver && ./builder ${GOBUILDIMAGE} ${CHART_SERVER_CODE_BASE} ${CHARTMUSEUM_SRC_TAG} ${CHART_SERVER_MAIN_PATH} ${CHART_SERVER_BIN_NAME} && cd -
+
+            # compile notary
+            NOTARYVERSION=v0.6.1
+            NOTARYMIGRATEVERSION=v4.11.0
+            cd ${BUILDPATH}/make/photon/notary && ./builder ${NOTARYVERSION} ${NOTARYMIGRATEVERSION} && cd -
 
     ##########
     # build-image amd64 and arm64
@@ -181,8 +269,8 @@ spec:
       timeout: 2.5h
       retries: 3
       runAfter:
-#        - build-image-prepare-amd64
-        - compile-amd64
+        - compile-amd64-harbor
+        - compile-amd64-thirdparty
       taskRef:
         kind: Task
         name: buildx
@@ -207,7 +295,7 @@ spec:
       params:
         - name: pre-command
           value: |
-            set -x
+            set -ex
             # some build target need wget
             apt-get update && apt-get install -y wget
             
@@ -216,23 +304,19 @@ spec:
             export BASEIMAGETAG=2.6.4-$(build.git.lastCommit.shortID)-amd64
             export BASEIMAGENAMESPACE=build-harbor.alauda.cn/devops
             export PUSHBASEIMAGE=false
+            export SKIP_BUILDBIN=true
             export DOCKERBUILD="docker buildx build --platform=linux/amd64 --builder builder --push --metadata-file .build-metadata.txt --iidfile $(results.container-image-digest.path)"
         - name: post-command
           value: |
             echo "==== build metadata"
             cat .build-metadata.txt
             echo "==== build metadata"
-            
-            cat .build-metadata.txt | jq -r '."containerimage.buildinfo".sources[0].ref' > $(results.ociContainerImageBuild-url.path)
-            
-            cat $(results.ociContainerImageBuild-url.path)
-            cat $(results.container-image-digest.path)
     - name: build-image-arm64
       timeout: 2.5h
       retries: 3
       runAfter:
-#        - build-image-prepare-arm64
-        - compile-arm64
+        - compile-arm64-harbor
+        - compile-arm64-thirdparty
       taskRef:
         kind: Task
         name: buildx
@@ -257,7 +341,7 @@ spec:
       params:
         - name: pre-command
           value: |
-            set -x
+            set -ex
             # some build target need wget
             apt-get update && apt-get install -y wget
             
@@ -266,17 +350,13 @@ spec:
             export BASEIMAGETAG=2.6.4-$(build.git.lastCommit.shortID)-arm64
             export BASEIMAGENAMESPACE=build-harbor.alauda.cn/devops
             export PUSHBASEIMAGE=false
+            export SKIP_BUILDBIN=true
             export DOCKERBUILD="docker buildx build --platform=linux/arm64 --builder builder --push --metadata-file .build-metadata.txt --iidfile $(results.container-image-digest.path)"
         - name: post-command
           value: |
             echo "==== build metadata"
             cat .build-metadata.txt
             echo "==== build metadata"
-            
-            cat .build-metadata.txt | jq -r '."containerimage.buildinfo".sources[0].ref' > $(results.ociContainerImageBuild-url.path)
-            
-            cat $(results.ociContainerImageBuild-url.path)
-            cat $(results.container-image-digest.path)
 
     ##########
     # merge
@@ -330,4 +410,4 @@ spec:
             - build-harbor.alauda.cn/devops/goharbor-trivy-adapter-photon:2.6.4-$(build.git.lastCommit.shortID)-amd64
             - build-harbor.alauda.cn/devops/goharbor-trivy-adapter-photon:2.6.4-$(build.git.lastCommit.shortID)-arm64
             - build-harbor.alauda.cn/devops/goharbor-harbor-exporter:2.6.4-$(build.git.lastCommit.shortID)-amd64
-            - build-harbor.alauda.cn/devops/goharbor-harbor-exporter:2.6.4-$(build.git.lastCommit.shortID)-arm64
+            - build-harbor.alauda.cn/devops/goharbor-harbor-exporter:2.6.4-$(build.git.lastCommit.shortID)-arm64
diff --git a/make/photon/Makefile b/make/photon/Makefile
@@ -165,15 +165,17 @@ _build_log:
 _build_trivy_adapter:
 	@if [ "$(TRIVYFLAG)" = "true" ] ; then \
 		$(call _build_base,$(TRIVY_ADAPTER),$(DOCKERFILEPATH_TRIVY_ADAPTER)) ; \
-		rm -rf $(DOCKERFILEPATH_TRIVY_ADAPTER)/binary && mkdir -p $(DOCKERFILEPATH_TRIVY_ADAPTER)/binary ; \
+		mkdir -p $(DOCKERFILEPATH_TRIVY_ADAPTER)/binary ; \
 		echo "Downloading Trivy scanner $(TRIVYVERSION)..." ; \
 		$(call _extract_archive, $(TRIVY_DOWNLOAD_URL), $(DOCKERFILEPATH_TRIVY_ADAPTER)/binary/) ; \
-		if [ "$(BUILDBIN)" != "true" ] ; then \
-			echo "Downloading Trivy adapter $(TRIVYADAPTERVERSION)..." ; \
-			$(call _extract_archive, $(TRIVY_ADAPTER_DOWNLOAD_URL), $(DOCKERFILEPATH_TRIVY_ADAPTER)/binary/) ; \
-		else \
-			echo "Building Trivy adapter $(TRIVYADAPTERVERSION) from sources..." ; \
-			cd $(DOCKERFILEPATH_TRIVY_ADAPTER) && $(DOCKERFILEPATH_TRIVY_ADAPTER)/builder.sh $(TRIVYADAPTERVERSION) && cd - ; \
+		if [ ! "$(SKIP_BUILDBIN)" ] ; then \
+			if [ "$(BUILDBIN)" != "true" ] ; then \
+				echo "Downloading Trivy adapter $(TRIVYADAPTERVERSION)..." ; \
+				$(call _extract_archive, $(TRIVY_ADAPTER_DOWNLOAD_URL), $(DOCKERFILEPATH_TRIVY_ADAPTER)/binary/) ; \
+			else \
+				echo "Building Trivy adapter $(TRIVYADAPTERVERSION) from sources..." ; \
+				cd $(DOCKERFILEPATH_TRIVY_ADAPTER) && $(DOCKERFILEPATH_TRIVY_ADAPTER)/builder.sh $(TRIVYADAPTERVERSION) && cd - ; \
+			fi ; \
 		fi ; \
 		echo "Building Trivy adapter container for photon..." ; \
 		$(DOCKERBUILD_WITH_PULL_PARA) --build-arg harbor_base_image_version=$(BASEIMAGETAG) \
@@ -187,12 +189,14 @@ _build_trivy_adapter:
 
 _build_chart_server:
 	@if [ "$(CHARTFLAG)" = "true" ] ; then \
-		$(call _build_base,$(CHARTSERVER),$(DOCKERFILEPATH_CHART_SERVER)); \
-		if [ "$(BUILDBIN)" != "true" ] ; then \
-			rm -rf $(DOCKERFILEPATH_CHART_SERVER)/binary && mkdir -p $(DOCKERFILEPATH_CHART_SERVER)/binary && \
-			$(call _get_binary, $(CHARTURL), $(DOCKERFILEPATH_CHART_SERVER)/binary/chartm); \
-		else \
-			cd $(DOCKERFILEPATH_CHART_SERVER) && $(DOCKERFILEPATH_CHART_SERVER)/builder $(GOBUILDIMAGE) $(CHART_SERVER_CODE_BASE) $(CHARTMUSEUM_SRC_TAG) $(CHART_SERVER_MAIN_PATH) $(CHART_SERVER_BIN_NAME) && cd - ; \
+  		$(call _build_base,$(CHARTSERVER),$(DOCKERFILEPATH_CHART_SERVER)); \
+		if [ ! "$(SKIP_BUILDBIN)" ] ; then \
+			if [ "$(BUILDBIN)" != "true" ] ; then \
+				rm -rf $(DOCKERFILEPATH_CHART_SERVER)/binary && mkdir -p $(DOCKERFILEPATH_CHART_SERVER)/binary && \
+				$(call _get_binary, $(CHARTURL), $(DOCKERFILEPATH_CHART_SERVER)/binary/chartm); \
+			else \
+				cd $(DOCKERFILEPATH_CHART_SERVER) && $(DOCKERFILEPATH_CHART_SERVER)/builder $(GOBUILDIMAGE) $(CHART_SERVER_CODE_BASE) $(CHARTMUSEUM_SRC_TAG) $(CHART_SERVER_MAIN_PATH) $(CHART_SERVER_BIN_NAME) && cd - ; \
+			fi ; \
 		fi ; \
 		echo "building chartmuseum container for photon..." ; \
 		$(DOCKERBUILD_WITH_PULL_PARA) --build-arg harbor_base_image_version=$(BASEIMAGETAG) --build-arg harbor_base_namespace=$(BASEIMAGENAMESPACE) -f $(DOCKERFILEPATH_CHART_SERVER)/$(DOCKERFILENAME_CHART_SERVER) -t $(DOCKERIMAGENAME_CHART_SERVER):$(VERSIONTAG) . ; \
@@ -210,12 +214,14 @@ _build_notary:
 	@if [ "$(NOTARYFLAG)" = "true" ] ; then \
 		$(call _build_base,$(NOTARYSERVER),$(DOCKERFILEPATH_NOTARYSERVER)) ; \
 		$(call _build_base,$(NOTARYSIGNER),$(DOCKERFILEPATH_NOTARYSIGNER)) ; \
-		if [ "$(BUILDBIN)" != "true" ] ; then \
-			rm -rf $(DOCKERFILEPATH_NOTARY)/binary && mkdir -p $(DOCKERFILEPATH_NOTARY)/binary && \
-			$(call _get_binary, $(NOTARYURL), $(DOCKERFILEPATH_NOTARY)/binary-bundle.tgz); \
-			cd $(DOCKERFILEPATH_NOTARY) && tar -zvxf binary-bundle.tgz && cd - ; \
-		else \
-			cd $(DOCKERFILEPATH_NOTARY) && $(DOCKERFILEPATH_NOTARY)/builder $(NOTARYVERSION) $(NOTARYMIGRATEVERSION) && cd - ; \
+		if [ ! "$(SKIP_BUILDBIN)" ] ; then \
+			if [ "$(BUILDBIN)" != "true" ] ; then \
+				rm -rf $(DOCKERFILEPATH_NOTARY)/binary && mkdir -p $(DOCKERFILEPATH_NOTARY)/binary && \
+				$(call _get_binary, $(NOTARYURL), $(DOCKERFILEPATH_NOTARY)/binary-bundle.tgz); \
+				cd $(DOCKERFILEPATH_NOTARY) && tar -zvxf binary-bundle.tgz && cd - ; \
+			else \
+				cd $(DOCKERFILEPATH_NOTARY) && $(DOCKERFILEPATH_NOTARY)/builder $(NOTARYVERSION) $(NOTARYMIGRATEVERSION) && cd - ; \
+			fi ; \
 		fi ; \
 		echo "building notary container for photon..."; \
 		chmod 655 $(DOCKERFILEPATH_NOTARY)/binary/notary-signer && $(DOCKERBUILD_WITH_PULL_PARA) --build-arg harbor_base_image_version=$(BASEIMAGETAG) --build-arg harbor_base_namespace=$(BASEIMAGENAMESPACE) -f $(DOCKERFILEPATH_NOTARYSIGNER)/$(DOCKERFILENAME_NOTARYSIGNER) -t $(DOCKERIMAGENAME_NOTARYSIGNER):$(VERSIONTAG) . ; \
@@ -226,11 +232,13 @@ _build_notary:
 
 _build_registry:
 	@$(call _build_base,$(REGISTRY),$(DOCKERFILEPATH_REG))
-	@if [ "$(BUILDBIN)" != "true" ] ; then \
-		rm -rf $(DOCKERFILEPATH_REG)/binary && mkdir -p $(DOCKERFILEPATH_REG)/binary && \
-		$(call _get_binary, $(REGISTRYURL), $(DOCKERFILEPATH_REG)/binary/registry); \
-	else \
-		cd $(DOCKERFILEPATH_REG) && $(DOCKERFILEPATH_REG)/builder $(REGISTRY_SRC_TAG) && cd - ; \
+	@if [ ! "$(SKIP_BUILDBIN)" ] ; then \
+		if [ "$(BUILDBIN)" != "true" ] ; then \
+			rm -rf $(DOCKERFILEPATH_REG)/binary && mkdir -p $(DOCKERFILEPATH_REG)/binary && \
+			$(call _get_binary, $(REGISTRYURL), $(DOCKERFILEPATH_REG)/binary/registry); \
+		else \
+			cd $(DOCKERFILEPATH_REG) && $(DOCKERFILEPATH_REG)/builder $(REGISTRY_SRC_TAG) && cd - ; \
+		fi ; \
 	fi
 	@echo "building registry container for photon..."
 	@chmod 655 $(DOCKERFILEPATH_REG)/binary/registry && $(DOCKERBUILD_WITH_PULL_PARA) --build-arg harbor_base_image_version=$(BASEIMAGETAG) --build-arg harbor_base_namespace=$(BASEIMAGENAMESPACE) -f $(DOCKERFILEPATH_REG)/$(DOCKERFILENAME_REG) -t $(DOCKERIMAGENAME_REG):$(VERSIONTAG) .

diff --git a/make/photon/chartserver/compile.sh b/make/photon/chartserver/compile.sh
@@ -16,11 +16,12 @@ VERSION="$2"
 MAIN_GO_PATH="$3"
 BIN_NAME="$4"
 
+set -e
+
 #Get the source code
 git clone $GIT_PATH src_code
 ls
 SRC_PATH=$(pwd)/src_code
-set -e
 
 #Checkout the released tag branch
 cd $SRC_PATH

diff --git a/make/photon/registry/builder b/make/photon/registry/builder
@@ -26,7 +26,7 @@ git clone -b $VERSION https://github.com/distribution/distribution.git $TEMP
 # add patch 2815
 echo 'add patch https://github.com/distribution/distribution/pull/2815 ...'
 cd $TEMP
-curl -O -L https://github.com/distribution/distribution/pull/2815.patch
+curl -O -L --retry 3 -f https://github.com/distribution/distribution/pull/2815.patch
 git apply 2815.patch
 
 # add patch redis

diff --git a/make/photon/trivy-adapter/Dockerfile b/make/photon/trivy-adapter/Dockerfile
@@ -6,7 +6,7 @@ ARG trivy_version
 
 COPY ./make/photon/common/install_cert.sh /home/scanner
 COPY ./make/photon/trivy-adapter/entrypoint.sh /home/scanner
-COPY ./make/photon/common/exports_env_in_dir.sh /home/scanner/
+COPY ./make/photon/common/exports_env_in_dir.sh /home/scanner
 COPY ./make/photon/trivy-adapter/binary/trivy /usr/local/bin/trivy
 COPY ./make/photon/trivy-adapter/binary/scanner-trivy /home/scanner/bin/scanner-trivy