Cross compile from GitHub actions

.github/workflows/e2e_parallel_jobs.yaml

@@ -31,8 +31,8 @@ jobs:
 
       - name: Install dependences
         run: |
-          GO111MODULE="on" go get sigs.k8s.io/kind@v0.11.0
-          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.23.0/bin/linux/amd64/kubectl && sudo install kubectl /usr/local/bin/kubectl
+          GO111MODULE="on" go install sigs.k8s.io/kind@v0.12.0
+          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.23.4/bin/linux/amd64/kubectl && sudo install kubectl /usr/local/bin/kubectl
       - name: Checkout code
         uses: actions/checkout@v2
 

.github/workflows/e2e_scheduling_actions.yaml

@@ -31,8 +31,8 @@ jobs:
 
       - name: Install dependences
         run: |
-          GO111MODULE="on" go get sigs.k8s.io/kind@v0.11.0
-          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.23.0/bin/linux/amd64/kubectl && sudo install kubectl /usr/local/bin/kubectl
+          GO111MODULE="on" go get sigs.k8s.io/kind@v0.12.0
+          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.23.4/bin/linux/amd64/kubectl && sudo install kubectl /usr/local/bin/kubectl
       - name: Checkout code
         uses: actions/checkout@v2
 

.github/workflows/e2e_scheduling_basic.yaml

@@ -31,8 +31,8 @@ jobs:
 
       - name: Install dependences
         run: |
-          GO111MODULE="on" go get sigs.k8s.io/kind@v0.11.0
-          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.23.0/bin/linux/amd64/kubectl && sudo install kubectl /usr/local/bin/kubectl
+          GO111MODULE="on" go get sigs.k8s.io/kind@v0.12.0
+          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.23.4/bin/linux/amd64/kubectl && sudo install kubectl /usr/local/bin/kubectl
       - name: Checkout code
         uses: actions/checkout@v2
 

.github/workflows/e2e_sequence.yaml

@@ -31,8 +31,8 @@ jobs:
 
       - name: Install dependences
         run: |
-          GO111MODULE="on" go get sigs.k8s.io/kind@v0.11.0
-          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.23.0/bin/linux/amd64/kubectl && sudo install kubectl /usr/local/bin/kubectl
+          GO111MODULE="on" go get sigs.k8s.io/kind@v0.12.0
+          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.23.4/bin/linux/amd64/kubectl && sudo install kubectl /usr/local/bin/kubectl
       - name: Checkout code
         uses: actions/checkout@v2
 

.github/workflows/e2e_vcctl.yaml

@@ -31,8 +31,8 @@ jobs:
 
       - name: Install dependences
         run: |
-          GO111MODULE="on" go get sigs.k8s.io/kind@v0.11.0
-          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.23.0/bin/linux/amd64/kubectl && sudo install kubectl /usr/local/bin/kubectl
+          GO111MODULE="on" go get sigs.k8s.io/kind@v0.12.0
+          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.23.4/bin/linux/amd64/kubectl && sudo install kubectl /usr/local/bin/kubectl
       - name: Checkout code
         uses: actions/checkout@v2
 

.github/workflows/release.yaml

@@ -1,8 +1,10 @@
 name: daily release
 on:
-  schedule:
-  # Runs at 0:00 and 12:00 UTC every day
-  - cron:  '0 */12 * * *'
+  push:
+    branches:
+      - master
+    tags:
+  pull_request:
 
 jobs:
   docker:
@@ -41,15 +43,24 @@ jobs:
           make unit-test
         working-directory: ./src/github.com/${{ github.repository }}
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+        with:
+          platforms: arm64
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
       - name: Login to DockerHub
         uses: docker/login-action@v1 
         with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Daily Release
-        run: make TAG=latest RELEASE_VER=latest DOCKER_USERNAME=${{ secrets.DOCKERHUB_USERNAME }} DOCKER_PASSWORD=${{ secrets.DOCKERHUB_TOKEN }} CC=/usr/local/musl/bin/musl-gcc release
+        run: make DOCKER_PLATFORMS="linux/amd64,linux/arm64" BUILDX_OUTPUT_TYPE=registry IMAGE_PREFIX=ghcr.io/martin-g/volcano-from-gha TAG=latest RELEASE_VER=latest CC=/usr/local/musl/bin/musl-gcc release
         working-directory: ./src/github.com/${{ github.repository }}
 
-      - name: Loginout DockerHub
+      - name: Logout DockerHub
         run: docker logout

Makefile

@@ -20,6 +20,7 @@ CRD_OPTIONS ?= "crd:crdVersions=v1,generateEmbeddedObjectMeta=true"
 CC ?= "gcc"
 SUPPORT_PLUGINS ?= "no"
 CRD_VERSION ?= v1
+BUILDX_OUTPUT_TYPE ?= "docker"
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -52,6 +53,10 @@ else
 REL_OSARCH=linux/$(OSARCH)
 endif
 
+# Run make images DOCKER_PLATFORMS="linux/amd64,linux/arm64" BUILDX_OUTPUT_TYPE=registry IMAGE_PREFIX=[yourregistry]/vc to push multi-platform
+DOCKER_PLATFORMS ?= "${REL_OSARCH}"
+
+
 include Makefile.def
 
 .EXPORT_ALL_VARIABLES:
@@ -84,30 +89,16 @@ image_bins: init
 	if [ ${SUPPORT_PLUGINS} = "yes" ];then\
 		CC=${CC} CGO_ENABLED=1 $(GOBIN)/gox -osarch=${REL_OSARCH} -ldflags ${LD_FLAGS} -output ${BIN_DIR}/${REL_OSARCH}/vc-scheduler ./cmd/scheduler;\
 	else\
-	 	CC=${CC} CGO_ENABLED=0 $(GOBIN)/gox -osarch=${REL_OSARCH} -ldflags ${LD_FLAGS} -output ${BIN_DIR}/${REL_OSARCH}/vc-scheduler ./cmd/scheduler;\
-  	fi;
+		CC=${CC} CGO_ENABLED=0 $(GOBIN)/gox -osarch=${REL_OSARCH} -ldflags ${LD_FLAGS} -output ${BIN_DIR}/${REL_OSARCH}/vc-scheduler ./cmd/scheduler;\
+	fi;
 
-images: image_bins
+images:
 	for name in controller-manager scheduler webhook-manager; do\
-		cp ${BIN_DIR}/${REL_OSARCH}/vc-$$name ./installer/dockerfile/$$name/;\
-		if [ ${REL_OSARCH} = linux/amd64 ];then\
-			docker build --no-cache -t $(IMAGE_PREFIX)-$$name:$(TAG) ./installer/dockerfile/$$name;\
-		elif [ ${REL_OSARCH} = linux/arm64 ];then\
-			docker build --no-cache -t $(IMAGE_PREFIX)-$$name-arm64:$(TAG) -f ./installer/dockerfile/$$name/Dockerfile.arm64 ./installer/dockerfile/$$name;\
-		else\
-			echo "only support x86_64 and arm64. Please build image according to your architecture";\
-		fi;\
-		rm installer/dockerfile/$$name/vc-$$name;\
+		docker buildx build -t "${IMAGE_PREFIX}-$$name:$(TAG)" . -f ./installer/dockerfile/$$name/Dockerfile --output=type="${BUILDX_OUTPUT_TYPE}" --platform "${DOCKER_PLATFORMS}"; \
 	done
 
 webhook-manager-base-image:
-	if [ ${REL_OSARCH} = linux/amd64 ];then\
-		docker build --no-cache -t $(IMAGE_PREFIX)-webhook-manager-base:$(TAG) ./installer/dockerfile/webhook-manager/ -f ./installer/dockerfile/webhook-manager/Dockerfile.base;\
-	elif [ ${REL_OSARCH} = linux/arm64 ];then\
-		docker build --no-cache -t $(IMAGE_PREFIX)-webhook-manager-base-arm64:$(TAG) ./installer/dockerfile/webhook-manager/ -f ./installer/dockerfile/webhook-manager/Dockerfile.base.arm64;\
-	else\
-		echo "only support x86_64 and arm64. Please build webhook-manager-base-image according to your architecture";\
-	fi
+	docker buildx build -t ${IMAGE_PREFIX}-webhook-manager-base:$(TAG) . -f ./installer/dockerfile/webhook-manager/Dockerfile.base --output=type="${BUILDX_OUTPUT_TYPE}" --platform "${DOCKER_PLATFORMS}"
 
 generate-code:
 	./hack/update-gencode.sh

docs/development/development.md

@@ -51,6 +51,12 @@ Build the containers in your local docker cache:
 make images
 ```
 
+To build cross-platform images:
+
+```bash
+make images DOCKER_PLATFORMS="linux/amd64,linux/arm64" BUILDX_OUTPUT_TYPE=registry IMAGE_PREFIX=[yourregistry]
+```
+
 ## Building a specific docker image
 
 If you want to make a local change and test some component, say `vc-controller-manager`, you

hack/publish.sh

@@ -30,7 +30,7 @@ set -o pipefail
 #   7. generate zip file
 
 VK_ROOT=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/..
-BINARY_FOLDER=${VK_ROOT}/${BIN_DIR}/${REL_OSARCH}
+BINARY_FOLDER=${VK_ROOT}/${BIN_DIR}
 RELEASE_FOLDER=${VK_ROOT}/${RELEASE_DIR}
 RELEASE_BINARY=${RELEASE_FOLDER}/bin
 README_FILE=${VK_ROOT}/installer/README.md
@@ -44,7 +44,7 @@ if [[ ! -d ${RELEASE_BINARY} ]];then
     mkdir ${RELEASE_BINARY}
 fi
 
-cp -r ${BINARY_FOLDER} ${RELEASE_BINARY}
+cp -r ${BINARY_FOLDER}/* ${RELEASE_BINARY}
 
 cp ${README_FILE} ${RELEASE_FOLDER}
 
@@ -57,11 +57,9 @@ fi
 # overwrite the tag name into values yaml
 sed -i "s/latest/${VOLCANO_IMAGE_TAG}/g" ${RELEASE_FOLDER}/helm/chart/volcano/values.yaml
 
-if [[ "${DOCKER_USERNAME}xxx" == "xxx" ]];then
-  if [[ "${DOCKER_PASSWORD}xxx" == "xxx" ]];then
-    echo "docker username or password not found, quit uploading images"
-    exit 0
-  fi
+if [ "${DOCKER_USERNAME}xxx" == "xxx" ] || [ "${DOCKER_PASSWORD}xxx" == "xxx" ];then
+  echo "docker username or password not found, quit uploading images"
+  exit 0
 fi
 
 echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin

installer/dockerfile/controller-manager/Dockerfile

@@ -12,8 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+FROM golang:1.17.7 AS builder
+WORKDIR /go/src/volcano.sh/
+ADD . volcano
+RUN cd volcano && make vc-controller-manager
 
 FROM alpine:latest
 
-ADD vc-controller-manager /vc-controller-manager
+COPY --from=builder /go/src/volcano.sh/volcano/_output/bin/vc-controller-manager /vc-controller-manager
 ENTRYPOINT ["/vc-controller-manager"]

installer/dockerfile/scheduler/Dockerfile

@@ -12,8 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
+FROM golang:1.17.7 AS builder
+WORKDIR /go/src/volcano.sh/
+ADD . volcano
+RUN cd volcano && make vc-scheduler
 FROM alpine:latest
 
-ADD vc-scheduler /vc-scheduler
+COPY --from=builder /go/src/volcano.sh/volcano/_output/bin/vc-scheduler /vc-scheduler
 ENTRYPOINT ["/vc-scheduler"]

installer/dockerfile/webhook-manager/Dockerfile

@@ -15,8 +15,13 @@
 
 # The base image is created via `Dockerfile.base`, the base image is cached
 # since the required packages change very rarely.
+FROM golang:1.17.7 AS builder
+WORKDIR /go/src/volcano.sh/
+ADD . volcano
+RUN cd volcano && make vc-webhook-manager
+
 FROM volcanosh/vc-webhook-manager-base:1.3.1
 
-ADD vc-webhook-manager /vc-webhook-manager
-ADD gen-admission-secret.sh /gen-admission-secret.sh
+COPY --from=builder /go/src/volcano.sh/volcano/_output/bin/vc-webhook-manager /vc-webhook-manager
+ADD ./installer/dockerfile/webhook-manager/gen-admission-secret.sh /gen-admission-secret.sh
 ENTRYPOINT ["/vc-webhook-manager"]