Merge pull request #57 from senthilrch/senthilrch

PR for v0.7.0

.travis.yml

@@ -1,17 +1,54 @@
+dist: bionic
 language: go
 go:
-  - 1.13.8
+  - 1.14.2
+services:
+  - docker
+env:
+  # Enable docker buildx plugin
+  - DOCKER_CLI_EXPERIMENTAL=enabled
+before_install:
+  # Install latest stable version of docker-engine as described in docker documentation:-
+  # https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository
+  - sudo apt-get remove docker docker-engine docker.io containerd runc
+  - sudo apt-get update
+  - sudo apt-get install apt-transport-https ca-certificates curl gnupg-agent software-properties-common -y
+  - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+  - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+  - sudo apt-get install docker-ce docker-ce-cli containerd.io -y
 install:
+  # Install dependencies required for unit test coverage
   - go get golang.org/x/tools/cmd/cover
   - go get github.com/mattn/goveralls
   - go get -u golang.org/x/lint/golint
-script:
-  - hack/verify-codegen.sh
-  - hack/verify-boilerplate.sh
-  - hack/verify-gofmt.sh
-  - hack/verify-golint.sh
-  - hack/verify-govet.sh
-  - make fledged-dev
-  - make GIT_BRANCH=${TRAVIS_BRANCH} build-images
-  - make test
-  - $(go env GOPATH | awk 'BEGIN{FS=":"} {print $1}')/bin/goveralls -coverprofile=coverage.out -service=travis-ci
+stages:
+  # This stage builds fledged controller image, performs tests in "hack" and runs unit tests with coverage      
+  - build_amd64
+  # This stage builds multi-arch images for fledged controller, fledged client and operator (only for PR builds)
+  - name: build_multiarch
+    if: type = pull_request
+  # This stage builds all three multi-arch images and pushes them to docker hub (only for release tags)  
+  - name: build_release
+    if: tag IS present
+jobs:
+  include:
+    - stage: build_amd64
+      script:
+        - hack/verify-codegen.sh
+        - hack/verify-boilerplate.sh
+        - hack/verify-gofmt.sh
+        - hack/verify-golint.sh
+        - hack/verify-govet.sh
+        - make fledged-amd64
+        - make test
+        - $(go env GOPATH | awk 'BEGIN{FS=":"} {print $1}')/bin/goveralls -coverprofile=coverage.out -service=travis-ci
+    - stage: build_multiarch
+      script:
+        # BUILD_OUTPUT is left empty to indicate buildx to leave the built images in it's cache
+        - travis_wait 60 make GIT_BRANCH=${TRAVIS_BRANCH} BUILD_OUTPUT= release 
+    - stage: build_release
+      script:
+        - docker login -u ${DOCKERHUB_USER} -p ${DOCKERHUB_PSWD}
+        # BUILD_OUTPUT=--push requests buildx to push the built images to docker hub
+        - travis_wait 60 make GIT_BRANCH=${TRAVIS_BRANCH} BUILD_OUTPUT=--push release 
+

Makefile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-.PHONY: clean clean-fledged clean-client clean-operator fledged-dev fledged-image client-image operator-image build-images push-images test deploy update remove
+.PHONY: clean clean-fledged clean-client clean-operator fledged-amd64 fledged-image client-image operator-image build-images push-images test deploy update remove
 # Default tag and architecture. Can be overridden
 TAG?=$(shell git describe --tags --dirty)
 ARCH?=amd64
@@ -38,33 +38,45 @@ ifndef OPERATOR_IMAGE_REPO
 endif
 
 ifndef RELEASE_VERSION
-  RELEASE_VERSION=v0.6.0
+  RELEASE_VERSION=v0.7.0
 endif
 
 ifndef DOCKER_VERSION
-  DOCKER_VERSION=19.03.7
+  DOCKER_VERSION=19.03.8
 endif
 
 ifndef CRICTL_VERSION
-  CRICTL_VERSION=v1.17.0
+  CRICTL_VERSION=v1.18.0
 endif
 
 ifndef GOLANG_VERSION
-  GOLANG_VERSION=1.13.8
+  GOLANG_VERSION=1.14.2
 endif
 
 ifndef ALPINE_VERSION
-  ALPINE_VERSION=3.11
+  ALPINE_VERSION=3.11.6
 endif
 
 ifndef OPERATORSDK_VERSION
-  OPERATORSDK_VERSION=v0.15.2
+  OPERATORSDK_VERSION=v0.17.0
 endif
 
 ifndef GIT_BRANCH
   GIT_BRANCH=master
 endif
 
+ifndef TARGET_PLATFORMS
+  TARGET_PLATFORMS=linux/amd64,linux/arm/v7,linux/arm64/v8
+endif
+
+ifndef OPERATOR_TARGET_PLATFORMS
+  OPERATOR_TARGET_PLATFORMS=linux/amd64,linux/arm64
+endif
+
+ifndef BUILD_OUTPUT
+  BUILD_OUTPUT=--push
+endif
+
 HTTP_PROXY_CONFIG=
 ifdef HTTP_PROXY
   HTTP_PROXY_CONFIG=--build-arg http_proxy=${HTTP_PROXY}
@@ -76,67 +88,92 @@ ifdef HTTPS_PROXY
 endif
 
 
-### BUILDING
+### BUILD
 clean: clean-fledged clean-client clean-operator
 
 clean-fledged:
 	-rm -f build/fledged
-	-docker image rm $(FLEDGED_IMAGE_REPO):$(RELEASE_VERSION)
+	-docker image rm ${FLEDGED_IMAGE_REPO}:${RELEASE_VERSION}
 	-docker image rm `docker image ls -f dangling=true -q`
 
 clean-client:
-	-docker image rm $(FLEDGED_DOCKER_CLIENT_IMAGE_REPO):$(RELEASE_VERSION)
+	-docker image rm ${FLEDGED_DOCKER_CLIENT_IMAGE_REPO}:${RELEASE_VERSION}
 	-docker image rm `docker image ls -f dangling=true -q`
 
 clean-operator:
-	-docker image rm $(OPERATOR_IMAGE_REPO):$(RELEASE_VERSION)
+	-docker image rm ${OPERATOR_IMAGE_REPO}:${RELEASE_VERSION}
 	-docker image rm `docker image ls -f dangling=true -q`
 
-fledged-dev: clean-fledged
+fledged-amd64: clean-fledged
 	CGO_ENABLED=0 go build -o build/fledged -ldflags '-s -w -extldflags "-static"' cmd/fledged.go && \
-	cd build && docker build -t $(FLEDGED_IMAGE_REPO):$(RELEASE_VERSION) -f Dockerfile.fledged_dev \
+	cd build && docker build -t ${FLEDGED_IMAGE_REPO}:${RELEASE_VERSION} -f Dockerfile.fledged_dev \
 	--build-arg ALPINE_VERSION=${ALPINE_VERSION} .
 
+fledged-dev: fledged-amd64
+	docker push ${FLEDGED_IMAGE_REPO}:${RELEASE_VERSION}
+
 fledged-image: clean-fledged
-	cd build && docker build -t $(FLEDGED_IMAGE_REPO):$(RELEASE_VERSION) -f Dockerfile.fledged \
-	${HTTP_PROXY_CONFIG} ${HTTPS_PROXY_CONFIG} --build-arg GIT_BRANCH=${GIT_BRANCH} \
-	--build-arg GOLANG_VERSION=${GOLANG_VERSION} --build-arg ALPINE_VERSION=${ALPINE_VERSION} .
+	cd build && docker buildx build --platform=${TARGET_PLATFORMS} -t ${FLEDGED_IMAGE_REPO}:${RELEASE_VERSION} \
+	-f Dockerfile.fledged ${HTTP_PROXY_CONFIG} ${HTTPS_PROXY_CONFIG} --build-arg GIT_BRANCH=${GIT_BRANCH} \
+	--build-arg GOLANG_VERSION=${GOLANG_VERSION} --build-arg ALPINE_VERSION=${ALPINE_VERSION} --progress=plain ${BUILD_OUTPUT} .
 
 client-image: clean-client
-	cd build && docker build -t $(FLEDGED_DOCKER_CLIENT_IMAGE_REPO):$(RELEASE_VERSION) \
+	cd build && docker buildx build --platform=${TARGET_PLATFORMS} -t ${FLEDGED_DOCKER_CLIENT_IMAGE_REPO}:${RELEASE_VERSION} \
 	-f Dockerfile.docker_client  ${HTTP_PROXY_CONFIG} ${HTTPS_PROXY_CONFIG} \
 	--build-arg DOCKER_VERSION=${DOCKER_VERSION} --build-arg CRICTL_VERSION=${CRICTL_VERSION} \
-	--build-arg ALPINE_VERSION=${ALPINE_VERSION} .
+	--build-arg ALPINE_VERSION=${ALPINE_VERSION} --progress=plain ${BUILD_OUTPUT} .
 
 operator-image: clean-operator
 	cd deploy/kubefledged-operator && \
-	docker build -t $(OPERATOR_IMAGE_REPO):$(RELEASE_VERSION) -f ./build/Dockerfile \
-	--build-arg OPERATORSDK_VERSION=${OPERATORSDK_VERSION} .
-
-build-images: fledged-image client-image operator-image
+	docker buildx build --platform=${OPERATOR_TARGET_PLATFORMS} -t ${OPERATOR_IMAGE_REPO}:${RELEASE_VERSION} \
+	-f ./build/Dockerfile --build-arg OPERATORSDK_VERSION=${OPERATORSDK_VERSION} --progress=plain ${BUILD_OUTPUT} .
 
 push-images:
-	-docker push $(FLEDGED_IMAGE_REPO):$(RELEASE_VERSION)
-	-docker push $(FLEDGED_DOCKER_CLIENT_IMAGE_REPO):$(RELEASE_VERSION)
-	-docker push $(OPERATOR_IMAGE_REPO):$(RELEASE_VERSION)
+	-docker push ${FLEDGED_IMAGE_REPO}:${RELEASE_VERSION}
+	-docker push ${FLEDGED_DOCKER_CLIENT_IMAGE_REPO}:${RELEASE_VERSION}
+	-docker push ${OPERATOR_IMAGE_REPO}:${RELEASE_VERSION}
 
-release: build-images push-images
+release: install-buildx fledged-image client-image operator-image
+
+install-buildx:
+	docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+	-docker buildx rm multibuilder
+	docker buildx create --name multibuilder --driver docker-container --use
+	docker buildx inspect --bootstrap
+	docker buildx ls
 
 test:
 	-rm -f coverage.out
 	bash hack/run-unit-tests.sh
 
-deploy:
-	kubectl apply -f deploy/kubefledged-crd.yaml && sleep 2 && \
-	kubectl apply -f deploy/kubefledged-namespace.yaml && sleep 2 && \
+deploy-using-yaml:
+	kubectl apply -f deploy/kubefledged-crd.yaml && \
+	kubectl apply -f deploy/kubefledged-namespace.yaml && \
 	kubectl apply -f deploy/kubefledged-serviceaccount.yaml && \
 	kubectl apply -f deploy/kubefledged-clusterrole.yaml && \
 	kubectl apply -f deploy/kubefledged-clusterrolebinding.yaml && \
 	kubectl apply -f deploy/kubefledged-deployment.yaml
 
+deploy-using-operator:
+	# Deploy the operator to a separate namespace called "operators"
+	sed -i "s|OPERATOR_NAMESPACE|operators|g" deploy/kubefledged-operator/deploy/service_account.yaml
+	sed -i "s|OPERATOR_NAMESPACE|operators|g" deploy/kubefledged-operator/deploy/clusterrole_binding.yaml
+	sed -i "s|OPERATOR_NAMESPACE|operators|g" deploy/kubefledged-operator/deploy/operator.yaml
+	-kubectl create namespace operators
+	kubectl create -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_kubefledgeds_crd.yaml
+	kubectl create -f deploy/kubefledged-operator/deploy/service_account.yaml
+	kubectl create -f deploy/kubefledged-operator/deploy/clusterrole.yaml
+	kubectl create -f deploy/kubefledged-operator/deploy/clusterrole_binding.yaml
+	kubectl create -f deploy/kubefledged-operator/deploy/operator.yaml
+	# Deploy kube-fledged to a separate namespace called "kube-fledged"
+	sed -i "s|OPERATOR_NAMESPACE|operators|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml
+	sed -i "s|KUBEFLEDGED_NAMESPACE|kube-fledged|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml
+	-kubectl create namespace kube-fledged
+	kubectl create -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml
+
 update:
-	kubectl scale deployment kubefledged --replicas=0 -n kube-fledged && sleep 5 && \
-	kubectl scale deployment kubefledged --replicas=1 -n kube-fledged && sleep 5 && \
+	kubectl scale deployment kubefledged --replicas=0 -n kube-fledged && sleep 1 && \
+	kubectl scale deployment kubefledged --replicas=1 -n kube-fledged && sleep 1 && \
 	kubectl get pods -l app=kubefledged -n kube-fledged
 
 remove:
@@ -145,3 +182,20 @@ remove:
 	kubectl delete -f deploy/kubefledged-clusterrole.yaml && \
 	kubectl delete -f deploy/kubefledged-crd.yaml
 
+remove-all:
+	# Remove kube-fledged and the namespace "kube-fledged"
+	kubectl delete -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml
+	-kubectl delete namespace kube-fledged
+	sed -i "s|kube-fledged|KUBEFLEDGED_NAMESPACE|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml
+	sed -i "s|operators|OPERATOR_NAMESPACE|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml
+	# Remove the operator and the namespace "operators"
+	kubectl delete -f deploy/kubefledged-operator/deploy/operator.yaml
+	kubectl delete -f deploy/kubefledged-operator/deploy/clusterrole_binding.yaml
+	kubectl delete -f deploy/kubefledged-operator/deploy/clusterrole.yaml
+	kubectl delete -f deploy/kubefledged-operator/deploy/service_account.yaml
+	kubectl delete -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_kubefledgeds_crd.yaml
+	-kubectl delete namespace operators
+	sed -i "s|operators|OPERATOR_NAMESPACE|g" deploy/kubefledged-operator/deploy/operator.yaml
+	sed -i "s|operators|OPERATOR_NAMESPACE|g" deploy/kubefledged-operator/deploy/clusterrole_binding.yaml
+	sed -i "s|operators|OPERATOR_NAMESPACE|g" deploy/kubefledged-operator/deploy/service_account.yaml
+

README.md

@@ -93,27 +93,10 @@ These instructions install _kube-fledged_ to a separate namespace called "kube-f
   $ cd $HOME/src/github.com/senthilrch/kube-fledged
   ```
 
-- Deploy the operator to a separate namespace called "operators"
+- Deploy the operator to a separate namespace called "operators" and _kube-fledged_ to a separate namespace called "kube-fledged"
 
   ```
-  $ sed -i "s|OPERATOR_NAMESPACE|operators|g" deploy/kubefledged-operator/deploy/service_account.yaml
-  $ sed -i "s|OPERATOR_NAMESPACE|operators|g" deploy/kubefledged-operator/deploy/clusterrole_binding.yaml
-  $ sed -i "s|OPERATOR_NAMESPACE|operators|g" deploy/kubefledged-operator/deploy/operator.yaml
-  $ kubectl create namespace operators
-  $ kubectl create -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_kubefledgeds_crd.yaml
-  $ kubectl create -f deploy/kubefledged-operator/deploy/service_account.yaml
-  $ kubectl create -f deploy/kubefledged-operator/deploy/clusterrole.yaml
-  $ kubectl create -f deploy/kubefledged-operator/deploy/clusterrole_binding.yaml
-  $ kubectl create -f deploy/kubefledged-operator/deploy/operator.yaml
-  ```
-
-- Deploy _kube-fledged_ to a separate namespace called "kube-fledged"
-
-  ```
-  $ sed -i "s|OPERATOR_NAMESPACE|operators|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml
-  $ sed -i "s|KUBEFLEDGED_NAMESPACE|kube-fledged|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml
-  $ kubectl create namespace kube-fledged
-  $ kubectl create -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml
+  $ make deploy-using-operator
   ```
 
 - Verify if _kube-fledged_ deployed successfully
@@ -259,7 +242,8 @@ $ kubectl delete imagecaches imagecache1 -n kube-fledged
 Run the following command to remove _kube-fledged_ from the cluster. 
 
 ```
-$ make remove
+$ make remove (if you deployed using YAML manifests)
+$ make remove-all (if you deployed using Helm Operator)
 ```
 
 ## How it works
@@ -277,20 +261,26 @@ For more detailed description, go through _kube-fledged's_ [design proposal](doc
 
 `--image-cache-refresh-frequency:` The image cache is refreshed periodically to ensure the cache is up to date. Setting this flag to "0s" will disable refresh. default "15m"
 
-`--docker-client-image:` The image name of the docker client. the docker client is used when deleting images during purging the cache".
+`--docker-client-image:` The image name of the docker client. The docker client is used when deleting images during purging the cache".
 
-`--image-pull-policy:` Image pull policy for pulling images into the cache. Possible values are 'IfNotPresent' and 'Always'. Default value is 'IfNotPresent'. Default value for Images with ':latest' tag is 'Always'
+`--image-pull-policy:` Image pull policy for pulling images into and refreshing the cache. Possible values are 'IfNotPresent' and 'Always'. Default value is 'IfNotPresent'. Image with no or ":latest" tag are always pulled.
 
 `--stderrthreshold:` Log level. set the value of this flag to INFO
 
 ## Supported Platforms
 
 - linux/amd64
+- linux/arm
+- linux/arm64
 
 
 ## Built With
 
 * [kubernetes/sample-controller](https://github.com/kubernetes/sample-controller) - Building our own kubernetes-style controller using CRD.
+* [kubebuilder](https://github.com/kubernetes-sigs/kubebuilder) - SDK for building Kubernetes APIs using CRDs
+* [operator-sdk](https://github.com/operator-framework/operator-sdk) - SDK for building Kubernetes applications
+* [cri-tools](https://github.com/kubernetes-sigs/cri-tools) - CLI and validation tools for Kubelet Container Runtime Interface (CRI).
+* [buildx](https://github.com/docker/buildx) - Docker CLI plugin for extended build capabilities with BuildKit 
 * [Go Modules](https://golang.org/doc/go1.11#modules) - Go Modules for Dependency Management
 * [Make](https://www.gnu.org/software/make/) - GNU Make
 

cmd/app/controller.go

@@ -194,6 +194,7 @@ func (c *Controller) danglingImageCaches() error {
 	}
 	for _, imagecache := range imagecachelist.Items {
 		if imagecache.Status.Status == fledgedv1alpha1.ImageCacheActionStatusProcessing {
+			status.StartTime = imagecache.Status.StartTime
 			err := c.updateImageCacheStatus(&imagecache, status)
 			if err != nil {
 				glog.Errorf("Error updating ImageCache(%s) status to '%s': %v", imagecache.Name, fledgedv1alpha1.ImageCacheActionStatusAborted, err)
@@ -567,7 +568,7 @@ func (c *Controller) syncHandler(wqKey images.WorkQueueKey) error {
 				for m := range i.Images {
 					ipr := images.ImageWorkRequest{
 						Image:                   i.Images[m],
-						Node:                    n.Labels["kubernetes.io/hostname"],
+						Node:                    n,
 						ContainerRuntimeVersion: n.Status.NodeInfo.ContainerRuntimeVersion,
 						WorkType:                wqKey.WorkType,
 						Imagecache:              imageCache,
@@ -586,7 +587,7 @@ func (c *Controller) syncHandler(wqKey images.WorkQueueKey) error {
 						if !matched {
 							ipr := images.ImageWorkRequest{
 								Image:                   oldimage,
-								Node:                    n.Labels["kubernetes.io/hostname"],
+								Node:                    n,
 								ContainerRuntimeVersion: n.Status.NodeInfo.ContainerRuntimeVersion,
 								WorkType:                images.ImageCachePurge,
 								Imagecache:              imageCache,
@@ -603,7 +604,7 @@ func (c *Controller) syncHandler(wqKey images.WorkQueueKey) error {
 		c.imageworkqueue.AddRateLimited(images.ImageWorkRequest{WorkType: wqKey.WorkType, Imagecache: imageCache})
 
 	case images.ImageCacheStatusUpdate:
-		glog.Infof("wqKey.Status = %+v", wqKey.Status)
+		glog.V(4).Infof("wqKey.Status = %+v", wqKey.Status)
 		// Finally, we update the status block of the ImageCache resource to reflect the
 		// current state of the world
 		// Get the ImageCache resource with this namespace/name
@@ -621,7 +622,7 @@ func (c *Controller) syncHandler(wqKey images.WorkQueueKey) error {
 
 		failures := false
 		for _, v := range *wqKey.Status {
-			if v.Status == images.ImageWorkResultStatusSucceeded && !failures {
+			if (v.Status == images.ImageWorkResultStatusSucceeded || v.Status == images.ImageWorkResultStatusAlreadyPulled) && !failures {
 				status.Status = fledgedv1alpha1.ImageCacheActionStatusSucceeded
 				if v.ImageWorkRequest.WorkType == images.ImageCachePurge {
 					status.Message = fledgedv1alpha1.ImageCacheMessageImagesDeletedSuccessfully
@@ -641,7 +642,7 @@ func (c *Controller) syncHandler(wqKey images.WorkQueueKey) error {
 			if v.Status == images.ImageWorkResultStatusFailed {
 				status.Failures[v.ImageWorkRequest.Image] = append(
 					status.Failures[v.ImageWorkRequest.Image], fledgedv1alpha1.NodeReasonMessage{
-						Node:    v.ImageWorkRequest.Node,
+						Node:    v.ImageWorkRequest.Node.Labels["kubernetes.io/hostname"],
 						Reason:  v.Reason,
 						Message: v.Message,
 					})