make: Build KF images in parallel

Currently there is a single build-all rule in the top-level Makefile
under components/ dir that calls all sub-makefiles for building all
images in a serial manner. This makes the whole proccess very time-consuming
as we can't use the "-j" option of make to run jobs in parallel.

Changes to the top-level Makefile for building all KF images:

- create a single rule for each directory containing a sub-makefile

- have each directory rule as a dependency to the build-all rule in the
  central Makefile

Similarly, the central makefile for building all the example-notebook-servers images
calls each sub-Makefile for each of the notebook servers in a serial manner.

Changes to the central Makefile for building all notebook-server-images:

- Split the single target rule into multiple rules which perform recursive make
  calls (sub-makefiles) to build all the notebook-server-images.

- Use the variable MAKE for recursive make commands instead of explicit 'make'
  command:
  - https://www.gnu.org/software/make/manual/html_node/MAKE-Variable.html#MAKE-Variable

Signed-off-by: Apostolos Gerakaris <apoger@arrikto.com>

components/Makefile

@@ -1,46 +1,25 @@
 REGISTRY ?= kubeflownotebookswg
 TAG ?= $(shell git describe --tags --always --dirty)
 
-build-all:
-	@echo "\nBuilding KFAM image...\n"
-	make docker-build -C access-management 
-
-	@echo "\nBuilding Admission-webhook image...\n"
-	make docker-build -C admission-webhook
-
-	@echo "\nBuilding Centraldashboard image...\n"
-	make docker-build -C centraldashboard
-
-	@echo "\nBuilding Jupyter-Web-App image...\n"
-	make docker-build -C crud-web-apps/jupyter
-
-	@echo "\nBuilding Tensorboard-Web-App image...\n"
-	make docker-build -C crud-web-apps/tensorboards
-
-	@echo "\nBuilding Volumes-Web-App image...\n"
-	make docker-build -C crud-web-apps/volumes
-
-	@echo "\nBuilding Notebook-controller image...\n"
-	make docker-build -C notebook-controller
-
-	@echo "\nBuilding Profile-controller image...\n"
-	make docker-build -C profile-controller
-
-	@echo "\nBuilding tensorboard-controller image...\n"
-	make docker-build -C tensorboard-controller
-
-	@echo "\nBuilding notebook-server images...\n"
-	make docker-build-all -C example-notebook-servers
+SUBDIRS := access-management admission-webhook centraldashboard crud-web-apps/jupyter crud-web-apps/tensorboards \
+			crud-web-apps/volumes notebook-controller profile-controller tensorboard-controller example-notebook-servers
 
+.PHONY: build-all
+build-all: $(SUBDIRS)
 	@echo "\nAll images of prv-kubeflow-kubeflow repo have been built successfully!\n"
 
-push-all:	
-	for img in kfam admission-webhook centraldashboard jupyter-web-app tensorboards-web-app volumes-web-app notebook-controller profile-controller \
+.PHONY: $(SUBDIRS)
+$(SUBDIRS):
+	$(MAKE) docker-build -C $@ TAG=${TAG} REGISTRY=${REGISTRY}
+
+.PHONY: push-all
+push-all:
+	for img in kfam poddefaults-webhook centraldashboard jupyter-web-app tensorboards-web-app volumes-web-app notebook-controller profile-controller \
 	tensorboard-controller ; do \
 		docker tag $$img:$(TAG) ${REGISTRY}/$$img:${TAG} ; \
 		docker push ${REGISTRY}/$$img:${TAG} ; \
 	done
 
-	make docker-push-all -C example-notebook-servers TAG=${TAG} REGISTRY=${REGISTRY}
+	$(MAKE) docker-push -C example-notebook-servers TAG=${TAG} REGISTRY=${REGISTRY}
 
-	@echo "\nAll images have been succesfully retaged and pushed with registry: ${REGISTRY} and tag: ${TAG}...\n"
+	@echo "\nAll images have been succesfully retaged and pushed with registry: ${REGISTRY} and tag: ${TAG}...\n"

components/example-notebook-servers/Makefile

@@ -1,98 +1,97 @@
 TAG ?= $(shell git describe --tags --always --dirty)
 REGISTRY ?= kubeflownotebookswg
 
-docker-build-all:
-	@echo "\nBuilding base image...\n"
-	make docker-build -C base
 
-	@echo "\nBuilding codeserver image...\n"
-	make docker-build -C codeserver
+.PHONY: docker-build
+docker-build: build-jupyter-pytorch-full build-jupyter-tensorflow-full build-jupyter-scipy build-codeserver-python build-rstudio-tidyverse
+		@echo "\nAll notebook-server images have been successfully built...\n"
 
-	@echo "\nBuilding codeserver-python image...\n"
-	make docker-build -C codeserver-python
-
-	@echo "\nBuilding rstudio image...\n"
-	make docker-build -C rstudio
-
-	@echo "\nBuilding rstudio-tidyverse image...\n"
-	make docker-build -C rstudio-tidyverse
-
-	@echo "\nBuilding jupyter image...\n"
-	make docker-build -C jupyter 
+.PHONY: build-jupyter-pytorch-full
+build-jupyter-pytorch-full: build-jupyter-pytorch-full-cpu build-jupyter-pytorch-full-cuda
 
-	@echo "\nBuilding jupyter-scipy image...\n"
-	make docker-build -C jupyter-scipy
+.PHONY: build-jupyter-pytorch-full-cpu
+build-jupyter-pytorch-full-cpu:
+	@echo "\nBuilding jupyter-pytorch-full-cpu image...\n"
+	$(MAKE) docker-build-cpu -C jupyter-pytorch-full TAG=${TAG}
 
-	@echo "\nBuilding jupyter-pytorch image...\n"
-	make docker-build-cpu -C jupyter-pytorch
+.PHONY: build-jupyter-pytorch-full-cuda
+build-jupyter-pytorch-full-cuda:
+	@echo "\nBuilding jupyter-pytorch-full-cuda image...\n"
+	$(MAKE) docker-build-cuda -C jupyter-pytorch-full TAG=${TAG}
 
-	@echo "\nBuilding jupyter-pytorch-cuda image...\n"
-	make docker-build-cuda -C jupyter-pytorch
+.PHONY: build-jupyter-tensorflow-full
+build-jupyter-tensorflow-full: build-jupyter-tensorflow-full-cpu build-jupyter-tensorflow-full-cuda
 
-	@echo "\nBuilding jupyter-pytorch-full image...\n"
-	make docker-build-cpu -C jupyter-pytorch-full
+.PHONY: jupyter-tensorflow-full-cpu
+build-jupyter-tensorflow-full-cpu:
+	@echo "\nBuilding jupyter-tensorflow-full-cpu image...\n"
+	$(MAKE) docker-build-cpu -C jupyter-tensorflow-full TAG=${TAG}
 
-	@echo "\nBuilding jupyter-pytorch-cuda-full image...\n"
-	make docker-build-cuda -C jupyter-pytorch-full
+.PHONY: build-jupyter-tensorflow-full-cuda
+build-jupyter-tensorflow-full-cuda:
+	@echo "\nBuilding jupyter-tensorflow-full-cuda image...\n"
+	$(MAKE) docker-build-cuda -C jupyter-tensorflow-full TAG=${TAG}
 
-	@echo "\nBuilding jupyter-tensorflow image...\n"
-	make docker-build-cpu -C jupyter-tensorflow
-
-	@echo "\nBuilding jupyter-tensorflow-cuda image...\n"
-	make docker-build-cuda -C jupyter-tensorflow
-
-	@echo "\nBuilding jupyter-tensorflow-full image...\n"
-	make docker-build-cpu -C jupyter-tensorflow-full
+.PHONY: build-jupyter-scipy
+build-jupyter-scipy:
+	@echo "\nBuilding jupyter-scipy image...\n"
+	$(MAKE) docker-build -C jupyter-scipy TAG=${TAG}
 
-	@echo "\nBuilding jupyter-tensorflow-cuda-full image...\n"
-	make docker-build-cuda -C jupyter-tensorflow-full
+.PHONY: build-codeserver-python
+build-codeserver-python:
+	@echo "\nBuilding codeserver-python image...\n"
+	$(MAKE) docker-build -C codeserver-python TAG=${TAG}
 
-	@echo "\nAll notebook-server images have been successfully built...\n"
+.PHONY: build-rstudio-tidyverse
+build-rstudio-tidyverse:
+	@echo "\nBuilding rstudio-tidyverse image...\n"
+	$(MAKE) docker-build -C rstudio-tidyverse TAG=${TAG}
 
-docker-push-all:
+.PHONY: docker-push
+docker-push:
 	@echo "\nPushing base image...\n"
-	make docker-push -C base
+	$(MAKE) docker-push -C base
 
 	@echo "\nPushing codeserver image...\n"
-	make docker-push -C codeserver
+	$(MAKE) docker-push -C codeserver
 
 	@echo "\nPushing codeserver-python image...\n"
-	make docker-push -C codeserver-python 
+	$(MAKE) docker-push -C codeserver-python 
 
 	@echo "\nPushing rstudio image...\n"
-	make docker-push -C rstudio
+	$(MAKE) docker-push -C rstudio
 
 	@echo "\nPushing rstudio-tidyverse image...\n"
-	make docker-push -C rstudio-tidyverse
+	$(MAKE) docker-push -C rstudio-tidyverse
 
 	@echo "\nPushing jupyter image...\n"
-	make docker-push -C jupyter
+	$(MAKE) docker-push -C jupyter
 
 	@echo "\nPushing jupyter-scipy image...\n"
-	make docker-push -C jupyter-scipy
+	$(MAKE) docker-push -C jupyter-scipy
 
 	@echo "\nPushing jupyter-pytorch image...\n"
-	make docker-push-cpu -C jupyter-pytorch
+	$(MAKE) docker-push-cpu -C jupyter-pytorch
 
 	@echo "\nPushing jupyter-pytorch-cuda image...\n"
-	make docker-push-cuda -C jupyter-pytorch
+	$(MAKE) docker-push-cuda -C jupyter-pytorch
 
 	@echo "\nPushing jupyter-pytorch-full image...\n"
-	make docker-push-cpu -C jupyter-pytorch-full
+	$(MAKE) docker-push-cpu -C jupyter-pytorch-full
 
 	@echo "\nPushing jupyter-pytorch-cuda-full image...\n"
-	make docker-push-cuda -C jupyter-pytorch-full
+	$(MAKE) docker-push-cuda -C jupyter-pytorch-full
 
 	@echo "\nPushing jupyter-tensorflow image...\n"
-	make docker-push-cpu -C jupyter-tensorflow
+	$(MAKE) docker-push-cpu -C jupyter-tensorflow
 
 	@echo "\nPushing jupyter-tensorflow-cuda image...\n"
-	make docker-push-cuda -C jupyter-tensorflow
+	$(MAKE) docker-push-cuda -C jupyter-tensorflow
 
 	@echo "\nPushing jupyter-tensorflow-full image...\n"
-	make docker-push-cpu -C jupyter-tensorflow-full
+	$(MAKE) docker-push-cpu -C jupyter-tensorflow-full
 
 	@echo "\nPushing jupyter-tensorflow-cuda-full image...\n"
-	make docker-push-cuda -C jupyter-tensorflow-full
+	$(MAKE) docker-push-cuda -C jupyter-tensorflow-full
 
 	@echo "\nAll notebook-server images have been successfully pushed...\n"

components/example-notebook-servers/codeserver-python/Makefile

@@ -2,7 +2,7 @@ TAG ?= $(shell git describe --tags --always --dirty)
 REGISTRY ?= kubeflownotebookswg
 
 docker-build-codeserver:
-	make docker-build -C ../codeserver TAG=${TAG}
+	$(MAKE) docker-build -C ../codeserver TAG=${TAG}
 
 docker-build: docker-build-codeserver
 	docker build -t ${REGISTRY}/codeserver-python:${TAG} --build-arg BASE_IMG=${REGISTRY}/codeserver:${TAG} -f Dockerfile . 

components/example-notebook-servers/codeserver/Makefile

@@ -2,7 +2,7 @@ TAG ?= $(shell git describe --tags --always --dirty)
 REGISTRY ?= kubeflownotebookswg
 
 docker-build-base:
-	make docker-build -C ../base TAG=${TAG}
+	$(MAKE) docker-build -C ../base TAG=${TAG}
 
 docker-build: docker-build-base
 	docker build -t ${REGISTRY}/codeserver:${TAG} --build-arg BASE_IMG=${REGISTRY}/base:${TAG} -f Dockerfile . 

components/example-notebook-servers/jupyter-pytorch-full/Makefile

@@ -2,10 +2,10 @@ TAG ?= $(shell git describe --tags --always --dirty)
 REGISTRY ?= kubeflownotebookswg
 
 docker-build-jupyter-pytorch:
-	make docker-build-cpu -C ../jupyter-pytorch TAG=${TAG} 
+	$(MAKE) docker-build-cpu -C ../jupyter-pytorch TAG=${TAG} 
 
 docker-build-jupyter-pytorch-cuda:
-	make docker-build-cuda -C ../jupyter-pytorch TAG=${TAG} 
+	$(MAKE) docker-build-cuda -C ../jupyter-pytorch TAG=${TAG} 
 
 docker-build-cpu: docker-build-jupyter-pytorch
 	docker build -t ${REGISTRY}/jupyter-pytorch-full:${TAG} --build-arg BASE_IMG=${REGISTRY}/jupyter-pytorch:${TAG} -f cpu.Dockerfile . 

components/example-notebook-servers/jupyter-pytorch/Makefile

@@ -2,7 +2,7 @@ TAG ?= $(shell git describe --tags --always --dirty)
 REGISTRY ?= kubeflownotebookswg
 
 docker-build-jupyter:
-	make docker-build -C ../jupyter TAG=${TAG}
+	$(MAKE) docker-build -C ../jupyter TAG=${TAG}
 
 docker-build-cpu: docker-build-jupyter
 	docker build -t ${REGISTRY}/jupyter-pytorch:${TAG} --build-arg BASE_IMG=${REGISTRY}/jupyter:${TAG} -f cpu.Dockerfile . 

components/example-notebook-servers/jupyter-scipy/Makefile

@@ -2,7 +2,7 @@ TAG ?= $(shell git describe --tags --always --dirty)
 REGISTRY ?= kubeflownotebookswg
 
 docker-build-jupyter:
-	make docker-build -C ../jupyter TAG=${TAG} 
+	$(MAKE) docker-build -C ../jupyter TAG=${TAG} 
 
 docker-build: docker-build-jupyter
 	docker build -t ${REGISTRY}/jupyter-scipy:${TAG} --build-arg BASE_IMG=${REGISTRY}/jupyter:${TAG} -f Dockerfile . 

components/example-notebook-servers/jupyter-tensorflow-full/Makefile

@@ -2,10 +2,10 @@ TAG ?= $(shell git describe --tags --always --dirty)
 REGISTRY ?= kubeflownotebookswg
 
 docker-build-jupyter-tensorflow:
-	make docker-build-cpu -C ../jupyter-tensorflow TAG=${TAG}
+	$(MAKE) docker-build-cpu -C ../jupyter-tensorflow TAG=${TAG}
 
 docker-build-jupyter-tensorflow-cuda:
-	make docker-build-cuda -C ../jupyter-tensorflow TAG=${TAG}
+	$(MAKE) docker-build-cuda -C ../jupyter-tensorflow TAG=${TAG}
 
 docker-build-cpu: docker-build-jupyter-tensorflow
 	docker build -t ${REGISTRY}/jupyter-tensorflow-full:${TAG} --build-arg BASE_IMG=${REGISTRY}/jupyter-tensorflow:${TAG} -f cpu.Dockerfile . 

components/example-notebook-servers/jupyter-tensorflow/Makefile

@@ -2,7 +2,7 @@ TAG ?= $(shell git describe --tags --always --dirty)
 REGISTRY ?= kubeflownotebookswg
 
 docker-build-jupyter:
-	make docker-build -C ../jupyter TAG=${TAG} 
+	$(MAKE) docker-build -C ../jupyter TAG=${TAG} 
 
 docker-build-cpu: docker-build-jupyter
 	docker build -t ${REGISTRY}/jupyter-tensorflow:${TAG} --build-arg BASE_IMG=${REGISTRY}/jupyter:${TAG} -f cpu.Dockerfile . 

components/example-notebook-servers/jupyter/Makefile

@@ -2,7 +2,7 @@ TAG ?= $(shell git describe --tags --always --dirty)
 REGISTRY ?= kubeflownotebookswg
 
 docker-build-base:
-	make docker-build -C ../base TAG=${TAG}
+	$(MAKE) docker-build -C ../base TAG=${TAG}
 
 docker-build: docker-build-base
 	docker build -t ${REGISTRY}/jupyter:${TAG} --build-arg BASE_IMG=${REGISTRY}/base:${TAG} -f Dockerfile . 

components/example-notebook-servers/rstudio-tidyverse/Makefile

@@ -2,7 +2,7 @@ TAG ?= $(shell git describe --tags --always --dirty)
 REGISTRY ?= kubeflownotebookswg
 
 docker-build-rstudio:
-	make docker-build -C ../rstudio TAG=${TAG} 
+	$(MAKE) docker-build -C ../rstudio TAG=${TAG} 
 
 docker-build: docker-build-rstudio
 	docker build -t ${REGISTRY}/rstudio-tidyverse:${TAG} --build-arg BASE_IMG=${REGISTRY}/rstudio:${TAG} -f Dockerfile . 

components/example-notebook-servers/rstudio/Makefile

@@ -2,7 +2,7 @@ TAG ?= $(shell git describe --tags --always --dirty)
 REGISTRY ?= kubeflownotebookswg
 
 docker-build-base:
-	make docker-build -C ../base TAG=${TAG}
+	$(MAKE) docker-build -C ../base TAG=${TAG}
 
 docker-build: docker-build-base
 	docker build -t ${REGISTRY}/rstudio:${TAG} --build-arg BASE_IMG=${REGISTRY}/base:${TAG} -f Dockerfile .