Scripts -> makefile (#265)

* replace docker-build.sh and build-imgs.sh with makefile

* support serving image in makefile

* don't recursive

* fix

* working

* more

* fixes

* echo when pushing

* one central PHONY

* Remove unnecessary ifs, tag image with un dotted python version

* Add make help

* tags instead of spaces

* newline

* 3 -> 2 python versions

* fixed dockerfiles args name remove redundant env definition

* fixed README

Dockerfile

@@ -12,9 +12,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-ARG PYTHON_VER=3.6
+ARG MLRUN_PYTHON_VERSION=3.6
 
-FROM python:${PYTHON_VER}
+FROM python:${MLRUN_PYTHON_VERSION}
 
 WORKDIR /mlrun
 

Makefile

@@ -12,62 +12,213 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-.PHONY: all
+MLRUN_DOCKER_TAG ?= latest
+MLRUN_DOCKER_REPO ?= mlrun
+MLRUN_DOCKER_REGISTRY ?=  # empty be default (dockerhub), can be set to something like "quay.io/"
+MLRUN_ML_DOCKER_IMAGE_NAME_PREFIX ?= ml-
+MLRUN_PACKAGE_TAG ?= development
+MLRUN_GITHUB_REPO ?= mlrun
+MLRUN_PYTHON_VERSION ?= 3.7
+MLRUN_LEGACY_ML_PYTHON_VERSION ?= 3.6
+
+
+
+MLRUN_DOCKER_IMAGE_PREFIX := $(if $(MLRUN_DOCKER_REGISTRY),$(strip $(MLRUN_DOCKER_REGISTRY))$(MLRUN_DOCKER_REPO),$(MLRUN_DOCKER_REPO))
+MLRUN_LEGACY_DOCKER_TAG_SUFFIX := -py$(subst .,,$(MLRUN_LEGACY_ML_PYTHON_VERSION))
+MLRUN_LEGACY_DOCKERFILE_DIR_NAME := py$(subst .,,$(MLRUN_LEGACY_ML_PYTHON_VERSION))
+
+
+help: ## Display available commands
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+
 all:
 	$(error please pick a target)
 
-.PHONY: upload
-upload: wheel
-	python -m twine upload dist/mlrun-*.whl
 
-.PHONY: wheel
-wheel: clean
+build: docker-images package-wheel ## Build all artifacts
+	@echo Done.
+
+
+DOCKER_IMAGES_RULES = \
+	api \
+	base \
+	models \
+	models-gpu \
+	mlrun
+
+docker-images: $(DOCKER_IMAGES_RULES) ## Build all docker images
+	@echo Done.
+
+
+push-docker-images: docker-images ## Push all docker images
+	@for image in $(IMAGES_TO_PUSH); do \
+		echo "Pushing $$image" ; \
+		docker push $$image ; \
+	done
+	@echo Done.
+
+
+print-docker-images: ## Print all docker images
+	@for image in $(IMAGES_TO_PUSH); do \
+		echo $$image ; \
+	done
+
+
+MLRUN_BASE_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/$(MLRUN_ML_DOCKER_IMAGE_NAME_PREFIX)base:$(MLRUN_DOCKER_TAG)
+MLRUN_LEGACY_BASE_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/$(MLRUN_ML_DOCKER_IMAGE_NAME_PREFIX)base:$(MLRUN_DOCKER_TAG)$(MLRUN_LEGACY_DOCKER_TAG_SUFFIX)
+
+base: ## Build base and legacy-base docker images
+	docker build \
+		--file dockerfiles/base/Dockerfile \
+		--build-arg MLRUN_PYTHON_VERSION=$(MLRUN_PYTHON_VERSION) \
+		--build-arg MLRUN_PACKAGE_TAG=$(MLRUN_PACKAGE_TAG) \
+		--build-arg MLRUN_GITHUB_REPO=$(MLRUN_GITHUB_REPO) \
+		--tag $(MLRUN_BASE_IMAGE_NAME) .
+
+	docker build \
+		--file dockerfiles/base/$(MLRUN_LEGACY_DOCKERFILE_DIR_NAME)/Dockerfile \
+		--build-arg MLRUN_PYTHON_VERSION=$(MLRUN_LEGACY_ML_PYTHON_VERSION) \
+		--build-arg MLRUN_PACKAGE_TAG=$(MLRUN_PACKAGE_TAG) \
+		--build-arg MLRUN_GITHUB_REPO=$(MLRUN_GITHUB_REPO) \
+		--tag $(MLRUN_LEGACY_BASE_IMAGE_NAME) .
+
+IMAGES_TO_PUSH += $(MLRUN_BASE_IMAGE_NAME)
+IMAGES_TO_PUSH += $(MLRUN_LEGACY_BASE_IMAGE_NAME)
+
+
+MLRUN_MODELS_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/$(MLRUN_ML_DOCKER_IMAGE_NAME_PREFIX)models:$(MLRUN_DOCKER_TAG)
+MLRUN_LEGACY_MODELS_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/$(MLRUN_ML_DOCKER_IMAGE_NAME_PREFIX)models:$(MLRUN_DOCKER_TAG)$(MLRUN_LEGACY_DOCKER_TAG_SUFFIX)
+
+models: ## Build models and legacy-models docker images
+	docker build \
+		--file dockerfiles/models/Dockerfile \
+		--build-arg MLRUN_PACKAGE_TAG=$(MLRUN_PACKAGE_TAG) \
+		--build-arg MLRUN_GITHUB_REPO=$(MLRUN_GITHUB_REPO) \
+		--tag $(MLRUN_MODELS_IMAGE_NAME) .
+
+	docker build \
+		--file dockerfiles/models/$(MLRUN_LEGACY_DOCKERFILE_DIR_NAME)/Dockerfile \
+		--build-arg MLRUN_PACKAGE_TAG=$(MLRUN_PACKAGE_TAG) \
+		--build-arg MLRUN_GITHUB_REPO=$(MLRUN_GITHUB_REPO) \
+		--tag $(MLRUN_LEGACY_MODELS_IMAGE_NAME) .
+
+IMAGES_TO_PUSH += $(MLRUN_MODELS_IMAGE_NAME)
+IMAGES_TO_PUSH += $(MLRUN_LEGACY_MODELS_IMAGE_NAME)
+
+
+MLRUN_MODELS_GPU_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/$(MLRUN_ML_DOCKER_IMAGE_NAME_PREFIX)models-gpu:$(MLRUN_DOCKER_TAG)
+MLRUN_LEGACY_MODELS_GPU_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/$(MLRUN_ML_DOCKER_IMAGE_NAME_PREFIX)models-gpu:$(MLRUN_DOCKER_TAG)$(MLRUN_LEGACY_DOCKER_TAG_SUFFIX)
+
+models-gpu: ## Build models-gpu and legacy-models-gpu docker images
+	docker build \
+		--file dockerfiles/models-gpu/Dockerfile \
+		--build-arg MLRUN_PACKAGE_TAG=$(MLRUN_PACKAGE_TAG) \
+		--build-arg MLRUN_GITHUB_REPO=$(MLRUN_GITHUB_REPO) \
+		--tag $(MLRUN_MODELS_GPU_IMAGE_NAME) .
+
+	docker build \
+		--file dockerfiles/models-gpu/$(MLRUN_LEGACY_DOCKERFILE_DIR_NAME)/Dockerfile \
+		--build-arg MLRUN_PACKAGE_TAG=$(MLRUN_PACKAGE_TAG) \
+		--build-arg MLRUN_GITHUB_REPO=$(MLRUN_GITHUB_REPO) \
+		--tag $(MLRUN_LEGACY_MODELS_GPU_IMAGE_NAME) .
+
+IMAGES_TO_PUSH += $(MLRUN_MODELS_GPU_IMAGE_NAME)
+IMAGES_TO_PUSH += $(MLRUN_LEGACY_MODELS_GPU_IMAGE_NAME)
+
+
+MLRUN_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/mlrun:$(MLRUN_DOCKER_TAG)
+
+mlrun: ## Build mlrun docker image
+	docker build \
+		--file ./Dockerfile \
+		--build-arg MLRUN_PYTHON_VERSION=$(MLRUN_PYTHON_VERSION) \
+		--tag $(MLRUN_IMAGE_NAME) .
+
+IMAGES_TO_PUSH += $(MLRUN_IMAGE_NAME)
+
+
+MLRUN_SERVING_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/$(MLRUN_ML_DOCKER_IMAGE_NAME_PREFIX)serving:$(MLRUN_DOCKER_TAG)
+
+serving: ## Build serving docker image
+	docker build \
+		--file dockerfiles/serving/Dockerfile \
+		--build-arg MLRUN_DOCKER_TAG=$(MLRUN_DOCKER_TAG) \
+		--build-arg MLRUN_DOCKER_REPO=$(MLRUN_DOCKER_REPO) \
+		--build-arg MLRUN_ML_DOCKER_IMAGE_NAME_PREFIX=$(MLRUN_ML_DOCKER_IMAGE_NAME_PREFIX) \
+		--tag $(MLRUN_SERVING_IMAGE_NAME) .
+
+
+MLRUN_API_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/mlrun-api:$(MLRUN_DOCKER_TAG)
+
+api: ## Build mlrun-api docker image
+	docker build \
+		--file dockerfiles/mlrun-api/Dockerfile \
+		--build-arg MLRUN_PYTHON_VERSION=$(MLRUN_PYTHON_VERSION) \
+		--tag $(MLRUN_API_IMAGE_NAME) .
+
+IMAGES_TO_PUSH += $(MLRUN_API_IMAGE_NAME)
+
+
+MLRUN_TEST_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/test:$(MLRUN_DOCKER_TAG)
+
+build-test: ## Build test docker image
+	docker build \
+		--file dockerfiles/test/Dockerfile \
+		--build-arg MLRUN_PYTHON_VERSION=$(MLRUN_PYTHON_VERSION) \
+		--tag $(MLRUN_TEST_IMAGE_NAME) .
+
+
+package-wheel: clean ## Build python package wheel
 	python setup.py bdist_wheel
 
-.PHONY: clean
-clean:
+
+publish-package: package-wheel ## Publish python package wheel
+	python -m twine upload dist/mlrun-*.whl
+
+
+clean: ## Clean python package build artifacts
 	rm -rf build
 	rm -rf dist
 	rm -rf mlrun.egg-info
 	find . -name '*.pyc' -exec rm {} \;
 
 
-.PHONY: test
-test: clean
+test-dockerized: build-test ## Run mlrun tests in docker container
+	-docker network create mlrun
+	docker run \
+		-v /var/run/docker.sock:/var/run/docker.sock \
+		--network mlrun \
+		$(MLRUN_TEST_IMAGE_NAME) make test
+
+
+test: clean ## Run mlrun tests
 	python -m pytest -v \
-	    --disable-warnings \
-	    -rf \
-	    tests
+		--disable-warnings \
+		-rf \
+		tests
 
-.PHONY: run-api
-run-api:
+
+run-api-undockerized: ## Run mlrun api locally (un-dockerized)
 	python -m mlrun db
 
-.PHONY: docker-api
-docker-api:
-	docker build \
-	    -f dockerfiles/mlrun-api/Dockerfile \
-	    -t mlrun/mlrun-api .
 
-.PHONY: circleci
-circleci:
-	docker build \
-		-f dockerfiles/test/Dockerfile \
-		-t mlrun/test .
-	-docker network create mlrun
-	docker run \
-	    -v /var/run/docker.sock:/var/run/docker.sock \
-	    --network mlrun \
-	    mlrun/test make test
+circleci: test-dockerized
 	docker run \
-	    -v $(PWD)/docs/_build:/mlrun/docs/_build \
-	    mlrun/test make html-docs
+		-v $(PWD)/docs/_build:/mlrun/docs/_build \
+		mlrun/test make html-docs
 
-docs-requirements:
+
+docs-requirements: ## Build docs requirements
 	cp requirements.txt docs/requirements.txt
 	echo numpydoc >> docs/requirements.txt
 
-.PHONY: html-docs
-html-docs: docs-requirements
+
+html-docs: docs-requirements ## Build html docs
 	rm -f docs/external/*.md
 	cd docs && make html
+
+
+.PHONY: all, help ,build, docker-images, push-docker-images, print-docker-images, base, models, models-gpu, mlrun, \
+serving, api, build-test, package-wheel, publish-package, clean, test-dockerized, test, run-api-undockerized, \
+circleci, docs-requirements, html-docs

dockerfiles/README.md

@@ -1,32 +1,32 @@
-# dockerfiles and build script
+# dockerfiles and building
 
 ## build
-to build run this script from the root directory of the mlrun repository:<br>
+to build run this command from the root directory of the mlrun repository:<br>
 
-    ./docker-build REPO PREFIX MLRUN_TAG NEW_TAG PYTHON_VER_ML PYTHON_VER_CORE
+    MLRUN_DOCKER_TAG=X MLRUN_DOCKER_REPO=X MLRUN_ML_DOCKER_IMAGE_NAME_PREFIX=X MLRUN_PACKAGE_TAG=X MLRUN_GITHUB_REPO=X MLRUN_PYTHON_VERSION=X make push-docker-images
 
 where:<br>
-* `REPO` is your docker hub account (like `mlrun`)
-* `PREFIX` is some prefix common to the machine-learning/AI images created here (like `ml`)
-* `MLRUN_TAG` is a specific mlrun commit or tag _(prefix only tags with a `v`, like `v0.4.5`)_
-* `NEW_TAG` this is the tag created and pushed (like `latest` or `0.4.5`)
-* `PYTHON_VER_ML` is the version for the ml-xxxx series. 
-* `PYTHON_VER_CORE` is the python version for `mlrun-api`, `dask` and `test`
+* `MLRUN_DOCKER_TAG` this is the tag created and pushed (like `latest` or `0.4.5`, defaults to `latest`)
+* `MLRUN_DOCKER_REPO` is your docker hub account (defaults to `mlrun`)
+* `MLRUN_ML_DOCKER_IMAGE_NAME_PREFIX` is some prefix common to the machine-learning/AI images created here (defaults to `ml-`)
+* `MLRUN_PACKAGE_TAG` is the tag of mlrun package installed in the images _(prefix only tags with a `v`, like `v0.4.5`, defaults to `development`)_
+* `MLRUN_GITHUB_REPO` is the github repo from which we `pip install` mlrun (defaults to `mlrun`)
+* `MLRUN_PYTHON_VERSION` is the python version used (defaults to `3.7`). 
 
 for example,
-  `./docker-build.sh mlrun ml v0.4.7 0.4.6 3.7 3.6`
+  `MLRUN_DOCKER_TAG=0.4.6 MLRUN_DOCKER_REPO=mlrun MLRUN_ML_DOCKER_IMAGE_NAME_PREFIX=ml- MLRUN_PACKAGE_TAG=v0.4.7 MLRUN_GITHUB_REPO=mlrun MLRUN_PYTHON_VERSION=3.7 make docker-images`
 this will generate the following images:
-  * `mlrun/ml-base:0.4.7`       (python 3.7)
-  * `mlrun/ml-models:0.4.7`     (python 3.7)
-  * `mlrun/ml-models-gpu:0.4.7` (python 3.7) 
-  * `mlrun/mlrun-api:0.4.7`     (python 3.7)
-  * `mlrun/mlrun:0.4.7`         (python 3.7)
+  * `mlrun/ml-base:0.4.6`       (python 3.7)
+  * `mlrun/ml-models:0.4.6`     (python 3.7)
+  * `mlrun/ml-models-gpu:0.4.6` (python 3.7) 
+  * `mlrun/mlrun-api:0.4.6`     (python 3.7)
+  * `mlrun/mlrun:0.4.6`         (python 3.7)
 
 For compatability with some packages requiring py36, there is also an `ml-xxx` series of
-images tagged `0.4.7-py36`
+images tagged `0.4.6-py36`
 
 ## notable changes
-* `mlrun/dask` has been deprecated, used `mlrun/ml-base` or `mlrun/ml-models` instead
+* `mlrun/dask` has been deprecated, use `mlrun/ml-base` or `mlrun/ml-models` instead
 * `ml-models` and `ml-models-gpu` both contain OpenMPI and Horovod (and Dask)
 * `plotly` has been added to `ml-models` and `ml-models-gpu`, see **[plotly python](https://plotly.com/python/)** for details