Automating transplanting version in code (#437)

mlrun · Sep 21, 2020 · 33aef35 · 33aef35
1 parent 72762af
commit 33aef35
Show file tree

Hide file tree

Showing 23 changed files with 263 additions and 77 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -46,6 +46,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
+    - name: Set up python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Install automation scripts dependencies
+      run: pip install -r automation/requirements.txt
     - name: Run Dockerized tests
       run: make test-dockerized
 
@@ -54,6 +60,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+      - name: Set up python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Install automation scripts dependencies
+        run: pip install -r automation/requirements.txt
       - name: Generate HTML docs
         run: make html-docs-dockerized
       - name: Upload generated docs

diff --git a/.gitignore b/.gitignore
@@ -21,3 +21,4 @@ model.txt
 result*.html
 tests/test_results/
 venv
+mlrun/utils/version/version.json
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -22,71 +22,71 @@ podTemplate(label: "${git_project}-${label}", inheritFrom: "jnlp-docker-golang-p
                     container('docker-cmd') {
                         stage("build ${git_project}/api in dood") {
                             dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") {
-                                println(common.shellc("export MLRUN_DOCKER_TAG=${github.DOCKER_TAG_VERSION} && make api"))
+                                println(common.shellc("MLRUN_VERSION=${github.DOCKER_TAG_VERSION} make api"))
                             }
                         }
 
                         dockerx.images_push_multi_registries(["${git_project}/mlrun-api:${github.DOCKER_TAG_VERSION}"], [pipelinex.DockerRepo.ARTIFACTORY_IGUAZIO, pipelinex.DockerRepo.MLRUN_DOCKER_HUB, pipelinex.DockerRepo.MLRUN_QUAY_IO])
 
                         stage("build ${git_project}/mlrun in dood") {
                             dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") {
-                                println(common.shellc("export MLRUN_DOCKER_TAG=${github.DOCKER_TAG_VERSION} && make mlrun"))
+                                println(common.shellc("MLRUN_VERSION=${github.DOCKER_TAG_VERSION} make mlrun"))
                             }
                         }
 
                         dockerx.images_push_multi_registries(["${git_project}/mlrun:${github.DOCKER_TAG_VERSION}"], [pipelinex.DockerRepo.ARTIFACTORY_IGUAZIO, pipelinex.DockerRepo.MLRUN_DOCKER_HUB, pipelinex.DockerRepo.MLRUN_QUAY_IO])
 
                         stage("build ${git_project}/jupyter in dood") {
                             dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") {
-                                println(common.shellc("export MLRUN_DOCKER_TAG=${github.DOCKER_TAG_VERSION} && make jupyter"))
+                                println(common.shellc("MLRUN_VERSION=${github.DOCKER_TAG_VERSION} make jupyter"))
                             }
                         }
 
                         dockerx.images_push_multi_registries(["${git_project}/jupyter:${github.DOCKER_TAG_VERSION}"], [pipelinex.DockerRepo.ARTIFACTORY_IGUAZIO, pipelinex.DockerRepo.MLRUN_DOCKER_HUB, pipelinex.DockerRepo.MLRUN_QUAY_IO])
 
                         stage("build ${git_project}/base in dood") {
                             dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") {
-                                println(common.shellc("export MLRUN_DOCKER_TAG=${github.DOCKER_TAG_VERSION} && make base"))
+                                println(common.shellc("MLRUN_VERSION=${github.DOCKER_TAG_VERSION} make base"))
                             }
                         }
 
                         dockerx.images_push_multi_registries(["${git_project}/ml-base:${github.DOCKER_TAG_VERSION}"], [pipelinex.DockerRepo.ARTIFACTORY_IGUAZIO, pipelinex.DockerRepo.MLRUN_DOCKER_HUB, pipelinex.DockerRepo.MLRUN_QUAY_IO])
 
                         stage("build ${git_project}/base-legacy in dood") {
                             dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") {
-                                println(common.shellc("export MLRUN_DOCKER_TAG=${github.DOCKER_TAG_VERSION} && make base-legacy"))
+                                println(common.shellc("MLRUN_VERSION=${github.DOCKER_TAG_VERSION} make base-legacy"))
                             }
                         }
 
                         dockerx.images_push_multi_registries(["${git_project}/ml-base:${github.DOCKER_TAG_VERSION}-py36"], [pipelinex.DockerRepo.ARTIFACTORY_IGUAZIO, pipelinex.DockerRepo.MLRUN_DOCKER_HUB, pipelinex.DockerRepo.MLRUN_QUAY_IO])
 
                         stage("build ${git_project}/models in dood") {
                             dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") {
-                                println(common.shellc("export MLRUN_DOCKER_TAG=${github.DOCKER_TAG_VERSION} && make models"))
+                                println(common.shellc("MLRUN_VERSION=${github.DOCKER_TAG_VERSION} make models"))
                             }
                         }
 
                         dockerx.images_push_multi_registries(["${git_project}/ml-models:${github.DOCKER_TAG_VERSION}"], [pipelinex.DockerRepo.ARTIFACTORY_IGUAZIO, pipelinex.DockerRepo.MLRUN_DOCKER_HUB, pipelinex.DockerRepo.MLRUN_QUAY_IO])
 
                         stage("build ${git_project}/models-legacy in dood") {
                             dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") {
-                                println(common.shellc("export MLRUN_DOCKER_TAG=${github.DOCKER_TAG_VERSION} && make models-legacy"))
+                                println(common.shellc("MLRUN_VERSION=${github.DOCKER_TAG_VERSION} make models-legacy"))
                             }
                         }
 
                         dockerx.images_push_multi_registries(["${git_project}/ml-models:${github.DOCKER_TAG_VERSION}-py36"], [pipelinex.DockerRepo.ARTIFACTORY_IGUAZIO, pipelinex.DockerRepo.MLRUN_DOCKER_HUB, pipelinex.DockerRepo.MLRUN_QUAY_IO])
 
                         stage("build ${git_project}/models-gpu in dood") {
                             dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") {
-                                println(common.shellc("export MLRUN_DOCKER_TAG=${github.DOCKER_TAG_VERSION} && make models-gpu"))
+                                println(common.shellc("MLRUN_VERSION=${github.DOCKER_TAG_VERSION} make models-gpu"))
                             }
                         }
 
                         dockerx.images_push_multi_registries(["${git_project}/ml-models-gpu:${github.DOCKER_TAG_VERSION}"], [pipelinex.DockerRepo.ARTIFACTORY_IGUAZIO, pipelinex.DockerRepo.MLRUN_DOCKER_HUB, pipelinex.DockerRepo.MLRUN_QUAY_IO])
 
                         stage("build ${git_project}/models-gpu-legacy in dood") {
                             dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") {
-                                println(common.shellc("export MLRUN_DOCKER_TAG=${github.DOCKER_TAG_VERSION} && make models-gpu-legacy"))
+                                println(common.shellc("MLRUN_VERSION=${github.DOCKER_TAG_VERSION} make models-gpu-legacy"))
                             }
                         }
 
@@ -128,7 +128,7 @@ podTemplate(label: "${git_project}-${label}", inheritFrom: "jnlp-docker-golang-p
                                                     usernameVariable: 'TWINE_USERNAME')]) {
                                 dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") {
                                     println(common.shellc("pip install twine"))
-                                    println(common.shellc("make publish-package"))
+                                    println(common.shellc("MLRUN_VERSION=${github.DOCKER_TAG_VERSION} make publish-package"))
                                 }
                             }
                         }

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -15,6 +15,7 @@
 include Dockerfile*
 include Makefile
 include README.md
+include mlrun/utils/version/version.json
 include *requirements.txt
 recursive-include examples *
 recursive-include hack *

diff --git a/Makefile b/Makefile
@@ -12,9 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-MLRUN_DOCKER_TAG ?= unstable
+MLRUN_VERSION ?= unstable
+MLRUN_DOCKER_TAG ?= $(MLRUN_VERSION)
 MLRUN_DOCKER_REPO ?= mlrun
-MLRUN_DOCKER_REGISTRY ?=  # empty be default (dockerhub), can be set to something like "quay.io/"
+# empty by default (dockerhub), can be set to something like "quay.io/".
+# This will be used to tag the images built using this makefile
+MLRUN_DOCKER_REGISTRY ?=
 MLRUN_ML_DOCKER_IMAGE_NAME_PREFIX ?= ml-
 MLRUN_PYTHON_VERSION ?= 3.7
 MLRUN_LEGACY_ML_PYTHON_VERSION ?= 3.6
@@ -55,9 +58,10 @@ endif
 	echo $(MLRUN_OLD_VERSION_ESCAPED)
 	find . \( ! -regex '.*/\..*' \) -a \( -iname \*.md -o -iname \*.txt -o -iname \*.yaml -o -iname \*.yml \)  \
 	-type f -print0 | xargs -0 sed -i '' -e 's/:$(MLRUN_OLD_VERSION_ESCAPED)/:$(MLRUN_NEW_VERSION)/g'
-	sed -i ''  \
-	-e 's/__version__[[:space:]]=[[:space:]]"$(MLRUN_OLD_VERSION_ESCAPED)"/__version__ = "$(MLRUN_NEW_VERSION)"/g'  \
-	 ./mlrun/__init__.py
+
+.PHONY: update-version-file
+update-version-file: ## Update the version file
+	python ./automation/version/version_file.py create $(MLRUN_VERSION)
 
 .PHONY: build
 build: docker-images package-wheel ## Build all artifacts
@@ -95,7 +99,7 @@ MLRUN_BASE_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/$(MLRUN_ML_DOCKER_IMAGE_NA
 DEFAULT_IMAGES += $(MLRUN_BASE_IMAGE_NAME)
 
 .PHONY: base
-base: ## Build base docker image
+base: update-version-file ## Build base docker image
 	docker build \
 		--file dockerfiles/base/Dockerfile \
 		--build-arg MLRUN_PYTHON_VERSION=$(MLRUN_PYTHON_VERSION) \
@@ -112,7 +116,7 @@ MLRUN_LEGACY_BASE_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/$(MLRUN_ML_DOCKER_I
 DEFAULT_IMAGES += $(MLRUN_LEGACY_BASE_IMAGE_NAME)
 
 .PHONY: base-legacy
-base-legacy: ## Build base legacy docker image
+base-legacy: update-version-file ## Build base legacy docker image
 	docker build \
 		--file dockerfiles/base/Dockerfile \
 		--build-arg MLRUN_PYTHON_VERSION=$(MLRUN_LEGACY_ML_PYTHON_VERSION) \
@@ -129,7 +133,7 @@ MLRUN_MODELS_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/$(MLRUN_ML_DOCKER_IMAGE_
 DEFAULT_IMAGES += $(MLRUN_MODELS_IMAGE_NAME)
 
 .PHONY: models
-models: ## Build models docker image
+models: update-version-file ## Build models docker image
 	docker build \
 		--file dockerfiles/models/Dockerfile \
 		--build-arg MLRUN_MLUTILS_GITHUB_TAG=$(MLRUN_MLUTILS_GITHUB_TAG) \
@@ -145,7 +149,7 @@ MLRUN_LEGACY_MODELS_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/$(MLRUN_ML_DOCKER
 DEFAULT_IMAGES += $(MLRUN_LEGACY_MODELS_IMAGE_NAME)
 
 .PHONY: models-legacy
-models-legacy: ## Build models legacy docker image
+models-legacy: update-version-file ## Build models legacy docker image
 	docker build \
 		--file dockerfiles/models/$(MLRUN_LEGACY_DOCKERFILE_DIR_NAME)/Dockerfile \
 		--build-arg MLRUN_MLUTILS_GITHUB_TAG=$(MLRUN_MLUTILS_GITHUB_TAG) \
@@ -161,7 +165,7 @@ MLRUN_MODELS_GPU_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/$(MLRUN_ML_DOCKER_IM
 DEFAULT_IMAGES += $(MLRUN_MODELS_GPU_IMAGE_NAME)
 
 .PHONY: models-gpu
-models-gpu: ## Build models-gpu docker image
+models-gpu: update-version-file ## Build models-gpu docker image
 	docker build \
 		--file dockerfiles/models-gpu/Dockerfile \
 		--build-arg MLRUN_MLUTILS_GITHUB_TAG=$(MLRUN_MLUTILS_GITHUB_TAG) \
@@ -177,7 +181,7 @@ MLRUN_LEGACY_MODELS_GPU_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/$(MLRUN_ML_DO
 DEFAULT_IMAGES += $(MLRUN_LEGACY_MODELS_GPU_IMAGE_NAME)
 
 .PHONY: models-gpu-legacy
-models-gpu-legacy: ## Build models-gpu legacy docker image
+models-gpu-legacy: update-version-file ## Build models-gpu legacy docker image
 	docker build \
 		--file dockerfiles/models-gpu/$(MLRUN_LEGACY_DOCKERFILE_DIR_NAME)/Dockerfile \
 		--build-arg MLRUN_MLUTILS_GITHUB_TAG=$(MLRUN_MLUTILS_GITHUB_TAG) \
@@ -193,7 +197,7 @@ MLRUN_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/mlrun:$(MLRUN_DOCKER_TAG)
 DEFAULT_IMAGES += $(MLRUN_IMAGE_NAME)
 
 .PHONY: mlrun
-mlrun: ## Build mlrun docker image
+mlrun: update-version-file ## Build mlrun docker image
 	docker build \
 		--file dockerfiles/mlrun/Dockerfile \
 		--build-arg MLRUN_PYTHON_VERSION=$(MLRUN_PYTHON_VERSION) \
@@ -208,7 +212,7 @@ MLRUN_JUPYTER_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/jupyter:$(MLRUN_DOCKER_
 DEFAULT_IMAGES += $(MLRUN_JUPYTER_IMAGE_NAME)
 
 .PHONY: jupyter
-jupyter: ## Build mlrun jupyter docker image
+jupyter: update-version-file ## Build mlrun jupyter docker image
 	docker build \
 		--file dockerfiles/jupyter/Dockerfile \
 		--build-arg MLRUN_CACHE_DATE=$(MLRUN_CACHE_DATE) \
@@ -222,7 +226,7 @@ push-jupyter: jupyter ## Push mlrun jupyter docker image
 MLRUN_SERVING_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/$(MLRUN_ML_DOCKER_IMAGE_NAME_PREFIX)serving:$(MLRUN_DOCKER_TAG)
 
 .PHONY: serving
-serving: ## Build serving docker image
+serving: update-version-file ## Build serving docker image
 	docker build \
 		--file dockerfiles/serving/Dockerfile \
 		--build-arg MLRUN_DOCKER_TAG=$(MLRUN_DOCKER_TAG) \
@@ -239,7 +243,7 @@ MLRUN_API_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/mlrun-api:$(MLRUN_DOCKER_TA
 DEFAULT_IMAGES += $(MLRUN_API_IMAGE_NAME)
 
 .PHONY: api
-api: ## Build mlrun-api docker image
+api: update-version-file ## Build mlrun-api docker image
 	docker build \
 		--file dockerfiles/mlrun-api/Dockerfile \
 		--build-arg MLRUN_PYTHON_VERSION=$(MLRUN_PYTHON_VERSION) \
@@ -252,7 +256,7 @@ push-api: api ## Push api docker image
 MLRUN_TEST_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/test:$(MLRUN_DOCKER_TAG)
 
 .PHONY: build-test
-build-test: ## Build test docker image
+build-test: update-version-file ## Build test docker image
 	docker build \
 		--file dockerfiles/test/Dockerfile \
 		--build-arg MLRUN_PYTHON_VERSION=$(MLRUN_PYTHON_VERSION) \
@@ -261,7 +265,7 @@ build-test: ## Build test docker image
 MLRUN_SYSTEM_TEST_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/test-system:$(MLRUN_DOCKER_TAG)
 
 .PHONY: build-test-system
-build-test-system: ## Build system tests docker image
+build-test-system: update-version-file ## Build system tests docker image
 	docker build \
 		--file dockerfiles/test-system/Dockerfile \
 		--build-arg MLRUN_PYTHON_VERSION=$(MLRUN_PYTHON_VERSION) \
@@ -272,7 +276,7 @@ push-test: build-test ## Push test docker image
 	docker push $(MLRUN_TEST_IMAGE_NAME)
 
 .PHONY: package-wheel
-package-wheel: clean ## Build python package wheel
+package-wheel: clean update-version-file ## Build python package wheel
 	python setup.py bdist_wheel
 
 .PHONY: publish-package

diff --git a/ci/__init__.py → automation/__init__.py b/ci/__init__.py → automation/__init__.py
diff --git a/automation/requirements.txt b/automation/requirements.txt
@@ -0,0 +1,2 @@
+# ==7.0 from kfp
+click==7.0
diff --git a/ci/system_test/__init__.py → automation/system_test/__init__.py b/ci/system_test/__init__.py → automation/system_test/__init__.py
diff --git a/ci/system_test/run.py → automation/system_test/run.py b/ci/system_test/run.py → automation/system_test/run.py
@@ -10,14 +10,14 @@
 import mlrun.utils
 
 
-logger = mlrun.utils.create_logger(level="debug", name="ci")
+logger = mlrun.utils.create_logger(level="debug", name="automation")
 
 
 class SystemTestCIRunner:
     class Constants:
         ssh_username = "iguazio"
 
-        ci_dir_name = "mlrun-ci"
+        ci_dir_name = "mlrun-automation"
         homedir = pathlib.Path("/home/iguazio/")
         workdir = homedir / ci_dir_name
         mlrun_code_path = workdir / "mlrun"
@@ -366,7 +366,7 @@ def run(
     try:
         system_test_ci_runner.run()
     except Exception as e:
-        logger.error("Failed running system test ci", exception=e)
+        logger.error("Failed running system test automation", exception=e)
     finally:
         system_test_ci_runner.clean_up()
 

diff --git a/automation/version/__init__.py b/automation/version/__init__.py
diff --git a/automation/version/version_file.py b/automation/version/version_file.py
@@ -0,0 +1,55 @@
+import json
+import pathlib
+import subprocess
+
+import click
+
+import logging
+
+# not using mlrun.utils.logger to not require mlrun package
+logger = logging.Logger(name="version", level="DEBUG")
+
+
+@click.group()
+def main():
+    pass
+
+
+@main.command(context_settings=dict(ignore_unknown_options=True))
+@click.argument("mlrun-version", type=str, required=False, default="unstable")
+def create(mlrun_version: str):
+    git_commit = "unknown"
+    try:
+        out, _, _ = _run_command("git", args=["rev-parse", "HEAD"])
+        git_commit = out.strip()
+
+    except Exception as exc:
+        logger.warning("Failed to get version", exc_info=exc)
+
+    version_info = {
+        "version": mlrun_version,
+        "git_commit": git_commit,
+    }
+
+    repo_root = pathlib.Path(__file__).resolve().absolute().parent.parent.parent
+    version_file_path = repo_root / "mlrun" / "utils" / "version" / "version.json"
+    logger.info(f"Writing version info to file: {str(version_info)}")
+    with open(version_file_path, "w+") as version_file:
+        json.dump(version_info, version_file, sort_keys=True, indent=2)
+
+
+def _run_command(
+    command: str, args: list = None, suppress_errors: bool = False,
+) -> (str, str, int):
+    if args:
+        command += " " + " ".join(args)
+
+    process = subprocess.run(
+        command, shell=True, check=True, capture_output=True, encoding="utf-8",
+    )
+
+    return process.stdout, process.stderr, process.returncode
+
+
+if __name__ == "__main__":
+    main()
diff --git a/mlrun/__init__.py b/mlrun/__init__.py
@@ -23,8 +23,15 @@
 
 # flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 
-__version__ = "0.5.2"
+from os import environ, path
 
+from .config import config as mlconf
+from .datastore import DataItem
+from .db import get_run_db
+from .execution import MLClientCtx
+from .model import RunTemplate, NewTask, RunObject
+from .platforms import mount_v3io, v3io_cred
+from .projects import load_project, new_project
 from .run import (
     get_or_create_ctx,
     new_function,
@@ -37,16 +44,10 @@
     get_pipeline,
     wait_for_pipeline_completion,
 )
-from .db import get_run_db
-from .model import RunTemplate, NewTask, RunObject
-from .config import config as mlconf
 from .runtimes import new_model_server
-from .platforms import mount_v3io, v3io_cred
-from .projects import load_project, new_project
-from .datastore import DataItem
-from .execution import MLClientCtx
+from .utils.version import Version
 
-from os import environ, path
+__version__ = Version().get()["version"]
 
 
 def get_version():