From 33aef350597171a6e1138bfc4f131467c8e26822 Mon Sep 17 00:00:00 2001 From: Hedingber Date: Mon, 21 Sep 2020 11:55:12 +0300 Subject: [PATCH] Automating transplanting version in code (#437) --- .github/workflows/ci.yaml | 12 +++++ .gitignore | 1 + Jenkinsfile | 20 ++++---- MANIFEST.in | 1 + Makefile | 40 +++++++++------- {ci => automation}/__init__.py | 0 automation/requirements.txt | 2 + {ci => automation}/system_test/__init__.py | 0 {ci => automation}/system_test/run.py | 6 +-- automation/version/__init__.py | 0 automation/version/version_file.py | 55 ++++++++++++++++++++++ mlrun/__init__.py | 19 ++++---- mlrun/__main__.py | 14 +++--- mlrun/builder.py | 4 +- mlrun/config.py | 35 ++++++++++---- mlrun/runtimes/base.py | 4 +- mlrun/runtimes/function.py | 4 +- mlrun/utils/helpers.py | 23 +++++---- mlrun/utils/singleton.py | 7 +++ mlrun/utils/version/__init__.py | 1 + mlrun/utils/version/version.py | 23 +++++++++ setup.py | 20 ++++++-- tests/utils/test_helpers.py | 49 ++++++++++++++++++- 23 files changed, 263 insertions(+), 77 deletions(-) rename {ci => automation}/__init__.py (100%) create mode 100644 automation/requirements.txt rename {ci => automation}/system_test/__init__.py (100%) rename {ci => automation}/system_test/run.py (98%) create mode 100644 automation/version/__init__.py create mode 100644 automation/version/version_file.py create mode 100644 mlrun/utils/singleton.py create mode 100644 mlrun/utils/version/__init__.py create mode 100644 mlrun/utils/version/version.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index cb7b2e32fd0..b5f42cfb0b5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -46,6 +46,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - name: Set up python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Install automation scripts dependencies + run: pip install -r automation/requirements.txt - name: Run Dockerized tests run: make test-dockerized @@ -54,6 +60,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - name: Set up python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Install automation scripts dependencies + run: pip install -r automation/requirements.txt - name: Generate HTML docs run: make html-docs-dockerized - name: Upload generated docs diff --git a/.gitignore b/.gitignore index a4439cc7841..d5044dd406c 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ model.txt result*.html tests/test_results/ venv +mlrun/utils/version/version.json diff --git a/Jenkinsfile b/Jenkinsfile index 65f684f7b58..7a14bd6d56f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -22,7 +22,7 @@ podTemplate(label: "${git_project}-${label}", inheritFrom: "jnlp-docker-golang-p container('docker-cmd') { stage("build ${git_project}/api in dood") { dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") { - println(common.shellc("export MLRUN_DOCKER_TAG=${github.DOCKER_TAG_VERSION} && make api")) + println(common.shellc("MLRUN_VERSION=${github.DOCKER_TAG_VERSION} make api")) } } @@ -30,7 +30,7 @@ podTemplate(label: "${git_project}-${label}", inheritFrom: "jnlp-docker-golang-p stage("build ${git_project}/mlrun in dood") { dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") { - println(common.shellc("export MLRUN_DOCKER_TAG=${github.DOCKER_TAG_VERSION} && make mlrun")) + println(common.shellc("MLRUN_VERSION=${github.DOCKER_TAG_VERSION} make mlrun")) } } @@ -38,7 +38,7 @@ podTemplate(label: "${git_project}-${label}", inheritFrom: "jnlp-docker-golang-p stage("build ${git_project}/jupyter in dood") { dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") { - println(common.shellc("export MLRUN_DOCKER_TAG=${github.DOCKER_TAG_VERSION} && make jupyter")) + println(common.shellc("MLRUN_VERSION=${github.DOCKER_TAG_VERSION} make jupyter")) } } @@ -46,7 +46,7 @@ podTemplate(label: "${git_project}-${label}", inheritFrom: "jnlp-docker-golang-p stage("build ${git_project}/base in dood") { dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") { - println(common.shellc("export MLRUN_DOCKER_TAG=${github.DOCKER_TAG_VERSION} && make base")) + println(common.shellc("MLRUN_VERSION=${github.DOCKER_TAG_VERSION} make base")) } } @@ -54,7 +54,7 @@ podTemplate(label: "${git_project}-${label}", inheritFrom: "jnlp-docker-golang-p stage("build ${git_project}/base-legacy in dood") { dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") { - println(common.shellc("export MLRUN_DOCKER_TAG=${github.DOCKER_TAG_VERSION} && make base-legacy")) + println(common.shellc("MLRUN_VERSION=${github.DOCKER_TAG_VERSION} make base-legacy")) } } @@ -62,7 +62,7 @@ podTemplate(label: "${git_project}-${label}", inheritFrom: "jnlp-docker-golang-p stage("build ${git_project}/models in dood") { dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") { - println(common.shellc("export MLRUN_DOCKER_TAG=${github.DOCKER_TAG_VERSION} && make models")) + println(common.shellc("MLRUN_VERSION=${github.DOCKER_TAG_VERSION} make models")) } } @@ -70,7 +70,7 @@ podTemplate(label: "${git_project}-${label}", inheritFrom: "jnlp-docker-golang-p stage("build ${git_project}/models-legacy in dood") { dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") { - println(common.shellc("export MLRUN_DOCKER_TAG=${github.DOCKER_TAG_VERSION} && make models-legacy")) + println(common.shellc("MLRUN_VERSION=${github.DOCKER_TAG_VERSION} make models-legacy")) } } @@ -78,7 +78,7 @@ podTemplate(label: "${git_project}-${label}", inheritFrom: "jnlp-docker-golang-p stage("build ${git_project}/models-gpu in dood") { dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") { - println(common.shellc("export MLRUN_DOCKER_TAG=${github.DOCKER_TAG_VERSION} && make models-gpu")) + println(common.shellc("MLRUN_VERSION=${github.DOCKER_TAG_VERSION} make models-gpu")) } } @@ -86,7 +86,7 @@ podTemplate(label: "${git_project}-${label}", inheritFrom: "jnlp-docker-golang-p stage("build ${git_project}/models-gpu-legacy in dood") { dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") { - println(common.shellc("export MLRUN_DOCKER_TAG=${github.DOCKER_TAG_VERSION} && make models-gpu-legacy")) + println(common.shellc("MLRUN_VERSION=${github.DOCKER_TAG_VERSION} make models-gpu-legacy")) } } @@ -128,7 +128,7 @@ podTemplate(label: "${git_project}-${label}", inheritFrom: "jnlp-docker-golang-p usernameVariable: 'TWINE_USERNAME')]) { dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") { println(common.shellc("pip install twine")) - println(common.shellc("make publish-package")) + println(common.shellc("MLRUN_VERSION=${github.DOCKER_TAG_VERSION} make publish-package")) } } } diff --git a/MANIFEST.in b/MANIFEST.in index 0fa05655341..6303b358403 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -15,6 +15,7 @@ include Dockerfile* include Makefile include README.md +include mlrun/utils/version/version.json include *requirements.txt recursive-include examples * recursive-include hack * diff --git a/Makefile b/Makefile index 6b201b1c582..7f080c9906f 100644 --- a/Makefile +++ b/Makefile @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -MLRUN_DOCKER_TAG ?= unstable +MLRUN_VERSION ?= unstable +MLRUN_DOCKER_TAG ?= $(MLRUN_VERSION) MLRUN_DOCKER_REPO ?= mlrun -MLRUN_DOCKER_REGISTRY ?= # empty be default (dockerhub), can be set to something like "quay.io/" +# empty by default (dockerhub), can be set to something like "quay.io/". +# This will be used to tag the images built using this makefile +MLRUN_DOCKER_REGISTRY ?= MLRUN_ML_DOCKER_IMAGE_NAME_PREFIX ?= ml- MLRUN_PYTHON_VERSION ?= 3.7 MLRUN_LEGACY_ML_PYTHON_VERSION ?= 3.6 @@ -55,9 +58,10 @@ endif echo $(MLRUN_OLD_VERSION_ESCAPED) find . \( ! -regex '.*/\..*' \) -a \( -iname \*.md -o -iname \*.txt -o -iname \*.yaml -o -iname \*.yml \) \ -type f -print0 | xargs -0 sed -i '' -e 's/:$(MLRUN_OLD_VERSION_ESCAPED)/:$(MLRUN_NEW_VERSION)/g' - sed -i '' \ - -e 's/__version__[[:space:]]=[[:space:]]"$(MLRUN_OLD_VERSION_ESCAPED)"/__version__ = "$(MLRUN_NEW_VERSION)"/g' \ - ./mlrun/__init__.py + +.PHONY: update-version-file +update-version-file: ## Update the version file + python ./automation/version/version_file.py create $(MLRUN_VERSION) .PHONY: build build: docker-images package-wheel ## Build all artifacts @@ -95,7 +99,7 @@ MLRUN_BASE_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/$(MLRUN_ML_DOCKER_IMAGE_NA DEFAULT_IMAGES += $(MLRUN_BASE_IMAGE_NAME) .PHONY: base -base: ## Build base docker image +base: update-version-file ## Build base docker image docker build \ --file dockerfiles/base/Dockerfile \ --build-arg MLRUN_PYTHON_VERSION=$(MLRUN_PYTHON_VERSION) \ @@ -112,7 +116,7 @@ MLRUN_LEGACY_BASE_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/$(MLRUN_ML_DOCKER_I DEFAULT_IMAGES += $(MLRUN_LEGACY_BASE_IMAGE_NAME) .PHONY: base-legacy -base-legacy: ## Build base legacy docker image +base-legacy: update-version-file ## Build base legacy docker image docker build \ --file dockerfiles/base/Dockerfile \ --build-arg MLRUN_PYTHON_VERSION=$(MLRUN_LEGACY_ML_PYTHON_VERSION) \ @@ -129,7 +133,7 @@ MLRUN_MODELS_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/$(MLRUN_ML_DOCKER_IMAGE_ DEFAULT_IMAGES += $(MLRUN_MODELS_IMAGE_NAME) .PHONY: models -models: ## Build models docker image +models: update-version-file ## Build models docker image docker build \ --file dockerfiles/models/Dockerfile \ --build-arg MLRUN_MLUTILS_GITHUB_TAG=$(MLRUN_MLUTILS_GITHUB_TAG) \ @@ -145,7 +149,7 @@ MLRUN_LEGACY_MODELS_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/$(MLRUN_ML_DOCKER DEFAULT_IMAGES += $(MLRUN_LEGACY_MODELS_IMAGE_NAME) .PHONY: models-legacy -models-legacy: ## Build models legacy docker image +models-legacy: update-version-file ## Build models legacy docker image docker build \ --file dockerfiles/models/$(MLRUN_LEGACY_DOCKERFILE_DIR_NAME)/Dockerfile \ --build-arg MLRUN_MLUTILS_GITHUB_TAG=$(MLRUN_MLUTILS_GITHUB_TAG) \ @@ -161,7 +165,7 @@ MLRUN_MODELS_GPU_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/$(MLRUN_ML_DOCKER_IM DEFAULT_IMAGES += $(MLRUN_MODELS_GPU_IMAGE_NAME) .PHONY: models-gpu -models-gpu: ## Build models-gpu docker image +models-gpu: update-version-file ## Build models-gpu docker image docker build \ --file dockerfiles/models-gpu/Dockerfile \ --build-arg MLRUN_MLUTILS_GITHUB_TAG=$(MLRUN_MLUTILS_GITHUB_TAG) \ @@ -177,7 +181,7 @@ MLRUN_LEGACY_MODELS_GPU_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/$(MLRUN_ML_DO DEFAULT_IMAGES += $(MLRUN_LEGACY_MODELS_GPU_IMAGE_NAME) .PHONY: models-gpu-legacy -models-gpu-legacy: ## Build models-gpu legacy docker image +models-gpu-legacy: update-version-file ## Build models-gpu legacy docker image docker build \ --file dockerfiles/models-gpu/$(MLRUN_LEGACY_DOCKERFILE_DIR_NAME)/Dockerfile \ --build-arg MLRUN_MLUTILS_GITHUB_TAG=$(MLRUN_MLUTILS_GITHUB_TAG) \ @@ -193,7 +197,7 @@ MLRUN_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/mlrun:$(MLRUN_DOCKER_TAG) DEFAULT_IMAGES += $(MLRUN_IMAGE_NAME) .PHONY: mlrun -mlrun: ## Build mlrun docker image +mlrun: update-version-file ## Build mlrun docker image docker build \ --file dockerfiles/mlrun/Dockerfile \ --build-arg MLRUN_PYTHON_VERSION=$(MLRUN_PYTHON_VERSION) \ @@ -208,7 +212,7 @@ MLRUN_JUPYTER_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/jupyter:$(MLRUN_DOCKER_ DEFAULT_IMAGES += $(MLRUN_JUPYTER_IMAGE_NAME) .PHONY: jupyter -jupyter: ## Build mlrun jupyter docker image +jupyter: update-version-file ## Build mlrun jupyter docker image docker build \ --file dockerfiles/jupyter/Dockerfile \ --build-arg MLRUN_CACHE_DATE=$(MLRUN_CACHE_DATE) \ @@ -222,7 +226,7 @@ push-jupyter: jupyter ## Push mlrun jupyter docker image MLRUN_SERVING_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/$(MLRUN_ML_DOCKER_IMAGE_NAME_PREFIX)serving:$(MLRUN_DOCKER_TAG) .PHONY: serving -serving: ## Build serving docker image +serving: update-version-file ## Build serving docker image docker build \ --file dockerfiles/serving/Dockerfile \ --build-arg MLRUN_DOCKER_TAG=$(MLRUN_DOCKER_TAG) \ @@ -239,7 +243,7 @@ MLRUN_API_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/mlrun-api:$(MLRUN_DOCKER_TA DEFAULT_IMAGES += $(MLRUN_API_IMAGE_NAME) .PHONY: api -api: ## Build mlrun-api docker image +api: update-version-file ## Build mlrun-api docker image docker build \ --file dockerfiles/mlrun-api/Dockerfile \ --build-arg MLRUN_PYTHON_VERSION=$(MLRUN_PYTHON_VERSION) \ @@ -252,7 +256,7 @@ push-api: api ## Push api docker image MLRUN_TEST_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/test:$(MLRUN_DOCKER_TAG) .PHONY: build-test -build-test: ## Build test docker image +build-test: update-version-file ## Build test docker image docker build \ --file dockerfiles/test/Dockerfile \ --build-arg MLRUN_PYTHON_VERSION=$(MLRUN_PYTHON_VERSION) \ @@ -261,7 +265,7 @@ build-test: ## Build test docker image MLRUN_SYSTEM_TEST_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/test-system:$(MLRUN_DOCKER_TAG) .PHONY: build-test-system -build-test-system: ## Build system tests docker image +build-test-system: update-version-file ## Build system tests docker image docker build \ --file dockerfiles/test-system/Dockerfile \ --build-arg MLRUN_PYTHON_VERSION=$(MLRUN_PYTHON_VERSION) \ @@ -272,7 +276,7 @@ push-test: build-test ## Push test docker image docker push $(MLRUN_TEST_IMAGE_NAME) .PHONY: package-wheel -package-wheel: clean ## Build python package wheel +package-wheel: clean update-version-file ## Build python package wheel python setup.py bdist_wheel .PHONY: publish-package diff --git a/ci/__init__.py b/automation/__init__.py similarity index 100% rename from ci/__init__.py rename to automation/__init__.py diff --git a/automation/requirements.txt b/automation/requirements.txt new file mode 100644 index 00000000000..cc0cffdbe41 --- /dev/null +++ b/automation/requirements.txt @@ -0,0 +1,2 @@ +# ==7.0 from kfp +click==7.0 \ No newline at end of file diff --git a/ci/system_test/__init__.py b/automation/system_test/__init__.py similarity index 100% rename from ci/system_test/__init__.py rename to automation/system_test/__init__.py diff --git a/ci/system_test/run.py b/automation/system_test/run.py similarity index 98% rename from ci/system_test/run.py rename to automation/system_test/run.py index 4126d0429f6..3e084ec461d 100644 --- a/ci/system_test/run.py +++ b/automation/system_test/run.py @@ -10,14 +10,14 @@ import mlrun.utils -logger = mlrun.utils.create_logger(level="debug", name="ci") +logger = mlrun.utils.create_logger(level="debug", name="automation") class SystemTestCIRunner: class Constants: ssh_username = "iguazio" - ci_dir_name = "mlrun-ci" + ci_dir_name = "mlrun-automation" homedir = pathlib.Path("/home/iguazio/") workdir = homedir / ci_dir_name mlrun_code_path = workdir / "mlrun" @@ -366,7 +366,7 @@ def run( try: system_test_ci_runner.run() except Exception as e: - logger.error("Failed running system test ci", exception=e) + logger.error("Failed running system test automation", exception=e) finally: system_test_ci_runner.clean_up() diff --git a/automation/version/__init__.py b/automation/version/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/automation/version/version_file.py b/automation/version/version_file.py new file mode 100644 index 00000000000..a43fc8471cd --- /dev/null +++ b/automation/version/version_file.py @@ -0,0 +1,55 @@ +import json +import pathlib +import subprocess + +import click + +import logging + +# not using mlrun.utils.logger to not require mlrun package +logger = logging.Logger(name="version", level="DEBUG") + + +@click.group() +def main(): + pass + + +@main.command(context_settings=dict(ignore_unknown_options=True)) +@click.argument("mlrun-version", type=str, required=False, default="unstable") +def create(mlrun_version: str): + git_commit = "unknown" + try: + out, _, _ = _run_command("git", args=["rev-parse", "HEAD"]) + git_commit = out.strip() + + except Exception as exc: + logger.warning("Failed to get version", exc_info=exc) + + version_info = { + "version": mlrun_version, + "git_commit": git_commit, + } + + repo_root = pathlib.Path(__file__).resolve().absolute().parent.parent.parent + version_file_path = repo_root / "mlrun" / "utils" / "version" / "version.json" + logger.info(f"Writing version info to file: {str(version_info)}") + with open(version_file_path, "w+") as version_file: + json.dump(version_info, version_file, sort_keys=True, indent=2) + + +def _run_command( + command: str, args: list = None, suppress_errors: bool = False, +) -> (str, str, int): + if args: + command += " " + " ".join(args) + + process = subprocess.run( + command, shell=True, check=True, capture_output=True, encoding="utf-8", + ) + + return process.stdout, process.stderr, process.returncode + + +if __name__ == "__main__": + main() diff --git a/mlrun/__init__.py b/mlrun/__init__.py index b7a039cc60e..ad9f9deaf0f 100644 --- a/mlrun/__init__.py +++ b/mlrun/__init__.py @@ -23,8 +23,15 @@ # flake8: noqa - this is until we take care of the F401 violations with respect to __all__ & sphinx -__version__ = "0.5.2" +from os import environ, path +from .config import config as mlconf +from .datastore import DataItem +from .db import get_run_db +from .execution import MLClientCtx +from .model import RunTemplate, NewTask, RunObject +from .platforms import mount_v3io, v3io_cred +from .projects import load_project, new_project from .run import ( get_or_create_ctx, new_function, @@ -37,16 +44,10 @@ get_pipeline, wait_for_pipeline_completion, ) -from .db import get_run_db -from .model import RunTemplate, NewTask, RunObject -from .config import config as mlconf from .runtimes import new_model_server -from .platforms import mount_v3io, v3io_cred -from .projects import load_project, new_project -from .datastore import DataItem -from .execution import MLClientCtx +from .utils.version import Version -from os import environ, path +__version__ = Version().get()["version"] def get_version(): diff --git a/mlrun/__main__.py b/mlrun/__main__.py index 5e8d3c6b145..93de0ed51af 100644 --- a/mlrun/__main__.py +++ b/mlrun/__main__.py @@ -22,21 +22,20 @@ from pprint import pprint from subprocess import Popen from sys import executable + import click import yaml from tabulate import tabulate - -from .projects import load_project -from .secrets import SecretsStore -from . import get_version -from .config import config as mlconf from .builder import upload_tarball +from .config import config as mlconf from .db import get_run_db from .k8s_utils import K8sHelper from .model import RunTemplate +from .projects import load_project from .run import new_function, import_function_to_dict, import_function, get_object from .runtimes import RemoteRuntime, RunError, RuntimeKinds +from .secrets import SecretsStore from .utils import ( list2dict, logger, @@ -48,6 +47,7 @@ pr_comment, RunNotifications, ) +from .utils.version import Version @click.group() @@ -253,7 +253,7 @@ def run( ) if kfp or runobj.spec.verbose or verbose: - print("MLRun version: {}".format(get_version())) + print("MLRun version: {}".format(str(Version().get()))) print("Runtime:") pprint(runtime) print("Run:") @@ -587,7 +587,7 @@ def db(port, dirpath): @main.command() def version(): """get mlrun version""" - print("MLRun version: {}".format(get_version())) + print("MLRun version: {}".format(str(Version().get()))) @main.command() diff --git a/mlrun/builder.py b/mlrun/builder.py index 8be9a43e93a..7f5a263af63 100644 --- a/mlrun/builder.py +++ b/mlrun/builder.py @@ -20,7 +20,7 @@ from .datastore import store_manager from .k8s_utils import BasePod, get_k8s_helper -from .utils import logger, normalize_name, tag_image +from .utils import logger, normalize_name, enrich_image_url from .config import config @@ -251,7 +251,7 @@ def build_runtime(runtime, with_mlrun, interactive=False): extra = None name = normalize_name("mlrun-build-{}".format(runtime.metadata.name)) - base_image = tag_image(build.base_image or "mlrun/mlrun") + base_image = enrich_image_url(build.base_image or "mlrun/mlrun") if not build.base_image: with_mlrun = False diff --git a/mlrun/config.py b/mlrun/config.py index cbceb24dc39..335bf8dca31 100644 --- a/mlrun/config.py +++ b/mlrun/config.py @@ -28,8 +28,6 @@ from os.path import expanduser from threading import Lock -from . import __version__ - import yaml env_prefix = "MLRUN_" @@ -47,6 +45,7 @@ "remote_host": "", "version": "", # will be set to current version "images_tag": "", # tag to use with mlrun images e.g. mlrun/mlrun (defaults to version) + "images_registry": "", # registry to use with mlrun images e.g. quay.io/ (defaults to empty, for dockerhub) "kfp_ttl": "14400", # KFP ttl in sec, after that completed PODs will be deleted "kfp_image": "", # image to use for KFP runner (defaults to mlrun/mlrun) "kaniko_version": "v0.19.0", # kaniko builder version @@ -131,6 +130,27 @@ def dump_yaml(self, stream=None): def reload(): _populate() + @property + def version(self): + # importing here to avoid circular dependency + from mlrun.utils.version import Version + + return Version().get()["version"] + + @property + def kfp_image(self): + """ + When this configuration is not set we want to set it to mlrun/mlrun, but we need to use the enrich_image method. + The problem is that the mlrun.utils.helpers module is importing the config (this) module, so we must import the + module inside this function (and not on initialization), and then calculate this property value here. + """ + if not self._kfp_image: + # importing here to avoid circular dependency + import mlrun.utils.helpers + + return mlrun.utils.helpers.enrich_image_url("mlrun/mlrun") + return self._kfp_image + # Global configuration config = Config(default_config) @@ -166,6 +186,11 @@ def _do_populate(env=None): if data: config.update(data) + # HACK to enable kfp_image property to both have dynamic default and to use the value from dict/env like + # other configurations + config._cfg["_kfp_image"] = config._cfg["kfp_image"] + del config._cfg["kfp_image"] + def _convert_str(value, typ): if typ in (str, _none_type): @@ -228,12 +253,6 @@ def read_env(env=None, prefix=env_prefix): if igz_domain: config["ui_url"] = "https://mlrun-ui.{}".format(igz_domain) - if not config.get("kfp_image"): - tag = __version__ or "latest" - config["kfp_image"] = "mlrun/mlrun:{}".format(tag) - - config["version"] = __version__ - return config diff --git a/mlrun/runtimes/base.py b/mlrun/runtimes/base.py index efb6a30d6ef..d56cbd899b3 100644 --- a/mlrun/runtimes/base.py +++ b/mlrun/runtimes/base.py @@ -50,7 +50,7 @@ logger, is_ipython, now_date, - tag_image, + enrich_image_url, dict_to_yaml, dict_to_json, ) @@ -586,7 +586,7 @@ def _force_handler(self, handler): def full_image_path(self, image=None): image = image or self.spec.image or "" - image = tag_image(image) + image = enrich_image_url(image) if not image.startswith("."): return image if "DEFAULT_DOCKER_REGISTRY" in environ: diff --git a/mlrun/runtimes/function.py b/mlrun/runtimes/function.py index 6f1f9ead62c..56cf203af2e 100644 --- a/mlrun/runtimes/function.py +++ b/mlrun/runtimes/function.py @@ -32,7 +32,7 @@ ) from .base import RunError, FunctionStatus from .utils import log_std, set_named_item, get_item_name -from ..utils import logger, update_in, get_in, tag_image +from ..utils import logger, update_in, get_in, enrich_image_url from ..lists import RunList from ..model import RunObject from ..config import config as mlconf @@ -323,7 +323,7 @@ def get_fullname(config, name, project, tag): update_in(config, "spec.volumes", self.spec.to_nuclio_vol()) base_image = get_in(config, "spec.build.baseImage") if base_image: - update_in(config, "spec.build.baseImage", tag_image(base_image)) + update_in(config, "spec.build.baseImage", enrich_image_url(base_image)) logger.info("deploy started") name = get_fullname(config, self.metadata.name, project, tag) diff --git a/mlrun/utils/helpers.py b/mlrun/utils/helpers.py index 1959dcd0e72..0b0518fce35 100644 --- a/mlrun/utils/helpers.py +++ b/mlrun/utils/helpers.py @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import time import hashlib import json -import sys import re +import sys +import time from datetime import datetime, timezone from os import path, environ @@ -27,8 +27,9 @@ from tabulate import tabulate from yaml.representer import RepresenterError -from ..config import config +import mlrun.utils.version.version from .logger import create_logger +from ..config import config yaml.Dumper.ignore_aliases = lambda *args: True _missing = object() @@ -417,13 +418,15 @@ def _set_artifact_path(task): return conf -def tag_image(base: str): - ver = config.images_tag or config.version - if ver and ( - base == "mlrun/mlrun" or (base.startswith("mlrun/ml-") and ":" not in base) - ): - base += ":" + ver - return base +def enrich_image_url(image_url: str) -> str: + tag = config.images_tag or mlrun.utils.version.Version().get()["version"] + registry = config.images_registry + if image_url.startswith("mlrun/") or "/mlrun/" in image_url: + if tag and ":" not in image_url: + image_url = f"{image_url}:{tag}" + if registry and "/mlrun/" not in image_url: + image_url = f"{registry}{image_url}" + return image_url def get_artifact_target(item: dict, project=None): diff --git a/mlrun/utils/singleton.py b/mlrun/utils/singleton.py new file mode 100644 index 00000000000..3776cb92d12 --- /dev/null +++ b/mlrun/utils/singleton.py @@ -0,0 +1,7 @@ +class Singleton(type): + _instances = {} + + def __call__(cls, *args, **kwargs): + if cls not in cls._instances: + cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) + return cls._instances[cls] diff --git a/mlrun/utils/version/__init__.py b/mlrun/utils/version/__init__.py new file mode 100644 index 00000000000..e3aaa40a36a --- /dev/null +++ b/mlrun/utils/version/__init__.py @@ -0,0 +1 @@ +from .version import Version # noqa: F401 diff --git a/mlrun/utils/version/version.py b/mlrun/utils/version/version.py new file mode 100644 index 00000000000..cc33b397efa --- /dev/null +++ b/mlrun/utils/version/version.py @@ -0,0 +1,23 @@ +import importlib.resources +import json + +import mlrun.utils +from mlrun.utils.singleton import Singleton + + +class Version(metaclass=Singleton): + def __init__(self): + # When installing un-released version (e.g. by doing pip install git+https://github.com/mlrun/mlrun@development) + # it won't have a version file, so adding some sane defaults + self.version_info = {"git_commit": "unknown", "version": "unstable"} + try: + self.version_info = json.loads( + importlib.resources.read_text("mlrun.utils.version", "version.json") + ) + except Exception: + mlrun.utils.logger.warning( + "Failed resolving version info. Ignoring and using defaults" + ) + + def get(self): + return self.version_info diff --git a/setup.py b/setup.py index eddcee5c492..7641ed911c1 100644 --- a/setup.py +++ b/setup.py @@ -16,14 +16,23 @@ from setuptools import setup except ImportError: from distutils.core import setup +import json +import logging + +logger = logging.Logger(name="mlrun-setup", level="INFO") def version(): - with open("mlrun/__init__.py") as fp: - for line in fp: - if "__version__" in line: - _, version = line.split("=") - return version.replace('"', "").strip() + try: + with open("mlrun/utils/version/version.json") as version_file: + version_metadata = json.load(version_file) + return version_metadata["version"] + except (ValueError, KeyError, FileNotFoundError): + # When installing un-released version (e.g. by doing + # pip install git+https://github.com/mlrun/mlrun@development) + # it won't have a version file, so adding some sane default + logger.warning("Failed resolving version. Ignoring and using unstable") + return "unstable" def is_ignored(line): @@ -65,6 +74,7 @@ def load_deps(path): "mlrun.projects", "mlrun.artifacts", "mlrun.utils", + "mlrun.utils.version", "mlrun.datastore", "mlrun.api", "mlrun.api.api", diff --git a/tests/utils/test_helpers.py b/tests/utils/test_helpers.py index de697e1166b..0ca50be934f 100644 --- a/tests/utils/test_helpers.py +++ b/tests/utils/test_helpers.py @@ -1,4 +1,5 @@ -from mlrun.utils.helpers import verify_field_regex, extend_hub_uri +from mlrun.config import config +from mlrun.utils.helpers import verify_field_regex, extend_hub_uri, enrich_image_url from mlrun.utils.regex import run_name @@ -72,3 +73,49 @@ def test_extend_hub_uri(): expected_output = case["expected_output"] output = extend_hub_uri(input_uri) assert expected_output == output + + +def test_enrich_image(): + cases = [ + { + "image": "mlrun/mlrun", + "expected_output": "ghcr.io/mlrun/mlrun:0.5.2-unstable-adsf76s", + }, + { + "image": "mlrun/mlrun:some_tag", + "expected_output": "ghcr.io/mlrun/mlrun:some_tag", + }, + { + "image": "quay.io/mlrun/mlrun", + "expected_output": "quay.io/mlrun/mlrun:0.5.2-unstable-adsf76s", + }, + { + "image": "quay.io/mlrun/mlrun:some_tag", + "expected_output": "quay.io/mlrun/mlrun:some_tag", + }, + { + "image": "mlrun/ml-models", + "expected_output": "ghcr.io/mlrun/ml-models:0.5.2-unstable-adsf76s", + }, + { + "image": "mlrun/ml-models:some_tag", + "expected_output": "ghcr.io/mlrun/ml-models:some_tag", + }, + { + "image": "quay.io/mlrun/ml-models", + "expected_output": "quay.io/mlrun/ml-models:0.5.2-unstable-adsf76s", + }, + { + "image": "quay.io/mlrun/ml-models:some_tag", + "expected_output": "quay.io/mlrun/ml-models:some_tag", + }, + {"image": "fake_mlrun/ml-models", "expected_output": "fake_mlrun/ml-models"}, + {"image": "some_repo/some_image", "expected_output": "some_repo/some_image"}, + ] + config.images_registry = "ghcr.io/" + config.images_tag = "0.5.2-unstable-adsf76s" + for case in cases: + image = case["image"] + expected_output = case["expected_output"] + output = enrich_image_url(image) + assert expected_output == output