[Projects] Add ensure_project to CLI commands (#2094)

automation/release_notes/generate.py

@@ -57,6 +57,7 @@ def __init__(
             "Nick Brown": "ihs-nick",
             "Oded Messer": "omesser",
             "Tom Tankilevitch": "tankilevitch",
+            "Adam": "quaark",
         }
 
     def run(self):
@@ -87,7 +88,7 @@ def run(self):
                 args=[
                     "log",
                     '--pretty=format:"%h {%an} %s"',
-                    f"{self._previous_release}..HEAD",
+                    f"{self._previous_release}..{self._release}",
                 ],
                 cwd=repo_dir,
             )
@@ -97,7 +98,7 @@ def run(self):
                 args=[
                     "log",
                     '--pretty=format:"%h %s"',
-                    f"{self._previous_release}..HEAD",
+                    f"{self._previous_release}..{self._release}",
                 ],
                 cwd=repo_dir,
             )

mlrun/__main__.py

@@ -163,12 +163,10 @@ def main():
     help="when set functions will be built prior to run if needed",
 )
 @click.argument("run_args", nargs=-1, type=click.UNPROCESSED)
-# this is not a flag because we want the default to be True and be able to override to False
 @click.option(
-    "--save-project",
-    type=bool,
-    default=True,
-    help="save the project to MLRun DB when loading it",
+    "--ensure-project",
+    is_flag=True,
+    help="ensure the project exists, if not, create project",
 )
 def run(
     url,
@@ -211,7 +209,7 @@ def run(
     env_file,
     auto_build,
     run_args,
-    save_project,
+    ensure_project,
 ):
     """Execute a task and inject parameters."""
 
@@ -257,9 +255,14 @@ def run(
         if len(split) > 1:
             url_args = split[1]
 
+    if ensure_project and project:
+        mlrun.get_or_create_project(
+            name=project,
+            context="./",
+        )
     if func_url or kind or image:
         if func_url:
-            runtime = func_url_to_runtime(func_url, save_project)
+            runtime = func_url_to_runtime(func_url, ensure_project)
             kind = get_in(runtime, "kind", kind or "job")
             if runtime is None:
                 exit(1)
@@ -412,6 +415,11 @@ def run(
 @click.option(
     "--env-file", default="", help="path to .env file to load config/variables from"
 )
+@click.option(
+    "--ensure-project",
+    is_flag=True,
+    help="ensure the project exists, if not, create project",
+)
 def build(
     func_url,
     name,
@@ -430,6 +438,7 @@ def build(
     kfp,
     skip,
     env_file,
+    ensure_project,
 ):
     """Build a container image from code and requirements."""
 
@@ -495,6 +504,13 @@ def build(
         b.source = target
 
     with_mlrun = True if with_mlrun else None  # False will map to None
+
+    if ensure_project and project:
+        mlrun.get_or_create_project(
+            name=project,
+            context="./",
+        )
+
     if hasattr(func, "deploy"):
         logger.info("remote deployment started")
         try:
@@ -542,12 +558,10 @@ def build(
 @click.option(
     "--env-file", default="", help="path to .env file to load config/variables from"
 )
-# this is not a flag because we want the default to be True and be able to override to False
 @click.option(
-    "--save-project",
-    type=bool,
-    default=True,
-    help="save the project to MLRun DB when loading it",
+    "--ensure-project",
+    is_flag=True,
+    help="ensure the project exists, if not, create project",
 )
 def deploy(
     spec,
@@ -561,14 +575,20 @@ def deploy(
     env,
     verbose,
     env_file,
-    save_project,
+    ensure_project,
 ):
     """Deploy model or function"""
     if env_file:
         mlrun.set_env_from_file(env_file)
 
+    if ensure_project and project:
+        mlrun.get_or_create_project(
+            name=project,
+            context="./",
+        )
+
     if func_url:
-        runtime = func_url_to_runtime(func_url, save_project)
+        runtime = func_url_to_runtime(func_url, ensure_project)
         if runtime is None:
             exit(1)
     elif spec:
@@ -853,6 +873,11 @@ def logs(uid, project, offset, db, watch):
 @click.option(
     "--env-file", default="", help="path to .env file to load config/variables from"
 )
+@click.option(
+    "--ensure-project",
+    is_flag=True,
+    help="ensure the project exists, if not, create project",
+)
 def project(
     context,
     name,
@@ -876,6 +901,7 @@ def project(
     local,
     env_file,
     timeout,
+    ensure_project,
 ):
     """load and/or run a project"""
     if env_file:
@@ -884,7 +910,9 @@ def project(
     if db:
         mlconf.dbpath = db
 
-    proj = load_project(context, url, name, init_git=init_git, clone=clone)
+    proj = load_project(
+        context, url, name, init_git=init_git, clone=clone, save=ensure_project
+    )
     url_str = " from " + url if url else ""
     print(f"Loading project {proj.name}{url_str} into {context}:\n")
 
@@ -1113,7 +1141,7 @@ def dict_to_str(struct: dict):
     return ",".join([f"{k}={v}" for k, v in struct.items()])
 
 
-def func_url_to_runtime(func_url, save_project: bool = True):
+def func_url_to_runtime(func_url, ensure_project: bool = False):
     try:
         if func_url.startswith("db://"):
             func_url = func_url[5:]
@@ -1124,7 +1152,7 @@ def func_url_to_runtime(func_url, save_project: bool = True):
             func_url = "function.yaml" if func_url == "." else func_url
             runtime = import_function_to_dict(func_url, {})
         else:
-            mlrun_project = load_project(".", save=save_project)
+            mlrun_project = load_project(".", save=ensure_project)
             function = mlrun_project.get_function(func_url, enrich=True)
             if function.kind == "local":
                 command, function = load_func_code(function)

mlrun/api/utils/projects/follower.py

@@ -83,6 +83,7 @@ def create_project(
         projects_role: typing.Optional[mlrun.api.schemas.ProjectsRole] = None,
         leader_session: typing.Optional[str] = None,
         wait_for_completion: bool = True,
+        commit_before_get: bool = False,
     ) -> typing.Tuple[typing.Optional[mlrun.api.schemas.Project], bool]:
         if self._is_request_from_leader(projects_role):
             mlrun.api.crud.Projects().create_project(db_session, project)
@@ -107,7 +108,8 @@ def create_project(
                 # https://dev.mysql.com/doc/refman/8.0/en/innodb-transaction-isolation-levels.html
                 # TODO: there are multiple isolation level we can choose, READ COMMITTED seems to solve our issue
                 #  but will require deeper investigation and more test coverage
-                db_session.commit()
+                if commit_before_get:
+                    db_session.commit()
 
                 created_project = self.get_project(
                     db_session, project.metadata.name, leader_session
@@ -136,6 +138,7 @@ def store_project(
                     projects_role,
                     leader_session,
                     wait_for_completion,
+                    commit_before_get=True,
                 )
             else:
                 self._leader_client.update_project(leader_session, name, project)

mlrun/api/utils/projects/leader.py

@@ -47,6 +47,7 @@ def create_project(
         projects_role: typing.Optional[mlrun.api.schemas.ProjectsRole] = None,
         leader_session: typing.Optional[str] = None,
         wait_for_completion: bool = True,
+        commit_before_get: bool = False,
     ) -> typing.Tuple[typing.Optional[mlrun.api.schemas.Project], bool]:
         self._enrich_and_validate_before_creation(project)
         self._run_on_all_followers(True, "create_project", db_session, project)

mlrun/api/utils/projects/member.py

@@ -40,6 +40,7 @@ def create_project(
         projects_role: typing.Optional[mlrun.api.schemas.ProjectsRole] = None,
         leader_session: typing.Optional[str] = None,
         wait_for_completion: bool = True,
+        commit_before_get: bool = False,
     ) -> typing.Tuple[typing.Optional[mlrun.api.schemas.Project], bool]:
         pass
 

mlrun/projects/project.py

@@ -280,11 +280,11 @@ def get_or_create_project(
         project.pull("development")  # pull the latest code from git
         project.run("main", arguments={'data': data_url})  # run the workflow "main"
 
+    :param name:         project name
     :param context:      project local directory path
     :param url:          name (in DB) or git or tar.gz or .zip sources archive path e.g.:
                          git://github.com/mlrun/demo-xgb-project.git
                          http://mysite/archived-project.zip
-    :param name:         project name
     :param secrets:      key:secret dict or SecretsStore used to download sources
     :param init_git:     if True, will git init the context dir
     :param subpath:      project subpath (within the archive/context)
@@ -366,7 +366,9 @@ def _load_project_dir(context, name="", subpath=""):
             functions=[{"url": "function.yaml", "name": func.metadata.name}],
         )
     else:
-        raise ValueError("project or function YAML not found in path")
+        raise mlrun.errors.MLRunNotFoundError(
+            "project or function YAML not found in path"
+        )
 
     project.spec.context = context
     project.metadata.name = name or project.metadata.name

tests/integration/sdk_api/projects/test_project.py

@@ -1,4 +1,11 @@
+import pathlib
+
+import deepdiff
+import pytest
+
 import mlrun
+import mlrun.api.schemas
+import tests.conftest
 import tests.integration.sdk_api.base
 
 
@@ -16,3 +23,48 @@ def test_load_project_from_db(self):
         project = mlrun.new_project(project_name)
         project.save_to_db()
         mlrun.load_project(".", f"db://{project_name}")
+
+    def test_load_project_with_save(self):
+        project_name = "some-project"
+        project = mlrun.new_project(project_name)
+        project_file_path = pathlib.Path(tests.conftest.results) / "project.yaml"
+        project.export(str(project_file_path))
+
+        imported_project_name = "imported-project"
+        # loaded project but didn't saved
+        mlrun.load_project(
+            "./", str(project_file_path), name=imported_project_name, save=False
+        )
+
+        # loading project from db, but earlier load didn't saved, expected to fail
+        with pytest.raises(mlrun.errors.MLRunNotFoundError):
+            mlrun.load_project(".", f"db://{imported_project_name}", save=False)
+
+        # loading project and saving
+        expected_project = mlrun.load_project(
+            "./", str(project_file_path), name=imported_project_name
+        )
+
+        # loading project from db, expected to succeed
+        loaded_project_from_db = mlrun.load_project(
+            ".", f"db://{imported_project_name}", save=False
+        )
+        _assert_projects(expected_project, loaded_project_from_db)
+
+
+def _assert_projects(expected_project, project):
+    assert (
+        deepdiff.DeepDiff(
+            expected_project.to_dict(),
+            project.to_dict(),
+            ignore_order=True,
+            exclude_paths={
+                "root['metadata']['created']",
+                "root['spec']['desired_state']",
+                "root['status']",
+            },
+        )
+        == {}
+    )
+    assert expected_project.spec.desired_state == project.spec.desired_state
+    assert expected_project.spec.desired_state == project.status.state

tests/projects/test_project.py

@@ -1,6 +1,5 @@
 import os
 import pathlib
-import unittest.mock
 import zipfile
 
 import deepdiff
@@ -34,22 +33,6 @@ def test_sync_functions():
     assert fn.metadata.name == "train", "train func did not return"
 
 
-def test_load_save_project(monkeypatch):
-    project_name = "project-name"
-    project = mlrun.new_project(project_name, save=False)
-    project.set_function("hub://describe", "describe")
-    project_file_path = pathlib.Path(tests.conftest.results) / "project.yaml"
-
-    project.save = unittest.mock.Mock()
-    monkeypatch.setattr(
-        mlrun.projects.project, "_load_project_file", lambda *args, **kwargs: project
-    )
-
-    loaded_project = mlrun.load_project("./", str(project_file_path), save=True)
-    assert project.save.call_count == 1
-    assert loaded_project == project
-
-
 def test_create_project_from_file_with_legacy_structure():
     project_name = "project-name"
     description = "project description"
@@ -252,7 +235,7 @@ def test_function_run_cli():
     )
     project.export()
 
-    args = "-f my-func --local --save-project=False --dump -p x=3".split()
+    args = "-f my-func --local --dump -p x=3".split()
     out = tests.conftest.exec_mlrun(args, str(project_dir_path))
     assert out.find("state: completed") != -1, out
     assert out.find("y: 6") != -1, out  # = x * 2

tests/run/test_main.py

@@ -195,7 +195,7 @@ def test_main_run_archive_subdir():
 
 def test_main_local_project():
     project_path = str(pathlib.Path(__file__).parent / "assets")
-    args = "-f simple -p x=2 --save-project=False --dump"
+    args = "-f simple -p x=2 --dump"
     out = exec_main("run", args.split(), cwd=project_path)
     assert out.find("state: completed") != -1, out
     assert out.find("y: 4") != -1, out  # y = x * 2