[Runtimes] Add flag to allow printing build logs only on failure (#1777)

mlrun/runtimes/base.py

@@ -428,7 +428,7 @@ def run(
                 logger.info(
                     "Function is not deployed and auto_build flag is set, starting deploy..."
                 )
-                self.deploy(skip_deployed=True)
+                self.deploy(skip_deployed=True, show_on_failure=True)
             else:
                 raise RunError(
                     "function image is not built/ready, use .deploy() method first"

mlrun/runtimes/daskjob.py

@@ -378,14 +378,27 @@ def deploy(
         skip_deployed=False,
         is_kfp=False,
         mlrun_version_specifier=None,
+        show_on_failure: bool = False,
     ):
-        """deploy function, build container with dependencies"""
+        """deploy function, build container with dependencies
+
+        :param watch:      wait for the deploy to complete (and print build logs)
+        :param with_mlrun: add the current mlrun package to the container build
+        :param skip_deployed: skip the build if we already have an image for the function
+        :param mlrun_version_specifier:  which mlrun package version to include (if not current)
+        :param builder_env:   Kaniko builder pod env vars dict (for config/credentials)
+                              e.g. builder_env={"GIT_TOKEN": token}
+        :param show_on_failure:  show logs only in case of build failure
+
+        :return True if the function is ready (deployed)
+        """
         return super().deploy(
             watch,
             with_mlrun,
             skip_deployed,
             is_kfp=is_kfp,
             mlrun_version_specifier=mlrun_version_specifier,
+            show_on_failure=show_on_failure,
         )
 
     def gpus(self, gpus, gpu_type="nvidia.com/gpu"):

mlrun/runtimes/kubejob.py

@@ -126,6 +126,7 @@ def deploy(
         is_kfp=False,
         mlrun_version_specifier=None,
         builder_env: dict = None,
+        show_on_failure: bool = False,
     ) -> bool:
         """deploy function, build container with dependencies
 
@@ -135,6 +136,7 @@ def deploy(
         :param mlrun_version_specifier:  which mlrun package version to include (if not current)
         :param builder_env:   Kaniko builder pod env vars dict (for config/credentials)
                               e.g. builder_env={"GIT_TOKEN": token}
+        :param show_on_failure:  show logs only in case of build failure
 
         :return True if the function is ready (deployed)
         """
@@ -177,7 +179,7 @@ def deploy(
                     f"Started building image: {data.get('data', {}).get('spec', {}).get('build', {}).get('image')}"
                 )
             if watch and not ready:
-                state = self._build_watch(watch)
+                state = self._build_watch(watch, show_on_failure=show_on_failure)
                 ready = state == "ready"
                 self.status.state = state
         else:
@@ -196,24 +198,35 @@ def deploy(
             raise mlrun.errors.MLRunRuntimeError("Deploy failed")
         return ready
 
-    def _build_watch(self, watch=True, logs=True):
+    def _build_watch(self, watch=True, logs=True, show_on_failure=False):
         db = self._get_db()
         offset = 0
         try:
             text, _ = db.get_builder_status(self, 0, logs=logs)
         except RunDBError:
             raise ValueError("function or build process not found")
 
-        if text:
-            print(text)
+        def print_log(text):
+            if text and (not show_on_failure or self.status.state == "error"):
+                print(text, end="")
+
+        print_log(text)
+        offset += len(text)
         if watch:
             while self.status.state in ["pending", "running"]:
-                offset += len(text)
                 time.sleep(2)
-                text, _ = db.get_builder_status(self, offset, logs=logs)
-                if text:
-                    print(text, end="")
+                if show_on_failure:
+                    text = ""
+                    db.get_builder_status(self, 0, logs=False)
+                    if self.status.state == "error":
+                        # re-read the full log on failure
+                        text, _ = db.get_builder_status(self, offset, logs=logs)
+                else:
+                    text, _ = db.get_builder_status(self, offset, logs=logs)
+                print_log(text)
+                offset += len(text)
 
+        print()
         return self.status.state
 
     def builder_status(self, watch=True, logs=True):

mlrun/runtimes/remotesparkjob.py

@@ -157,8 +157,20 @@ def deploy(
         skip_deployed=False,
         is_kfp=False,
         mlrun_version_specifier=None,
+        show_on_failure: bool = False,
     ):
-        """deploy function, build container with dependencies"""
+        """deploy function, build container with dependencies
+
+        :param watch:      wait for the deploy to complete (and print build logs)
+        :param with_mlrun: add the current mlrun package to the container build
+        :param skip_deployed: skip the build if we already have an image for the function
+        :param mlrun_version_specifier:  which mlrun package version to include (if not current)
+        :param builder_env:   Kaniko builder pod env vars dict (for config/credentials)
+                              e.g. builder_env={"GIT_TOKEN": token}
+        :param show_on_failure:  show logs only in case of build failure
+
+        :return True if the function is ready (deployed)
+        """
         # connect will populate the config from the server config
         if not self.spec.build.base_image:
             self.spec.build.base_image = self._resolve_default_base_image
@@ -168,6 +180,7 @@ def deploy(
             skip_deployed=skip_deployed,
             is_kfp=is_kfp,
             mlrun_version_specifier=mlrun_version_specifier,
+            show_on_failure=show_on_failure,
         )
 
 

mlrun/runtimes/sparkjob/abstract.py

@@ -236,8 +236,20 @@ def deploy(
         skip_deployed=False,
         is_kfp=False,
         mlrun_version_specifier=None,
+        show_on_failure: bool = False,
     ):
-        """deploy function, build container with dependencies"""
+        """deploy function, build container with dependencies
+
+        :param watch:      wait for the deploy to complete (and print build logs)
+        :param with_mlrun: add the current mlrun package to the container build
+        :param skip_deployed: skip the build if we already have an image for the function
+        :param mlrun_version_specifier:  which mlrun package version to include (if not current)
+        :param builder_env:   Kaniko builder pod env vars dict (for config/credentials)
+                              e.g. builder_env={"GIT_TOKEN": token}
+        :param show_on_failure:  show logs only in case of build failure
+
+        :return True if the function is ready (deployed)
+        """
         # connect will populate the config from the server config
         get_run_db()
         if not self.spec.build.base_image:
@@ -248,6 +260,7 @@ def deploy(
             skip_deployed=skip_deployed,
             is_kfp=is_kfp,
             mlrun_version_specifier=mlrun_version_specifier,
+            show_on_failure=show_on_failure,
         )
 
     @staticmethod