feat: Add warning for submissions for requests that cannot currently …

…be fulfilled [DET-6410] (#5376)

e2e_tests/tests/command/command.py

@@ -23,8 +23,13 @@ def __init__(self, process: subprocess.Popen, detach: bool = False):
             self.task_id = line.decode().strip()
         else:
             iterator = iter(self.process.stdout)  # type: ignore
-            line = next(iterator)
-            m = re.search(rb"Scheduling .* \(id: (.*)\)", line)
+            m = None
+            max_iterations = 2
+            iterations = 0
+            while not m and iterations < max_iterations:
+                line = next(iterator)
+                iterations += 1
+                m = re.search(rb"Scheduling .* \(id: (.*)\)", line)
             assert m is not None
             self.task_id = m.group(1).decode() if m else None
 

harness/determined/cli/notebook.py

@@ -33,6 +33,11 @@ def start_notebook(args: Namespace) -> None:
         print(nb.id)
         return
 
+    request.handle_warnings(resp.warnings)
+    currentSlotsExceeded = (resp.warnings is not None) and (
+        bindings.v1LaunchWarning.LAUNCH_WARNING_CURRENT_SLOTS_EXCEEDED in resp.warnings
+    )
+
     with api.ws(args.master, "notebooks/{}/events".format(nb.id)) as ws:
         for msg in ws:
             if msg["service_ready_event"] and nb.serviceAddress and not args.no_browser:
@@ -44,6 +49,7 @@ def start_notebook(args: Namespace) -> None:
                         description=nb.description,
                         resource_pool=nb.resourcePool,
                         task_type="notebook",
+                        currentSlotsExceeded=currentSlotsExceeded,
                     ),
                 )
                 print(colored("Jupyter Notebook is running at: {}".format(url), "green"))
@@ -64,6 +70,7 @@ def open_notebook(args: Namespace) -> None:
             description=resp["description"],
             resource_pool=resp["resourcePool"],
             task_type="notebook",
+            currentSlotsExceeded=False,
         ),
     )
 

harness/determined/cli/tensorboard.py

@@ -9,7 +9,7 @@
 from determined import cli
 from determined.cli import command, task
 from determined.common import api, context
-from determined.common.api import authentication, request
+from determined.common.api import authentication, bindings, request
 from determined.common.check import check_eq
 from determined.common.declarative_argparse import Arg, Cmd, Group
 
@@ -21,21 +21,24 @@ def start_tensorboard(args: Namespace) -> None:
         sys.exit(1)
 
     config = command.parse_config(args.config_file, None, args.config, [])
-    req_body = {
-        "config": config,
-        "trial_ids": args.trial_ids,
-        "experiment_ids": args.experiment_ids,
-    }
-
-    req_body["files"] = context.read_legacy_context(args.context, args.include)
+    body = bindings.v1LaunchTensorboardRequest(
+        config=config,
+        trialIds=args.trial_ids,
+        experimentIds=args.experiment_ids,
+        files=context.read_v1_context(args.context, args.include),
+    )
 
-    resp = api.post(args.master, "api/v1/tensorboards", json=req_body).json()["tensorboard"]
+    resp = bindings.post_LaunchTensorboard(cli.setup_session(args), body=body)
 
     if args.detach:
-        print(resp["id"])
+        print(resp.tensorboard.id)
         return
 
-    url = "tensorboard/{}/events".format(resp["id"])
+    request.handle_warnings(resp.warnings)
+    currentSlotsExceeded = (resp.warnings is not None) and (
+        bindings.v1LaunchWarning.LAUNCH_WARNING_CURRENT_SLOTS_EXCEEDED in resp.warnings
+    )
+    url = "tensorboard/{}/events".format(resp.tensorboard.id)
     with api.ws(args.master, url) as ws:
         for msg in ws:
             if msg["log_event"] is not None:
@@ -44,18 +47,19 @@ def start_tensorboard(args: Namespace) -> None:
                 if "http" in msg["log_event"]:
                     continue
 
-            if msg["service_ready_event"]:
+            if msg["service_ready_event"] and resp.tensorboard.serviceAddress is not None:
                 if args.no_browser:
-                    url = api.make_url(args.master, resp["serviceAddress"])
+                    url = api.make_url(args.master, resp.tensorboard.serviceAddress)
                 else:
                     url = api.browser_open(
                         args.master,
                         request.make_interactive_task_url(
-                            task_id=resp["id"],
-                            service_address=resp["serviceAddress"],
-                            resource_pool=resp["resourcePool"],
-                            description=resp["description"],
+                            task_id=resp.tensorboard.id,
+                            service_address=resp.tensorboard.serviceAddress,
+                            resource_pool=resp.tensorboard.resourcePool,
+                            description=resp.tensorboard.description,
                             task_type="tensorboard",
+                            currentSlotsExceeded=currentSlotsExceeded,
                         ),
                     )
 
@@ -80,6 +84,7 @@ def open_tensorboard(args: Namespace) -> None:
             resource_pool=resp["resourcePool"],
             description=resp["description"],
             task_type="tensorboard",
+            currentSlotsExceeded=False,
         ),
     )
 

harness/determined/common/api/__init__.py

@@ -1,7 +1,7 @@
 from determined.common.api import authentication, errors, metric, request
 from determined.common.api._session import Session
 from determined.common.api import bindings
-from determined.common.api._util import PageOpts, read_paginated
+from determined.common.api._util import PageOpts, read_paginated, WARNING_MESSAGE_MAP
 from determined.common.api.authentication import Authentication, salt_and_hash
 from determined.common.api.experiment import (
     create_experiment,

harness/determined/common/api/_util.py

@@ -13,6 +13,14 @@ class PageOpts(str, enum.Enum):
 # The Paginated union type is generated based on response objects with a .pagination attribute.
 T = TypeVar("T", bound=bindings.Paginated)
 
+# Map of launch warnings to the warning message shown to users.
+WARNING_MESSAGE_MAP = {
+    bindings.v1LaunchWarning.LAUNCH_WARNING_CURRENT_SLOTS_EXCEEDED: (
+        "Warning: The requested job requires more slots than currently available. "
+        "You may need to increase cluster resources in order for the job to run."
+    )
+}
+
 
 def read_paginated(
     get_with_offset: Callable[[int], T],

harness/determined/common/api/bindings.py

@@ -1744,29 +1744,37 @@ def to_json(self, omit_unset: bool = False) -> typing.Dict[str, typing.Any]:
         return out
 
 class v1CreateExperimentResponse:
+    warnings: "typing.Optional[typing.Sequence[v1LaunchWarning]]" = None
 
     def __init__(
         self,
         *,
         config: "typing.Dict[str, typing.Any]",
         experiment: "v1Experiment",
+        warnings: "typing.Union[typing.Sequence[v1LaunchWarning], None, Unset]" = _unset,
     ):
         self.config = config
         self.experiment = experiment
+        if not isinstance(warnings, Unset):
+            self.warnings = warnings
 
     @classmethod
     def from_json(cls, obj: Json) -> "v1CreateExperimentResponse":
         kwargs: "typing.Dict[str, typing.Any]" = {
             "config": obj["config"],
             "experiment": v1Experiment.from_json(obj["experiment"]),
         }
+        if "warnings" in obj:
+            kwargs["warnings"] = [v1LaunchWarning(x) for x in obj["warnings"]] if obj["warnings"] is not None else None
         return cls(**kwargs)
 
     def to_json(self, omit_unset: bool = False) -> typing.Dict[str, typing.Any]:
         out: "typing.Dict[str, typing.Any]" = {
             "config": self.config,
             "experiment": self.experiment.to_json(omit_unset),
         }
+        if not omit_unset or "warnings" in vars(self):
+            out["warnings"] = None if self.warnings is None else [x.value for x in self.warnings]
         return out
 
 class v1CreateGroupRequest:
@@ -5284,29 +5292,37 @@ def to_json(self, omit_unset: bool = False) -> typing.Dict[str, typing.Any]:
         return out
 
 class v1LaunchCommandResponse:
+    warnings: "typing.Optional[typing.Sequence[v1LaunchWarning]]" = None
 
     def __init__(
         self,
         *,
         command: "v1Command",
         config: "typing.Dict[str, typing.Any]",
+        warnings: "typing.Union[typing.Sequence[v1LaunchWarning], None, Unset]" = _unset,
     ):
         self.command = command
         self.config = config
+        if not isinstance(warnings, Unset):
+            self.warnings = warnings
 
     @classmethod
     def from_json(cls, obj: Json) -> "v1LaunchCommandResponse":
         kwargs: "typing.Dict[str, typing.Any]" = {
             "command": v1Command.from_json(obj["command"]),
             "config": obj["config"],
         }
+        if "warnings" in obj:
+            kwargs["warnings"] = [v1LaunchWarning(x) for x in obj["warnings"]] if obj["warnings"] is not None else None
         return cls(**kwargs)
 
     def to_json(self, omit_unset: bool = False) -> typing.Dict[str, typing.Any]:
         out: "typing.Dict[str, typing.Any]" = {
             "command": self.command.to_json(omit_unset),
             "config": self.config,
         }
+        if not omit_unset or "warnings" in vars(self):
+            out["warnings"] = None if self.warnings is None else [x.value for x in self.warnings]
         return out
 
 class v1LaunchNotebookRequest:
@@ -5360,29 +5376,37 @@ def to_json(self, omit_unset: bool = False) -> typing.Dict[str, typing.Any]:
         return out
 
 class v1LaunchNotebookResponse:
+    warnings: "typing.Optional[typing.Sequence[v1LaunchWarning]]" = None
 
     def __init__(
         self,
         *,
         config: "typing.Dict[str, typing.Any]",
         notebook: "v1Notebook",
+        warnings: "typing.Union[typing.Sequence[v1LaunchWarning], None, Unset]" = _unset,
     ):
         self.config = config
         self.notebook = notebook
+        if not isinstance(warnings, Unset):
+            self.warnings = warnings
 
     @classmethod
     def from_json(cls, obj: Json) -> "v1LaunchNotebookResponse":
         kwargs: "typing.Dict[str, typing.Any]" = {
             "config": obj["config"],
             "notebook": v1Notebook.from_json(obj["notebook"]),
         }
+        if "warnings" in obj:
+            kwargs["warnings"] = [v1LaunchWarning(x) for x in obj["warnings"]] if obj["warnings"] is not None else None
         return cls(**kwargs)
 
     def to_json(self, omit_unset: bool = False) -> typing.Dict[str, typing.Any]:
         out: "typing.Dict[str, typing.Any]" = {
             "config": self.config,
             "notebook": self.notebook.to_json(omit_unset),
         }
+        if not omit_unset or "warnings" in vars(self):
+            out["warnings"] = None if self.warnings is None else [x.value for x in self.warnings]
         return out
 
 class v1LaunchShellRequest:
@@ -5436,29 +5460,37 @@ def to_json(self, omit_unset: bool = False) -> typing.Dict[str, typing.Any]:
         return out
 
 class v1LaunchShellResponse:
+    warnings: "typing.Optional[typing.Sequence[v1LaunchWarning]]" = None
 
     def __init__(
         self,
         *,
         config: "typing.Dict[str, typing.Any]",
         shell: "v1Shell",
+        warnings: "typing.Union[typing.Sequence[v1LaunchWarning], None, Unset]" = _unset,
     ):
         self.config = config
         self.shell = shell
+        if not isinstance(warnings, Unset):
+            self.warnings = warnings
 
     @classmethod
     def from_json(cls, obj: Json) -> "v1LaunchShellResponse":
         kwargs: "typing.Dict[str, typing.Any]" = {
             "config": obj["config"],
             "shell": v1Shell.from_json(obj["shell"]),
         }
+        if "warnings" in obj:
+            kwargs["warnings"] = [v1LaunchWarning(x) for x in obj["warnings"]] if obj["warnings"] is not None else None
         return cls(**kwargs)
 
     def to_json(self, omit_unset: bool = False) -> typing.Dict[str, typing.Any]:
         out: "typing.Dict[str, typing.Any]" = {
             "config": self.config,
             "shell": self.shell.to_json(omit_unset),
         }
+        if not omit_unset or "warnings" in vars(self):
+            out["warnings"] = None if self.warnings is None else [x.value for x in self.warnings]
         return out
 
 class v1LaunchTensorboardRequest:
@@ -5520,31 +5552,43 @@ def to_json(self, omit_unset: bool = False) -> typing.Dict[str, typing.Any]:
         return out
 
 class v1LaunchTensorboardResponse:
+    warnings: "typing.Optional[typing.Sequence[v1LaunchWarning]]" = None
 
     def __init__(
         self,
         *,
         config: "typing.Dict[str, typing.Any]",
         tensorboard: "v1Tensorboard",
+        warnings: "typing.Union[typing.Sequence[v1LaunchWarning], None, Unset]" = _unset,
     ):
         self.config = config
         self.tensorboard = tensorboard
+        if not isinstance(warnings, Unset):
+            self.warnings = warnings
 
     @classmethod
     def from_json(cls, obj: Json) -> "v1LaunchTensorboardResponse":
         kwargs: "typing.Dict[str, typing.Any]" = {
             "config": obj["config"],
             "tensorboard": v1Tensorboard.from_json(obj["tensorboard"]),
         }
+        if "warnings" in obj:
+            kwargs["warnings"] = [v1LaunchWarning(x) for x in obj["warnings"]] if obj["warnings"] is not None else None
         return cls(**kwargs)
 
     def to_json(self, omit_unset: bool = False) -> typing.Dict[str, typing.Any]:
         out: "typing.Dict[str, typing.Any]" = {
             "config": self.config,
             "tensorboard": self.tensorboard.to_json(omit_unset),
         }
+        if not omit_unset or "warnings" in vars(self):
+            out["warnings"] = None if self.warnings is None else [x.value for x in self.warnings]
         return out
 
+class v1LaunchWarning(enum.Enum):
+    LAUNCH_WARNING_UNSPECIFIED = "LAUNCH_WARNING_UNSPECIFIED"
+    LAUNCH_WARNING_CURRENT_SLOTS_EXCEEDED = "LAUNCH_WARNING_CURRENT_SLOTS_EXCEEDED"
+
 class v1ListRolesRequest:
     offset: "typing.Optional[int]" = None