[serve] Add --in-place/-i flag to serve deploy (ray-project#42965)

Enables faster in-place updates.

Also refactored to shove the options into a dataclass to avoid changing all of the call signatures for every added param.

---------

Signed-off-by: Edward Oakes <ed.nmi.oakes@gmail.com>
Signed-off-by: tterrysun <terry@anyscale.com>

python/ray/serve/_private/deploy_provider.py

@@ -1,5 +1,6 @@
 import os
 from abc import ABC, abstractmethod
+from dataclasses import dataclass
 from subprocess import CalledProcessError, check_output
 from tempfile import NamedTemporaryFile
 from typing import Callable, Optional
@@ -17,6 +18,14 @@
 DEPLOY_PROVIDER_ENV_VAR = "RAY_SERVE_DEPLOY_PROVIDER"
 
 
+@dataclass(frozen=True)
+class DeployOptions:
+    address: str
+    name: Optional[str] = None
+    base_image: Optional[str] = None
+    in_place: bool = False
+
+
 class DeployProvider(ABC):
     @abstractmethod
     def supports_local_uris(self) -> bool:
@@ -31,9 +40,7 @@ def deploy(
         self,
         config: ServeDeploySchema,
         *,
-        address: str,
-        name: Optional[str],
-        base_image: Optional[str] = None,
+        options: DeployOptions,
     ):
         """The primary method providers must implement to deploy a Serve config."""
         pass
@@ -79,17 +86,15 @@ def deploy(
         self,
         config: ServeDeploySchema,
         *,
-        address: str,
-        name: Optional[str],
-        base_image: Optional[str] = None,
+        options: DeployOptions,
     ):
-        if base_image is not None:
+        if options.base_image is not None:
             raise ValueError(
                 "`--base-image` is not supported when using the 'local' deploy "
                 "provider because it deploys to an existing Ray cluster."
             )
 
-        ServeSubmissionClient(address).deploy_applications(
+        ServeSubmissionClient(options.address).deploy_applications(
             config.dict(exclude_unset=True),
         )
         cli_logger.success(
@@ -109,17 +114,17 @@ def deploy(
         self,
         config: ServeDeploySchema,
         *,
-        address: str,
-        name: Optional[str],
-        base_image: Optional[str] = None,
+        options: DeployOptions,
     ):
         service_config = {
             "ray_serve_config": config.dict(exclude_unset=True),
         }
-        if name is not None:
-            service_config["name"] = name
-        if base_image is not None:
-            service_config["cluster_env"] = base_image
+        if options.name is not None:
+            service_config["name"] = options.name
+        if options.base_image is not None:
+            service_config["cluster_env"] = options.base_image
+        if options.in_place:
+            service_config["rollout_strategy"] = "IN_PLACE"
 
         # TODO(edoakes): use the Anyscale SDK (or another fixed entrypoint) instead of
         # subprocessing out to the CLI.

python/ray/serve/scripts.py

@@ -28,6 +28,7 @@
 )
 from ray.serve._private.deploy_provider import (
     DEPLOY_PROVIDER_ENV_VAR,
+    DeployOptions,
     get_deploy_provider,
 )
 from ray.serve._private.deployment_graph_build import build as pipeline_build
@@ -349,6 +350,15 @@ def _generate_config_from_file_or_import_path(
     type=str,
     help=RAY_DASHBOARD_ADDRESS_HELP_STR + " Only used by the 'local' provider.",
 )
+@click.option(
+    "--in-place",
+    "-i",
+    is_flag=True,
+    help=(
+        "Update the application(s) in-place in the same cluster rather than starting "
+        "a new cluster. The 'local' provider always performs in-place updates."
+    ),
+)
 def deploy(
     config_or_import_path: str,
     arguments: Tuple[str],
@@ -359,6 +369,7 @@ def deploy(
     base_image: Optional[str],
     provider: Optional[str],
     address: str,
+    in_place: bool,
 ):
     args_dict = convert_args_to_dict(arguments)
     final_runtime_env = parse_runtime_env_args(
@@ -387,9 +398,12 @@ def deploy(
 
     deploy_provider.deploy(
         config,
-        address=address,
-        name=name,
-        base_image=base_image,
+        options=DeployOptions(
+            address=address,
+            name=name,
+            base_image=base_image,
+            in_place=in_place,
+        ),
     )
 
 

python/ray/serve/tests/unit/fake_deploy_provider.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from ray.serve._private.deploy_provider import DeployProvider
+from ray.serve._private.deploy_provider import DeployOptions, DeployProvider
 from ray.serve.schema import ServeDeploySchema
 
 
@@ -18,23 +18,12 @@ def supports_local_uris(self):
         return self._supports_local_uris
 
     def reset(self):
-        self.deployed_config = None
-        self.deployed_address = None
-        self.deployed_name = None
-        self.deployed_base_image = None
-
-    def deploy(
-        self,
-        config: ServeDeploySchema,
-        *,
-        address: str,
-        name: Optional[str],
-        base_image: Optional[str] = None,
-    ):
+        self.deployed_config: Optional[ServeDeploySchema] = None
+        self.deployed_options: Optional[DeployOptions] = None
+
+    def deploy(self, config: ServeDeploySchema, *, options: DeployOptions):
         self.deployed_config = config
-        self.deployed_address = address
-        self.deployed_name = name
-        self.deployed_base_image = base_image
+        self.deployed_options = options
 
 
 DEPLOY_PROVIDER_SINGLETON = None

python/ray/serve/tests/unit/test_cli.py

@@ -8,6 +8,7 @@
 import yaml
 from click.testing import CliRunner
 
+from ray.serve._private.deploy_provider import DeployOptions
 from ray.serve.schema import ServeApplicationSchema, _skip_validating_runtime_env_uris
 from ray.serve.scripts import convert_args_to_dict, deploy
 from ray.serve.tests.unit.fake_deploy_provider import get_ray_serve_deploy_provider
@@ -44,9 +45,9 @@ def test_deploy_basic(self):
                 runtime_env={},
             )
         ]
-        assert deploy_provider.deployed_address == "http://localhost:8265"
-        assert deploy_provider.deployed_name is None
-        assert deploy_provider.deployed_base_image is None
+        assert deploy_provider.deployed_options == DeployOptions(
+            address="http://localhost:8265",
+        )
 
     def test_deploy_with_address(self):
         deploy_provider = get_ray_serve_deploy_provider()
@@ -65,9 +66,9 @@ def test_deploy_with_address(self):
                 runtime_env={},
             )
         ]
-        assert deploy_provider.deployed_address == "http://magic.com"
-        assert deploy_provider.deployed_name is None
-        assert deploy_provider.deployed_base_image is None
+        assert deploy_provider.deployed_options == DeployOptions(
+            address="http://magic.com",
+        )
 
     def test_deploy_with_name(self):
         deploy_provider = get_ray_serve_deploy_provider()
@@ -86,9 +87,10 @@ def test_deploy_with_name(self):
                 runtime_env={},
             )
         ]
-        assert deploy_provider.deployed_address == "http://localhost:8265"
-        assert deploy_provider.deployed_name == "test-name"
-        assert deploy_provider.deployed_base_image is None
+        assert deploy_provider.deployed_options == DeployOptions(
+            address="http://localhost:8265",
+            name="test-name",
+        )
 
     def test_deploy_with_base_image(self):
         deploy_provider = get_ray_serve_deploy_provider()
@@ -107,9 +109,10 @@ def test_deploy_with_base_image(self):
                 runtime_env={},
             )
         ]
-        assert deploy_provider.deployed_address == "http://localhost:8265"
-        assert deploy_provider.deployed_name is None
-        assert deploy_provider.deployed_base_image == "test-image"
+        assert deploy_provider.deployed_options == DeployOptions(
+            address="http://localhost:8265",
+            base_image="test-image",
+        )
 
     def test_deploy_with_args(self):
         deploy_provider = get_ray_serve_deploy_provider()
@@ -127,9 +130,9 @@ def test_deploy_with_args(self):
                 runtime_env={},
             )
         ]
-        assert deploy_provider.deployed_address == "http://localhost:8265"
-        assert deploy_provider.deployed_name is None
-        assert deploy_provider.deployed_base_image is None
+        assert deploy_provider.deployed_options == DeployOptions(
+            address="http://localhost:8265",
+        )
 
     @pytest.mark.skipif(sys.platform == "win32", reason="Tempfile not working.")
     @pytest.mark.parametrize(
@@ -177,9 +180,9 @@ def test_deploy_with_runtime_env(
                     runtime_env=runtime_env,
                 )
             ]
-        assert deploy_provider.deployed_address == "http://localhost:8265"
-        assert deploy_provider.deployed_name is None
-        assert deploy_provider.deployed_base_image is None
+        assert deploy_provider.deployed_options == DeployOptions(
+            address="http://localhost:8265",
+        )
 
     @pytest.mark.parametrize("supported", [False, True])
     def test_deploy_provider_supports_runtime_env_local_uri(self, supported: bool):

python/ray/serve/tests/unit/test_deploy_provider.py

@@ -6,9 +6,11 @@
 from ray.serve._private.deploy_provider import (
     DEPLOY_PROVIDER_ENV_VAR,
     AnyscaleDeployProvider,
+    DeployOptions,
     LocalDeployProvider,
     get_deploy_provider,
 )
+from ray.serve.schema import ServeDeploySchema
 from ray.serve.tests.unit.fake_deploy_provider import FakeDeployProvider
 
 
@@ -33,16 +35,17 @@ def test_get_custom_deploy_provider(from_env_var: bool):
         deploy_provider = get_deploy_provider(import_path)
 
     assert isinstance(deploy_provider, FakeDeployProvider)
-    deploy_provider.deploy(
-        {},
+
+    config = ServeDeploySchema(applications=[])
+    options = DeployOptions(
         address="http://localhost:8265",
         name="test-name",
         base_image="test-image",
+        in_place=True,
     )
-    assert deploy_provider.deployed_config == {}
-    assert deploy_provider.deployed_address == "http://localhost:8265"
-    assert deploy_provider.deployed_name == "test-name"
-    assert deploy_provider.deployed_base_image == "test-image"
+    deploy_provider.deploy(config, options=options)
+    assert deploy_provider.deployed_config == config
+    assert deploy_provider.deployed_options == options
 
 
 def test_get_nonexistent_custom_deploy_provider():