From 9e696190f9bec13d6b2084cbcb2968c48839d1bd Mon Sep 17 00:00:00 2001 From: peterschmidt85 Date: Mon, 30 Dec 2024 23:16:27 +0100 Subject: [PATCH 1/5] Replace termination_idle_time: int|str with idle_duration: int|str|off #1154 --- docs/docs/concepts/fleets.md | 7 ++--- docs/docs/dev-environments.md | 9 ++++--- docs/docs/guides/protips.md | 9 ++++--- docs/docs/services.md | 9 ++++--- docs/docs/tasks.md | 9 ++++--- frontend/src/types/fleet.d.ts | 6 ++--- .../cli/services/configurators/fleet.py | 1 + src/dstack/_internal/cli/utils/run.py | 25 ++++++++++------- src/dstack/_internal/core/models/fleets.py | 25 ++++++++++++----- src/dstack/_internal/core/models/profiles.py | 24 ++++++++++++++--- src/dstack/_internal/core/models/runs.py | 6 ----- .../_internal/core/services/profiles.py | 27 +++++++++++++++++-- .../tasks/process_submitted_jobs.py | 8 ++++-- src/dstack/_internal/server/models.py | 1 + .../_internal/server/services/fleets.py | 10 +++---- src/dstack/_internal/server/services/pools.py | 9 +++++-- src/dstack/_internal/utils/common.py | 2 ++ src/dstack/api/_public/runs.py | 2 ++ src/dstack/api/server/__init__.py | 2 ++ src/dstack/api/server/_fleets.py | 3 +++ src/dstack/api/server/_runs.py | 3 +++ .../_internal/server/routers/test_fleets.py | 4 +++ .../_internal/server/routers/test_runs.py | 5 ++++ 23 files changed, 146 insertions(+), 60 deletions(-) diff --git a/docs/docs/concepts/fleets.md b/docs/docs/concepts/fleets.md index 92de3c65c..36d6f1d7c 100644 --- a/docs/docs/concepts/fleets.md +++ b/docs/docs/concepts/fleets.md @@ -40,7 +40,7 @@ The filename must end with `.dstack.yml` (e.g. `.dstack.yml` or `fleet.dstack.ym placement: cluster # Terminate if idle for 3 days - termination_idle_time: 3d + idle_duration: 3d resources: gpu: @@ -199,9 +199,10 @@ $ dstack fleet Once the status of instances changes to `idle`, they can be used by dev environments, tasks, and services. -!!! info "Termination policy" +!!! info "Idle duration" If you want a fleet to be automatically deleted after a certain idle time, - you can set the [`termination_idle_time`](../reference/dstack.yml/fleet.md#termination_idle_time) property. + you can set the [`idle_duration`](../reference/dstack.yml/fleet.md#idle_duration) property. + By default, it's set to `3d`. [//]: # (Add Idle time example to the reference page) diff --git a/docs/docs/dev-environments.md b/docs/docs/dev-environments.md index 96055f4b9..7ee94ec6b 100644 --- a/docs/docs/dev-environments.md +++ b/docs/docs/dev-environments.md @@ -136,12 +136,13 @@ $ dstack apply -R -f examples/.dstack.yml Alternatively, set [`creation_policy`](reference/dstack.yml/dev-environment.md#creation_policy) to `reuse` in the run configuration. -### Termination policy +### Idle duration -If a fleet is created automatically, it remains `idle` for 5 minutes and can be reused within that time. +If a fleet is created automatically, it stays `idle` for 5 minutes by default and can be reused within that time. +If the fleet is not reused within this period, it is automatically terminated. To change the default idle duration, set -[`termination_idle_time`](reference/dstack.yml/fleet.md#termination_idle_time) in the run configuration (e.g., to 0 or a -longer duration). +[`idle_duration`](reference/dstack.yml/fleet.md#idle_duration) in the run configuration (e.g., `0s`, `1m`, or `off` for +unlimited). !!! info "Fleets" For greater control over fleet provisioning, configuration, and lifecycle management, it is recommended to use diff --git a/docs/docs/guides/protips.md b/docs/docs/guides/protips.md index 3e18f6294..8171ef2a1 100644 --- a/docs/docs/guides/protips.md +++ b/docs/docs/guides/protips.md @@ -22,12 +22,13 @@ $ dstack apply -R -f examples/.dstack.yml -### Termination policy +### Idle duration -If a fleet is created automatically, it remains `idle` for 5 minutes and can be reused within that time. +If a fleet is created automatically, it stays `idle` for 5 minutes by default and can be reused within that time. +If the fleet is not reused within this period, it is automatically terminated. To change the default idle duration, set -[`termination_idle_time`](../reference/dstack.yml/fleet.md#termination_idle_time) in the run configuration (e.g., to 0 or a -longer duration). +[`idle_duration`](../reference/dstack.yml/fleet.md#idle_duration) in the run configuration (e.g., `0s`, `1m`, or `off` for +unlimited). !!! info "Fleets" For greater control over fleet provisioning, configuration, and lifecycle management, it is recommended to use diff --git a/docs/docs/services.md b/docs/docs/services.md index cc68670a9..4a1599a84 100644 --- a/docs/docs/services.md +++ b/docs/docs/services.md @@ -176,12 +176,13 @@ $ dstack apply -R -f examples/.dstack.yml Alternatively, set [`creation_policy`](reference/dstack.yml/dev-environment.md#creation_policy) to `reuse` in the run configuration. -### Termination policy +### Idle duration -If a fleet is created automatically, it remains `idle` for 5 minutes and can be reused within that time. +If a fleet is created automatically, it stays `idle` for 5 minutes by default and can be reused within that time. +If the fleet is not reused within this period, it is automatically terminated. To change the default idle duration, set -[`termination_idle_time`](reference/dstack.yml/fleet.md#termination_idle_time) in the run configuration (e.g., to 0 or a -longer duration). +[`idle_duration`](reference/dstack.yml/fleet.md#idle_duration) in the run configuration (e.g., `0s`, `1m`, or `off` for +unlimited). !!! info "Fleets" For greater control over fleet provisioning, configuration, and lifecycle management, it is recommended to use diff --git a/docs/docs/tasks.md b/docs/docs/tasks.md index a8b11b4a6..53521a05d 100644 --- a/docs/docs/tasks.md +++ b/docs/docs/tasks.md @@ -141,12 +141,13 @@ $ dstack apply -R -f examples/.dstack.yml Alternatively, set [`creation_policy`](reference/dstack.yml/dev-environment.md#creation_policy) to `reuse` in the run configuration. -### Termination policy +### Idle duration -If a fleet is created automatically, it remains `idle` for 5 minutes and can be reused within that time. +If a fleet is created automatically, it stays `idle` for 5 minutes by default and can be reused within that time. +If the fleet is not reused within this period, it is automatically terminated. To change the default idle duration, set -[`termination_idle_time`](reference/dstack.yml/fleet.md#termination_idle_time) in the run configuration (e.g., to 0 or a -longer duration). +[`idle_duration`](reference/dstack.yml/fleet.md#idle_duration) in the run configuration (e.g., `0s`, `1m`, or `off` for +unlimited). !!! info "Fleets" For greater control over fleet provisioning, configuration, and lifecycle management, it is recommended to use diff --git a/frontend/src/types/fleet.d.ts b/frontend/src/types/fleet.d.ts index 6b19ea98d..6c48a2cac 100644 --- a/frontend/src/types/fleet.d.ts +++ b/frontend/src/types/fleet.d.ts @@ -86,8 +86,7 @@ declare interface IFleetConfigurationRequest { duration?: number | string; } | boolean; max_price?: number; - termination_policy?: "dont-destroy" | "destroy-after-idle"; - termination_idle_time?: number | string; + idle_duration?: number | string; } declare interface IProfileRequest { @@ -105,8 +104,7 @@ declare interface IProfileRequest { pool_name?: string; instance_name?: string; creation_policy?: "reuse" | "reuse-or-create"; - termination_policy?: "dont-destroy" | "destroy-after-idle"; - termination_idle_time?: number | string; + idle_duration?: number | string; name: string; default?: boolean; } diff --git a/src/dstack/_internal/cli/services/configurators/fleet.py b/src/dstack/_internal/cli/services/configurators/fleet.py index defb3b349..194d4a0ff 100644 --- a/src/dstack/_internal/cli/services/configurators/fleet.py +++ b/src/dstack/_internal/cli/services/configurators/fleet.py @@ -303,6 +303,7 @@ def th(s: str) -> str: configuration_table.add_row(th("Spot policy"), spot_policy) if reservation is not None: configuration_table.add_row(th("Reservation"), reservation) + # TODO: [Andrey] Display "Idle duration" offers_table = Table(box=None) offers_table.add_column("#") diff --git a/src/dstack/_internal/cli/utils/run.py b/src/dstack/_internal/cli/utils/run.py index b757417a6..af5289cfe 100644 --- a/src/dstack/_internal/cli/utils/run.py +++ b/src/dstack/_internal/cli/utils/run.py @@ -5,11 +5,15 @@ from dstack._internal.cli.utils.common import add_row_from_dict, console from dstack._internal.core.models.instances import InstanceAvailability -from dstack._internal.core.models.profiles import TerminationPolicy +from dstack._internal.core.models.profiles import ( + DEFAULT_RUN_TERMINATION_IDLE_TIME, + TerminationPolicy, +) from dstack._internal.core.models.runs import ( Job, RunPlan, ) +from dstack._internal.core.services.profiles import get_termination from dstack._internal.utils.common import DateFormatter, format_pretty_duration, pretty_date from dstack.api import Run @@ -25,20 +29,24 @@ def print_run_plan(run_plan: RunPlan, offers_limit: int = 3): pretty_req = req.pretty_format(resources_only=True) max_price = f"${req.max_price:g}" if req.max_price else "-" max_duration = ( - f"{job_plan.job_spec.max_duration / 3600:g}h" if job_plan.job_spec.max_duration else "-" + format_pretty_duration(job_plan.job_spec.max_duration) + if job_plan.job_spec.max_duration + else "-" ) if job_plan.job_spec.retry is None: - retry = "no" + retry = "-" else: retry = escape(job_plan.job_spec.retry.pretty_format()) profile = run_plan.run_spec.merged_profile creation_policy = profile.creation_policy - termination_policy = profile.termination_policy + termination_policy, termination_idle_time = get_termination( + profile, DEFAULT_RUN_TERMINATION_IDLE_TIME + ) if termination_policy == TerminationPolicy.DONT_DESTROY: - termination_idle_time = "-" + idle_duration = "-" else: - termination_idle_time = format_pretty_duration(profile.termination_idle_time) + idle_duration = format_pretty_duration(termination_idle_time) if req.spot is None: spot_policy = "auto" @@ -60,9 +68,8 @@ def th(s: str) -> str: props.add_row(th("Spot policy"), spot_policy) props.add_row(th("Retry policy"), retry) props.add_row(th("Creation policy"), creation_policy) - props.add_row(th("Termination policy"), termination_policy) - props.add_row(th("Termination idle time"), termination_idle_time) - props.add_row(th("Reservation"), run_plan.run_spec.configuration.reservation) + props.add_row(th("Idle duration"), idle_duration) + props.add_row(th("Reservation"), run_plan.run_spec.configuration.reservation or "-") offers = Table(box=None) offers.add_column("#") diff --git a/src/dstack/_internal/core/models/fleets.py b/src/dstack/_internal/core/models/fleets.py index d761499fc..75bd9d28c 100644 --- a/src/dstack/_internal/core/models/fleets.py +++ b/src/dstack/_internal/core/models/fleets.py @@ -13,13 +13,13 @@ from dstack._internal.core.models.instances import InstanceOfferWithAvailability, SSHKey from dstack._internal.core.models.pools import Instance from dstack._internal.core.models.profiles import ( - DEFAULT_POOL_TERMINATION_IDLE_TIME, Profile, ProfileParams, ProfileRetry, SpotPolicy, TerminationPolicy, parse_duration, + parse_idle_duration, ) from dstack._internal.core.models.resources import Range, ResourcesSpec @@ -172,18 +172,33 @@ class InstanceGroupParams(CoreModel): Optional[float], Field(description="The maximum instance price per hour, in dollars", gt=0.0), ] = None + + idle_duration: Annotated[ + Optional[Union[Literal["off"], str, int]], + Field( + description="Time to wait before terminating idle instances. Defaults to `5m` for runs and `3d` for fleets. Use `off` for unlimited duration" + ), + ] + # Deprecated: termination_policy: Annotated[ Optional[TerminationPolicy], - Field(description="The policy for instance termination. Defaults to `destroy-after-idle`"), + Field( + description="Deprecated in favor of `idle_duration`", + ), ] = None termination_idle_time: Annotated[ Optional[Union[str, int]], - Field(description="Time to wait before destroying idle instances. Defaults to `3d`"), + Field( + description="Deprecated in favor of `idle_duration`", + ), ] = None _validate_termination_idle_time = validator( "termination_idle_time", pre=True, allow_reuse=True )(parse_duration) + _validate_idle_duration = validator("idle_duration", pre=True, allow_reuse=True)( + parse_idle_duration + ) class FleetProps(CoreModel): @@ -224,10 +239,6 @@ def _merged_profile(cls, values) -> Dict: merged_profile.spot_policy = SpotPolicy.ONDEMAND if merged_profile.retry is None: merged_profile.retry = False - if merged_profile.termination_policy is None: - merged_profile.termination_policy = TerminationPolicy.DESTROY_AFTER_IDLE - if merged_profile.termination_idle_time is None: - merged_profile.termination_idle_time = DEFAULT_POOL_TERMINATION_IDLE_TIME values["merged_profile"] = merged_profile return values diff --git a/src/dstack/_internal/core/models/profiles.py b/src/dstack/_internal/core/models/profiles.py index 1e69e9526..08d0b9ee9 100644 --- a/src/dstack/_internal/core/models/profiles.py +++ b/src/dstack/_internal/core/models/profiles.py @@ -39,11 +39,18 @@ def parse_duration(v: Optional[Union[int, str]]) -> Optional[int]: def parse_max_duration(v: Optional[Union[int, str]]) -> Optional[Union[str, int]]: + # TODO: [Andrey] Not sure this works (see `parse_idle_duration`) if v == "off": return v return parse_duration(v) +def parse_idle_duration(v: Optional[Union[int, str]]) -> Optional[Union[str, int]]: + if v is False: + return -1 + return parse_duration(v) + + class ProfileRetryPolicy(CoreModel): retry: Annotated[bool, Field(description="Whether to retry the run on failure or not")] = False duration: Annotated[ @@ -144,17 +151,25 @@ class ProfileParams(CoreModel): description="The policy for using instances from the pool. Defaults to `reuse-or-create`" ), ] + idle_duration: Annotated[ + Optional[Union[Literal["off"], str, int]], + Field( + description="Time to wait before terminating idle instances. Defaults to `5m` for runs and `3d` for fleets. Use `off` for unlimited duration" + ), + ] + # Deprecated: termination_policy: Annotated[ Optional[TerminationPolicy], - Field(description="The policy for instance termination. Defaults to `destroy-after-idle`"), + Field( + description="Deprecated in favor of `idle_duration`", + ), ] termination_idle_time: Annotated[ Optional[Union[str, int]], Field( - description="Time to wait before destroying the idle instance. Defaults to `5m` for `dstack run` and to `3d` for `dstack pool add`" + description="Deprecated in favor of `idle_duration`", ), ] - # Deprecated: # The name of the pool. If not set, dstack will use the default name pool_name: Optional[str] # The name of the instance @@ -168,6 +183,9 @@ class ProfileParams(CoreModel): _validate_termination_idle_time = validator( "termination_idle_time", pre=True, allow_reuse=True )(parse_duration) + _validate_idle_duration = validator("idle_duration", pre=True, allow_reuse=True)( + parse_idle_duration + ) class ProfileProps(CoreModel): diff --git a/src/dstack/_internal/core/models/runs.py b/src/dstack/_internal/core/models/runs.py index a5f778083..7b5798b39 100644 --- a/src/dstack/_internal/core/models/runs.py +++ b/src/dstack/_internal/core/models/runs.py @@ -17,14 +17,12 @@ SSHConnectionParams, ) from dstack._internal.core.models.profiles import ( - DEFAULT_RUN_TERMINATION_IDLE_TIME, CreationPolicy, Profile, ProfileParams, ProfileRetryPolicy, RetryEvent, SpotPolicy, - TerminationPolicy, ) from dstack._internal.core.models.repos import AnyRunRepoData from dstack._internal.core.models.resources import ResourcesSpec @@ -338,10 +336,6 @@ def _merged_profile(cls, values) -> Dict: setattr(merged_profile, key, conf_val) if merged_profile.creation_policy is None: merged_profile.creation_policy = CreationPolicy.REUSE_OR_CREATE - if merged_profile.termination_policy is None: - merged_profile.termination_policy = TerminationPolicy.DESTROY_AFTER_IDLE - if merged_profile.termination_idle_time is None: - merged_profile.termination_idle_time = DEFAULT_RUN_TERMINATION_IDLE_TIME values["merged_profile"] = merged_profile return values diff --git a/src/dstack/_internal/core/services/profiles.py b/src/dstack/_internal/core/services/profiles.py index 904ae7d1c..7eca214f4 100644 --- a/src/dstack/_internal/core/services/profiles.py +++ b/src/dstack/_internal/core/services/profiles.py @@ -1,6 +1,11 @@ -from typing import Optional +from typing import Optional, Tuple -from dstack._internal.core.models.profiles import DEFAULT_RETRY_DURATION, Profile, RetryEvent +from dstack._internal.core.models.profiles import ( + DEFAULT_RETRY_DURATION, + Profile, + RetryEvent, + TerminationPolicy, +) from dstack._internal.core.models.runs import Retry @@ -30,3 +35,21 @@ def get_retry(profile: Profile) -> Optional[Retry]: if profile_retry.duration is None: profile_retry.duration = DEFAULT_RETRY_DURATION return Retry.parse_obj(profile_retry) + + +def get_termination( + profile: Profile, default_termination_idle_time: int +) -> Tuple[TerminationPolicy, int]: + termination_policy = TerminationPolicy.DESTROY_AFTER_IDLE + termination_idle_time = default_termination_idle_time + if profile.termination_policy is not None: + termination_policy = profile.termination_policy + if profile.termination_idle_time is not None: + termination_idle_time = profile.termination_idle_time + if profile.idle_duration is not None and int(profile.idle_duration) < 0: + termination_policy = TerminationPolicy.DONT_DESTROY + elif profile.idle_duration is not None: + termination_idle_time = profile.idle_duration + if termination_policy == TerminationPolicy.DONT_DESTROY: + termination_idle_time = -1 + return termination_policy, int(termination_idle_time) diff --git a/src/dstack/_internal/server/background/tasks/process_submitted_jobs.py b/src/dstack/_internal/server/background/tasks/process_submitted_jobs.py index 581e8ef1c..732a09ae5 100644 --- a/src/dstack/_internal/server/background/tasks/process_submitted_jobs.py +++ b/src/dstack/_internal/server/background/tasks/process_submitted_jobs.py @@ -19,6 +19,7 @@ ) from dstack._internal.core.models.profiles import ( DEFAULT_POOL_NAME, + DEFAULT_RUN_TERMINATION_IDLE_TIME, CreationPolicy, TerminationPolicy, ) @@ -31,6 +32,7 @@ RunSpec, ) from dstack._internal.core.models.volumes import Volume +from dstack._internal.core.services.profiles import get_termination from dstack._internal.server.db import get_db, get_session_ctx from dstack._internal.server.models import ( FleetModel, @@ -499,12 +501,14 @@ def _create_instance_model_for_job( instance_num: int, ) -> InstanceModel: profile = run_spec.merged_profile - termination_policy = profile.termination_policy - termination_idle_time = profile.termination_idle_time if not job_provisioning_data.dockerized: # terminate vastai/k8s instances immediately termination_policy = TerminationPolicy.DESTROY_AFTER_IDLE termination_idle_time = 0 + else: + termination_policy, termination_idle_time = get_termination( + profile, DEFAULT_RUN_TERMINATION_IDLE_TIME + ) instance = InstanceModel( id=uuid.uuid4(), name=f"{fleet_model.name}-{instance_num}", diff --git a/src/dstack/_internal/server/models.py b/src/dstack/_internal/server/models.py index 496138764..a7c2d178b 100644 --- a/src/dstack/_internal/server/models.py +++ b/src/dstack/_internal/server/models.py @@ -513,6 +513,7 @@ class InstanceModel(BaseModel): # temination policy termination_policy: Mapped[Optional[TerminationPolicy]] = mapped_column(String(100)) + # TODO: Suggestion: do not assign DEFAULT_POOL_TERMINATION_IDLE_TIME as the default here (make Optional instead; also instead of -1) termination_idle_time: Mapped[int] = mapped_column( Integer, default=DEFAULT_POOL_TERMINATION_IDLE_TIME ) diff --git a/src/dstack/_internal/server/services/fleets.py b/src/dstack/_internal/server/services/fleets.py index 3846f9106..61630959b 100644 --- a/src/dstack/_internal/server/services/fleets.py +++ b/src/dstack/_internal/server/services/fleets.py @@ -38,12 +38,12 @@ DEFAULT_POOL_TERMINATION_IDLE_TIME, Profile, SpotPolicy, - TerminationPolicy, ) from dstack._internal.core.models.resources import ResourcesSpec from dstack._internal.core.models.runs import Requirements, get_policy_map from dstack._internal.core.models.users import GlobalRole from dstack._internal.core.services import validate_dstack_resource_name +from dstack._internal.core.services.profiles import get_termination from dstack._internal.server.db import get_db from dstack._internal.server.models import ( FleetModel, @@ -564,11 +564,9 @@ async def create_instance( pool_name=pool.name, ) - termination_policy = profile.termination_policy or TerminationPolicy.DESTROY_AFTER_IDLE - termination_idle_time = profile.termination_idle_time - if termination_idle_time is None: - termination_idle_time = DEFAULT_POOL_TERMINATION_IDLE_TIME - + termination_policy, termination_idle_time = get_termination( + profile, DEFAULT_POOL_TERMINATION_IDLE_TIME + ) instance = InstanceModel( id=uuid.uuid4(), name=instance_name, diff --git a/src/dstack/_internal/server/services/pools.py b/src/dstack/_internal/server/services/pools.py index fa0f218d9..5ca4f2140 100644 --- a/src/dstack/_internal/server/services/pools.py +++ b/src/dstack/_internal/server/services/pools.py @@ -34,12 +34,14 @@ from dstack._internal.core.models.pools import Instance, Pool, PoolInstances from dstack._internal.core.models.profiles import ( DEFAULT_POOL_NAME, + DEFAULT_POOL_TERMINATION_IDLE_TIME, Profile, TerminationPolicy, ) from dstack._internal.core.models.runs import JobProvisioningData, Requirements from dstack._internal.core.models.users import GlobalRole from dstack._internal.core.models.volumes import Volume +from dstack._internal.core.services.profiles import get_termination from dstack._internal.server import settings from dstack._internal.server.models import ( FleetModel, @@ -608,6 +610,9 @@ async def create_instance_model( placement_group_name: Optional[str], reservation: Optional[str], ) -> InstanceModel: + termination_policy, termination_idle_time = get_termination( + profile, DEFAULT_POOL_TERMINATION_IDLE_TIME + ) instance = InstanceModel( id=uuid.uuid4(), name=instance_name, @@ -620,8 +625,8 @@ async def create_instance_model( profile=profile.json(), requirements=requirements.json(), instance_configuration=None, - termination_policy=profile.termination_policy, - termination_idle_time=profile.termination_idle_time, + termination_policy=termination_policy, + termination_idle_time=termination_idle_time, ) session.add(instance) await session.flush() diff --git a/src/dstack/_internal/utils/common.py b/src/dstack/_internal/utils/common.py index 09beb808e..15c1dd0d0 100644 --- a/src/dstack/_internal/utils/common.py +++ b/src/dstack/_internal/utils/common.py @@ -158,6 +158,8 @@ def parse_pretty_duration(duration: str) -> int: def format_pretty_duration(seconds: int) -> str: + if seconds == 0: + return "0s" if seconds < 0: raise ValueError("Seconds cannot be negative") units = [ diff --git a/src/dstack/api/_public/runs.py b/src/dstack/api/_public/runs.py index b35ce1016..1e4dd0b80 100644 --- a/src/dstack/api/_public/runs.py +++ b/src/dstack/api/_public/runs.py @@ -457,6 +457,7 @@ def get_offers(self, profile: Profile, requirements: Requirements) -> PoolInstan def create_instance(self, profile: Profile, requirements: Requirements) -> Instance: return self._api_client.runs.create_instance(self._project, profile, requirements) + # TODO: [Andrey] I guess we need to drop profile-related fields (currently retry is not reflected there) def get_plan( self, configuration: AnyRunConfiguration, @@ -505,6 +506,7 @@ def get_plan( configuration = configuration.copy(deep=True) configuration.resources = resources + # TODO: [Andrey] "(python") looks as a hack profile = Profile( name="(python)", backends=backends, diff --git a/src/dstack/api/server/__init__.py b/src/dstack/api/server/__init__.py index e142f7f9b..5fc7b7cd3 100644 --- a/src/dstack/api/server/__init__.py +++ b/src/dstack/api/server/__init__.py @@ -143,6 +143,8 @@ def _request( code = kwargs.pop("code") raise _server_client_errors[code](**kwargs) if resp.status_code == 422: + print(path) + print(body) formatted_error = pprint.pformat(resp.json()) raise ClientError(f"Server validation error: \n{formatted_error}") resp.raise_for_status() diff --git a/src/dstack/api/server/_fleets.py b/src/dstack/api/server/_fleets.py index 901c43e69..ae3c36fba 100644 --- a/src/dstack/api/server/_fleets.py +++ b/src/dstack/api/server/_fleets.py @@ -73,6 +73,9 @@ def _get_fleet_spec_excludes(fleet_spec: FleetSpec) -> Optional[_ExcludeDict]: if not fleet_spec.configuration.reservation: configuration_excludes["reservation"] = True profile_excludes.add("reservation") + if not fleet_spec.configuration.idle_duration: + configuration_excludes["idle_duration"] = True + profile_excludes.add("idle_duration") if configuration_excludes: spec_excludes["configuration"] = configuration_excludes diff --git a/src/dstack/api/server/_runs.py b/src/dstack/api/server/_runs.py index 2a9efb838..0623d7910 100644 --- a/src/dstack/api/server/_runs.py +++ b/src/dstack/api/server/_runs.py @@ -132,6 +132,9 @@ def _get_run_spec_excludes(run_spec: RunSpec) -> Optional[dict]: if not configuration.reservation: configuration_excludes.add("reservation") profile_excludes.add("reservation") + if not configuration.idle_duration: + configuration_excludes.add("idle_duration") + profile_excludes.add("idle_duration") if configuration_excludes: spec_excludes["configuration"] = configuration_excludes diff --git a/src/tests/_internal/server/routers/test_fleets.py b/src/tests/_internal/server/routers/test_fleets.py index 43e210c78..b4b94e4ac 100644 --- a/src/tests/_internal/server/routers/test_fleets.py +++ b/src/tests/_internal/server/routers/test_fleets.py @@ -280,6 +280,7 @@ async def test_creates_fleet(self, test_db, session: AsyncSession, client: Async "spot_policy": None, "retry": None, "max_price": None, + "idle_duration": None, "termination_policy": None, "termination_idle_time": None, "type": "fleet", @@ -298,6 +299,7 @@ async def test_creates_fleet(self, test_db, session: AsyncSession, client: Async "pool_name": None, "instance_name": None, "creation_policy": None, + "idle_duration": None, "termination_policy": None, "termination_idle_time": None, "name": "", @@ -393,6 +395,7 @@ async def test_creates_ssh_fleet(self, test_db, session: AsyncSession, client: A "spot_policy": None, "retry": None, "max_price": None, + "idle_duration": None, "termination_policy": None, "termination_idle_time": None, "type": "fleet", @@ -411,6 +414,7 @@ async def test_creates_ssh_fleet(self, test_db, session: AsyncSession, client: A "pool_name": None, "instance_name": None, "creation_policy": None, + "idle_duration": None, "termination_policy": None, "termination_idle_time": None, "name": "", diff --git a/src/tests/_internal/server/routers/test_runs.py b/src/tests/_internal/server/routers/test_runs.py index 1a2e4f990..b7e98e02c 100644 --- a/src/tests/_internal/server/routers/test_runs.py +++ b/src/tests/_internal/server/routers/test_runs.py @@ -116,6 +116,7 @@ def get_dev_env_run_plan_dict( "retry": None, "retry_policy": None, "spot_policy": "spot", + "idle_duration": None, "termination_idle_time": 300, "termination_policy": None, "reservation": None, @@ -135,6 +136,7 @@ def get_dev_env_run_plan_dict( "retry": None, "retry_policy": None, "spot_policy": "spot", + "idle_duration": None, "termination_idle_time": 300, "termination_policy": None, "reservation": None, @@ -262,6 +264,7 @@ def get_dev_env_run_dict( "retry": None, "retry_policy": None, "spot_policy": "spot", + "idle_duration": None, "termination_idle_time": 300, "termination_policy": None, "reservation": None, @@ -281,6 +284,7 @@ def get_dev_env_run_dict( "retry": None, "retry_policy": None, "spot_policy": "spot", + "idle_duration": None, "termination_idle_time": 300, "termination_policy": None, "reservation": None, @@ -1449,6 +1453,7 @@ async def test_creates_instance(self, test_db, session: AsyncSession, client: As profile=Profile(name="test_profile"), requirements=Requirements(resources=ResourcesSpec(cpu=1)), ) + print(str(request)) instance_id = UUID("1b0e1b45-2f8c-4ab6-8010-a0d1a3e44e0e") with ( patch( From 63935fdd76f7149dd72d8568bab55f19a386fc73 Mon Sep 17 00:00:00 2001 From: peterschmidt85 Date: Mon, 30 Dec 2024 23:19:38 +0100 Subject: [PATCH 2/5] Replace termination_idle_time: int|str with idle_duration: int|str|off #1154 --- src/tests/_internal/server/routers/test_runs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tests/_internal/server/routers/test_runs.py b/src/tests/_internal/server/routers/test_runs.py index b7e98e02c..135d373b6 100644 --- a/src/tests/_internal/server/routers/test_runs.py +++ b/src/tests/_internal/server/routers/test_runs.py @@ -1453,7 +1453,6 @@ async def test_creates_instance(self, test_db, session: AsyncSession, client: As profile=Profile(name="test_profile"), requirements=Requirements(resources=ResourcesSpec(cpu=1)), ) - print(str(request)) instance_id = UUID("1b0e1b45-2f8c-4ab6-8010-a0d1a3e44e0e") with ( patch( From d3276d499a96e532171bf49e2ec0fa299475684f Mon Sep 17 00:00:00 2001 From: peterschmidt85 Date: Mon, 6 Jan 2025 11:53:58 +0100 Subject: [PATCH 3/5] Replace termination_idle_time: int|str with idle_duration: int|str|off #1154 --- src/dstack/api/server/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/dstack/api/server/__init__.py b/src/dstack/api/server/__init__.py index 5fc7b7cd3..e142f7f9b 100644 --- a/src/dstack/api/server/__init__.py +++ b/src/dstack/api/server/__init__.py @@ -143,8 +143,6 @@ def _request( code = kwargs.pop("code") raise _server_client_errors[code](**kwargs) if resp.status_code == 422: - print(path) - print(body) formatted_error = pprint.pformat(resp.json()) raise ClientError(f"Server validation error: \n{formatted_error}") resp.raise_for_status() From 626ef11b75c89c24dc619567759a2b5a04be891d Mon Sep 17 00:00:00 2001 From: peterschmidt85 Date: Mon, 6 Jan 2025 11:55:33 +0100 Subject: [PATCH 4/5] Replace termination_idle_time: int|str with idle_duration: int|str|off #1154 --- src/dstack/api/server/_fleets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dstack/api/server/_fleets.py b/src/dstack/api/server/_fleets.py index ae3c36fba..5e9195518 100644 --- a/src/dstack/api/server/_fleets.py +++ b/src/dstack/api/server/_fleets.py @@ -73,7 +73,7 @@ def _get_fleet_spec_excludes(fleet_spec: FleetSpec) -> Optional[_ExcludeDict]: if not fleet_spec.configuration.reservation: configuration_excludes["reservation"] = True profile_excludes.add("reservation") - if not fleet_spec.configuration.idle_duration: + if fleet_spec.configuration.idle_duration is None: configuration_excludes["idle_duration"] = True profile_excludes.add("idle_duration") From eaca4cc2a49d8a5185ed74ce3ccd9ef92fddfc14 Mon Sep 17 00:00:00 2001 From: peterschmidt85 Date: Mon, 6 Jan 2025 11:56:11 +0100 Subject: [PATCH 5/5] Replace termination_idle_time: int|str with idle_duration: int|str|off #1154 --- src/dstack/api/server/_runs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dstack/api/server/_runs.py b/src/dstack/api/server/_runs.py index 0623d7910..803d1ca9d 100644 --- a/src/dstack/api/server/_runs.py +++ b/src/dstack/api/server/_runs.py @@ -132,7 +132,7 @@ def _get_run_spec_excludes(run_spec: RunSpec) -> Optional[dict]: if not configuration.reservation: configuration_excludes.add("reservation") profile_excludes.add("reservation") - if not configuration.idle_duration: + if configuration.idle_duration is None: configuration_excludes.add("idle_duration") profile_excludes.add("idle_duration")