From 6c7a551deda037958c0c8470f7177b68d7b56de8 Mon Sep 17 00:00:00 2001 From: Aaron Pham <29749331+aarnphm@users.noreply.github.com> Date: Tue, 4 Oct 2022 18:21:41 -0700 Subject: [PATCH] -------wip include both docs and implementation cherrypick me------------ Signed-off-by: Aaron Pham <29749331+aarnphm@users.noreply.github.com> --- bentoml/_internal/configuration/containers.py | 29 +++++++++++++------ bentoml/_internal/configuration/helpers.py | 7 ++--- .../_internal/configuration/v2/__init__.py | 3 +- .../_internal/configuration/v2/defaults.yaml | 4 +-- .../_internal/runner/runner_handle/remote.py | 6 ++-- .../guides/snippets/configuration/v2.rst | 2 +- 6 files changed, 28 insertions(+), 23 deletions(-) diff --git a/bentoml/_internal/configuration/containers.py b/bentoml/_internal/configuration/containers.py index a23eeee5513..cb1b6a6aeaa 100644 --- a/bentoml/_internal/configuration/containers.py +++ b/bentoml/_internal/configuration/containers.py @@ -49,9 +49,6 @@ logger = logging.getLogger(__name__) -RUNNER_CFG_KEYS = ["batching", "resources", "logging", "metrics", "timeout"] - - CONFIG_LATEST_VERSION = 2 @@ -146,8 +143,16 @@ def __init__( try: override = unflatten(override_config_map) except ValueError as e: + if "version" not in override_config_map: + logger.warning( + "Config options override is invalid with current configuration version %d.\nIf you are using different config version of BentoML make sure to add 'version' to BENTOML_CONFIG_OPTIONS. i.e: %s" + % ( + use_version, + "BENTOML_CONFIG_OPTIONS='version=1 api_server.port=5000'", + ) + ) raise BentoMLConfigException( - f"Failed to parse config options from the env var: {e}.\n *** Note: You can use '\"' to quote the key if it contains special characters. ***" + f"Failed to parse config options from the env var:\n{e}.\n*** Note: You can use '\"' to quote the key if it contains special characters. ***" ) from None config_merger.merge(self.config, override) @@ -163,14 +168,20 @@ def __init__( ) from None def _finalize(self): - global_runner_cfg = {k: self.config["runners"][k] for k in RUNNER_CFG_KEYS} - for key in self.config["runners"]: - if key not in RUNNER_CFG_KEYS: - runner_cfg = self.config["runners"][key] + GLOBAL_RUNNERS_KEY = ["batching", "resources", "logging", "metrics"] + global_runner_cfg = {k: self.config["runners"][k] for k in GLOBAL_RUNNERS_KEY} + custom_runners_cfg = dict( + filter( + lambda kv: kv[0] not in GLOBAL_RUNNERS_KEY, + self.config["runners"].items(), + ) + ) + if custom_runners_cfg: + for runner_name, runner_cfg in custom_runners_cfg.items(): # key is a runner name if runner_cfg.get("resources") == "system": runner_cfg["resources"] = system_resources() - self.config["runners"][key] = config_merger.merge( + self.config["runners"][runner_name] = config_merger.merge( deepcopy(global_runner_cfg), runner_cfg, ) diff --git a/bentoml/_internal/configuration/helpers.py b/bentoml/_internal/configuration/helpers.py index fdf4a2012cf..7d63eb700ef 100644 --- a/bentoml/_internal/configuration/helpers.py +++ b/bentoml/_internal/configuration/helpers.py @@ -2,6 +2,7 @@ import os import socket +import string import typing as t import logging from typing import TYPE_CHECKING @@ -74,10 +75,8 @@ def flatten_dict( ) -> t.Generator[tuple[str, t.Any], None, None]: """Flatten nested dictionary into a single level dictionary.""" for k, v in d.items(): - # TODO: we probably need to find a better way - # to normalize slash and special characters keys. - key = f'"{k}"' if "/" in k else k - nkey = parent + sep + key if parent else key + k = f'"{k}"' if any(i in string.punctuation for i in k) else k + nkey = parent + sep + k if parent else k if isinstance(v, t.MutableMapping): yield from flatten_dict( t.cast(t.MutableMapping[str, t.Any], v), parent=nkey, sep=sep diff --git a/bentoml/_internal/configuration/v2/__init__.py b/bentoml/_internal/configuration/v2/__init__.py index d88be64b008..8767cb03da5 100644 --- a/bentoml/_internal/configuration/v2/__init__.py +++ b/bentoml/_internal/configuration/v2/__init__.py @@ -56,7 +56,6 @@ } _API_SERVER_CONFIG = { "workers": s.Or(s.And(int, ensure_larger_than_zero), None), - "timeout": s.And(int, ensure_larger_than_zero), "backlog": s.And(int, ensure_larger_than(64)), "metrics": { "enabled": bool, @@ -143,11 +142,11 @@ "enabled": bool, "namespace": str, }, - s.Optional("timeout"): s.And(int, ensure_larger_than_zero), } SCHEMA = s.Schema( { "version": s.And(int, lambda v: v == 2), + s.Optional("timeout"): s.And(int, ensure_larger_than_zero), "api_server": _API_SERVER_CONFIG, "runners": { **_RUNNER_CONFIG, diff --git a/bentoml/_internal/configuration/v2/defaults.yaml b/bentoml/_internal/configuration/v2/defaults.yaml index b952ea2feaf..62bd834ac7b 100644 --- a/bentoml/_internal/configuration/v2/defaults.yaml +++ b/bentoml/_internal/configuration/v2/defaults.yaml @@ -1,7 +1,7 @@ version: 2 +timeout: 300 api_server: workers: ~ # cpu_count() will be used when null - timeout: 60 backlog: 2048 metrics: enabled: true @@ -97,9 +97,7 @@ api_server: grpc: headers: ~ insecure: ~ - runners: - timeout: 300 resources: ~ batching: enabled: true diff --git a/bentoml/_internal/runner/runner_handle/remote.py b/bentoml/_internal/runner/runner_handle/remote.py index d557cb5a84c..b1dbfc8d72a 100644 --- a/bentoml/_internal/runner/runner_handle/remote.py +++ b/bentoml/_internal/runner/runner_handle/remote.py @@ -89,9 +89,7 @@ def _get_conn(self) -> BaseConnector: return self._conn @property - def _client( - self, - ) -> ClientSession: + def _client(self) -> ClientSession: import aiohttp if ( @@ -111,7 +109,7 @@ def strip_query_params(url: yarl.URL) -> str: trace_configs=[ create_trace_config( # Remove all query params from the URL attribute on the span. - url_filter=strip_query_params, # type: ignore + url_filter=strip_query_params, tracer_provider=BentoMLContainer.tracer_provider.get(), ) ], diff --git a/docs/source/guides/snippets/configuration/v2.rst b/docs/source/guides/snippets/configuration/v2.rst index 7c57c6d55c7..8556f669813 100644 --- a/docs/source/guides/snippets/configuration/v2.rst +++ b/docs/source/guides/snippets/configuration/v2.rst @@ -8,4 +8,4 @@ The following options are available for the ``api_server`` section: +-------------+-------------------------------------+-----------------+ | ``workers`` | Number of API workers for to spawn | None (which will be determined by BentoML) +-------------+-------------------------------------+-----------------+ -``timeout`` +| ``timeout`` | Timeout for API server in seconds | 60 |