Skip to content

Commit

Permalink
feat: hide more parameters from k8s and compose yamls (#4242)
Browse files Browse the repository at this point in the history
  • Loading branch information
@JoanFM
JoanFM committed Feb 7, 2022
1 parent 48871dd commit 6e24af6
Show file tree
Hide file tree
Showing 10 changed files with 107 additions and 81 deletions.
4 changes: 0 additions & 4 deletions cli/autocomplete.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,6 @@
'--port-in',
'--host-in',
'--native',
'--num-part',
'--entrypoint',
'--docker-kwargs',
'--pull-latest',
Expand Down Expand Up @@ -137,7 +136,6 @@
'--port-in',
'--host-in',
'--native',
'--num-part',
'--prefetch',
'--title',
'--description',
Expand Down Expand Up @@ -224,7 +222,6 @@
'--port-in',
'--host-in',
'--native',
'--num-part',
'--entrypoint',
'--docker-kwargs',
'--pull-latest',
Expand Down Expand Up @@ -276,7 +273,6 @@
'--port-in',
'--host-in',
'--native',
'--num-part',
'--entrypoint',
'--docker-kwargs',
'--pull-latest',
Expand Down
2 changes: 1 addition & 1 deletion docs/fundamentals/executor/executor-built-in-features.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ The following fields are valid for `metas` and `runtime_args`:
| Attribute | Fields |
|-------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `.metas` (static values from hard-coded values, YAML config) | `name`, `description`, `py_modules`, `workspace` |
| `.runtime_args` (runtime values from its containers, e.g. `Runtime`, `Pod`, `Deployment`) | `name`, `description`, `workspace`, `log_config`, `quiet`, `quiet_error`, `identity`, `port_ctrl`, `ctrl_with_ipc`, `timeout_ctrl`, `ssh_server`, `ssh_keyfile`, `ssh_password`, `uses`, `py_modules`, `port_in`, `port_out`, `host_in`, `host_out`, `socket_in`, `socket_out`, `memory_hwm`, `on_error_strategy`, `num_part`, `entrypoint`, `docker_kwargs`, `pull_latest`, `volumes`, `host`, `port_expose`, `quiet_remote_logs`, `upload_files`, `workspace_id`, `daemon`, `runtime_backend`, `runtime_cls`, `timeout_ready`, `env`, `expose_public`, `shard_id`, `pod_role`, `noblock_on_start`, `uses_before`, `uses_after`, `parallel`, `replicas`, `polling`, `scheduling`, `deployment_role``, `proxy`, `uses_metas`, `external`, `gpus`, `zmq_identity`, `hosts_in_connect`, `uses_with` |
| `.runtime_args` (runtime values from its containers, e.g. `Runtime`, `Pod`, `Deployment`) | `name`, `workspace`, `shard_id`, `replicas`, 'shards' |


````{admonition} Note
Expand Down
46 changes: 17 additions & 29 deletions jina/orchestrate/deployments/config/docker_compose.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
get_image_name,
to_compatible_name,
get_base_executor_version,
construct_runtime_container_args,
)
from jina.orchestrate.deployments import BaseDeployment

Expand Down Expand Up @@ -59,9 +60,20 @@ def get_gateway_config(
from jina.helper import ArgNamespace
from jina.parsers import set_gateway_parser

taboo = {
'uses_with',
'uses_metas',
'volumes',
'uses_before',
'uses_after',
'workspace',
'workspace_id',
'upload_files',
'noblock_on_start',
}

non_defaults = ArgNamespace.get_non_defaults_args(
cargs,
set_gateway_parser(),
cargs, set_gateway_parser(), taboo=taboo
)
_args = ArgNamespace.kwargs2list(non_defaults)
container_args = ['gateway'] + _args
Expand All @@ -79,32 +91,6 @@ def get_gateway_config(
],
}

@staticmethod
def _construct_runtime_container_args(cargs, uses_metas, uses_with):
import json
from jina.helper import ArgNamespace
from jina.parsers import set_pod_parser

non_defaults = ArgNamespace.get_non_defaults_args(
cargs,
set_pod_parser(),
taboo={
'uses_with',
'uses_metas',
'volumes',
'uses_before',
'uses_after',
},
)
_args = ArgNamespace.kwargs2list(non_defaults)
container_args = ['executor'] + _args
if uses_metas is not None:
container_args.extend(['--uses-metas', json.dumps(uses_metas)])
if uses_with is not None:
container_args.extend(['--uses-with', json.dumps(uses_with)])
container_args.append('--native')
return container_args

def _get_image_name(self, uses: Optional[str]):
import os

Expand All @@ -127,7 +113,9 @@ def _get_container_args(self, cargs):
uses_with = self.service_args.uses_with
if cargs.uses != __default_executor__:
cargs.uses = 'config.yml'
return self._construct_runtime_container_args(cargs, uses_metas, uses_with)
return construct_runtime_container_args(
cargs, uses_metas, uses_with, self.pod_type
)

def get_runtime_config(
self,
Expand Down
49 changes: 49 additions & 0 deletions jina/orchestrate/deployments/config/helper.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from jina import __version__
from jina.hubble.helper import parse_hub_uri
from jina.hubble.hubio import HubIO
from jina.enums import PodRoleType


def get_image_name(uses: str) -> str:
Expand Down Expand Up @@ -49,3 +50,51 @@ def get_base_executor_version():
return 'master'
except:
return 'master'


def construct_runtime_container_args(cargs, uses_metas, uses_with, pod_type):
"""
Construct a set of Namespace arguments into a list of arguments to pass to a container entrypoint
:param cargs: The namespace arguments
:param uses_metas: The uses_metas to override
:param uses_with: The uses_with to override
:param pod_type: The pod_type
:return: Arguments to pass to container
"""
import json
from jina.helper import ArgNamespace
from jina.parsers import set_pod_parser

taboo = {
'uses_with',
'uses_metas',
'volumes',
'uses_before',
'uses_after',
'workspace_id',
'upload_files',
'noblock_on_start',
}

if pod_type == PodRoleType.HEAD:
taboo.add('uses')
taboo.add('workspace')

if pod_type in {PodRoleType.WORKER, PodRoleType.GATEWAY}:
taboo.add('polling')

non_defaults = ArgNamespace.get_non_defaults_args(
cargs,
set_pod_parser(),
taboo=taboo,
)
_args = ArgNamespace.kwargs2list(non_defaults)
container_args = ['executor'] + _args
if not cargs.k8s_connection_pool and pod_type == PodRoleType.HEAD:
container_args.append('--k8s-disable-connection-pool')
if uses_metas is not None:
container_args.extend(['--uses-metas', json.dumps(uses_metas)])
if uses_with is not None:
container_args.extend(['--uses-with', json.dumps(uses_with)])
container_args.append('--native')
return container_args
46 changes: 15 additions & 31 deletions jina/orchestrate/deployments/config/k8s.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
get_image_name,
to_compatible_name,
get_base_executor_version,
construct_runtime_container_args,
)
from jina.serve.networking import K8sGrpcConnectionPool
from jina.orchestrate.deployments import BaseDeployment
Expand Down Expand Up @@ -63,9 +64,20 @@ def get_gateway_yamls(
from jina.helper import ArgNamespace
from jina.parsers import set_gateway_parser

taboo = {
'uses_with',
'uses_metas',
'volumes',
'uses_before',
'uses_after',
'workspace',
'workspace_id',
'upload_files',
'noblock_on_start',
}

non_defaults = ArgNamespace.get_non_defaults_args(
cargs,
set_gateway_parser(),
cargs, set_gateway_parser(), taboo=taboo
)
_args = ArgNamespace.kwargs2list(non_defaults)
container_args = ['gateway'] + _args
Expand All @@ -85,34 +97,6 @@ def get_gateway_yamls(
env=cargs.env,
)

@staticmethod
def _construct_runtime_container_args(cargs, uses_metas, uses_with, pod_type):
import json
from jina.helper import ArgNamespace
from jina.parsers import set_pod_parser

non_defaults = ArgNamespace.get_non_defaults_args(
cargs,
set_pod_parser(),
taboo={
'uses_with',
'uses_metas',
'volumes',
'uses_before',
'uses_after',
},
)
_args = ArgNamespace.kwargs2list(non_defaults)
container_args = ['executor'] + _args
if not cargs.k8s_connection_pool and pod_type == PodRoleType.HEAD:
container_args.append('--k8s-disable-connection-pool')
if uses_metas is not None:
container_args.extend(['--uses-metas', json.dumps(uses_metas)])
if uses_with is not None:
container_args.extend(['--uses-with', json.dumps(uses_with)])
container_args.append('--native')
return container_args

def _get_image_name(self, uses: Optional[str]):
import os

Expand All @@ -135,7 +119,7 @@ def _get_container_args(self, cargs, pod_type):
uses_with = self.deployment_args.uses_with
if cargs.uses != __default_executor__:
cargs.uses = 'config.yml'
return self._construct_runtime_container_args(
return construct_runtime_container_args(
cargs, uses_metas, uses_with, pod_type
)

Expand Down
27 changes: 23 additions & 4 deletions jina/orchestrate/flow/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -273,7 +273,7 @@ def __init__(
quiet_error: Optional[bool] = False,
timeout_ctrl: Optional[int] = 60,
uses: Optional[str] = None,
workspace: Optional[str] = './',
workspace: Optional[str] = None,
**kwargs,
):
"""Create a Flow. Flow is how Jina streamlines and scales Executors. This overloaded method provides arguments from `jina flow` CLI.
Expand Down Expand Up @@ -780,7 +780,6 @@ def add(
dict(
name=deployment_name,
deployment_role=deployment_role,
num_part=len(needs),
)
)

Expand All @@ -793,7 +792,10 @@ def add(
)

# deployment workspace if not set then derive from flow workspace
args.workspace = os.path.abspath(args.workspace or self.workspace)
if args.workspace:
args.workspace = os.path.abspath(args.workspace)
else:
args.workspace = self.workspace

args.noblock_on_start = True
args.extra_search_paths = self.args.extra_search_paths
Expand Down Expand Up @@ -1562,7 +1564,10 @@ def workspace(self) -> str:
"""Return the workspace path of the flow.
.. # noqa: DAR201"""
return os.path.abspath(self.args.workspace or './')
if self.args.workspace is not None:
return os.path.abspath(self.args.workspace)
else:
return None

@workspace.setter
def workspace(self, value: str):
Expand Down Expand Up @@ -1754,6 +1759,10 @@ def to_k8s_yaml(
if i < len(k8s_objects) - 1:
fp.write('---\n')

self.logger.info(
f'K8s yaml files have been created under {output_base_path}. You can use it by running `kubectl apply -R -f {output_base_path}`'
)

def to_docker_compose_yaml(
self, output_path: Optional[str] = None, network_name: Optional[str] = None
):
Expand Down Expand Up @@ -1795,6 +1804,16 @@ def to_docker_compose_yaml(
with open(output_path, 'w+') as fp:
yaml.dump(docker_compose_dict, fp, sort_keys=False)

command = (
'docker-compose up'
if output_path is None
else f'docker-compose up -f {output_path}'
)

self.logger.info(
f'Docker compose file has been created under {output_path}. You can use it by running `{command}`'
)

def scale(
self,
deployment_name: str,
Expand Down
2 changes: 0 additions & 2 deletions jina/parsers/flow.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,6 @@ def set_flow_parser(parser=None):

mixin_base_ppr_parser(parser)

parser.set_defaults(workspace='./')

mixin_flow_features_parser(parser)

return parser
1 change: 1 addition & 0 deletions jina/parsers/orchestrate/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ def mixin_base_ppr_parser(parser):
gp.add_argument(
'--workspace',
type=str,
default=None,
help='The working directory for any IO operations in this object. '
'If not set, then derive from its parent `workspace`.',
)
Expand Down
9 changes: 0 additions & 9 deletions jina/parsers/orchestrate/runtimes/worker.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,12 +92,3 @@ def mixin_worker_runtime_parser(parser):
default=False,
help='If set, only native Executors is allowed, and the Executor is always run inside WorkerRuntime.',
)

gp.add_argument(
'--num-part',
type=int,
default=0,
help='the number of messages expected from upstream, 0 and 1 means single part'
if _SHOW_ALL_ARGS
else argparse.SUPPRESS,
)
2 changes: 1 addition & 1 deletion scripts/latency-tracking/app.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ def _benchmark_qps() -> Dict[str, float]:
download_data(targets, args.download_proxy)

try:
f = Flow().add(uses=MyEncoder).add(uses=MyIndexer)
f = Flow().add(uses=MyEncoder).add(workspace='./', uses=MyIndexer)

with f:
# do index
Expand Down

0 comments on commit 6e24af6

Please sign in to comment.