Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] feat: executor to gcp custom container #6136

Draft
wants to merge 7 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions jina/enums.py
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,7 @@ class ProviderType(BetterEnum):

NONE = 0 #: no provider
SAGEMAKER = 1 #: AWS SageMaker
GCP = 2 #: GCP


def replace_enum_to_str(obj):
Expand Down
10 changes: 5 additions & 5 deletions jina/orchestrate/deployments/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -386,7 +386,7 @@ def __init__(
:param port_monitoring: The port on which the prometheus server is exposed, default is a random port between [49152, 65535]
:param prefer_platform: The preferred target Docker platform. (e.g. "linux/amd64", "linux/arm64")
:param protocol: Communication protocol of the server exposed by the Executor. This can be a single value or a list of protocols, depending on your chosen Gateway. Choose the convenient protocols from: ['GRPC', 'HTTP', 'WEBSOCKET'].
:param provider: If set, Executor is translated to a custom container compatible with the chosen provider. Choose the convenient providers from: ['NONE', 'SAGEMAKER'].
:param provider: If set, Executor is translated to a custom container compatible with the chosen provider. Choose the convenient providers from: ['NONE', 'SAGEMAKER', 'GCP'].
:param py_modules: The customized python modules need to be imported before loading the executor

Note that the recommended way is to only import a single module - a simple python file, if your
Expand Down Expand Up @@ -476,21 +476,21 @@ def __init__(
args = ArgNamespace.kwargs2namespace(kwargs, parser, True)
self.args = args
self._gateway_load_balancer = False
if self.args.provider == ProviderType.SAGEMAKER:
if self.args.provider in (ProviderType.SAGEMAKER, ProviderType.GCP):
if self._gateway_kwargs.get('port', 0) == 8080:
raise ValueError(
'Port 8080 is reserved for Sagemaker deployment. '
'Port 8080 is reserved for CSP deployment. '
'Please use another port'
)
if self.args.port != [8080]:
warnings.warn(
'Port is changed to 8080 for Sagemaker deployment. '
'Port is changed to 8080 for CSP deployment. '
f'Port {self.args.port} is ignored'
)
self.args.port = [8080]
if self.args.protocol != [ProtocolType.HTTP]:
warnings.warn(
'Protocol is changed to HTTP for Sagemaker deployment. '
'Protocol is changed to HTTP for CSP deployment. '
f'Protocol {self.args.protocol} is ignored'
)
self.args.protocol = [ProtocolType.HTTP]
Expand Down
12 changes: 6 additions & 6 deletions jina/orchestrate/flow/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -273,7 +273,7 @@ def __init__(

Used to control the speed of data input into a Flow. 0 disables prefetch (1000 requests is the default)
:param protocol: Communication protocol of the server exposed by the Gateway. This can be a single value or a list of protocols, depending on your chosen Gateway. Choose the convenient protocols from: ['GRPC', 'HTTP', 'WEBSOCKET'].
:param provider: If set, Executor is translated to a custom container compatible with the chosen provider. Choose the convenient providers from: ['NONE', 'SAGEMAKER'].
:param provider: If set, Executor is translated to a custom container compatible with the chosen provider. Choose the convenient providers from: ['NONE', 'SAGEMAKER', 'GCP'].
:param proxy: If set, respect the http_proxy and https_proxy environment variables. otherwise, it will unset these proxy variables before start. gRPC seems to prefer no proxy
:param py_modules: The customized python modules need to be imported before loading the gateway

Expand Down Expand Up @@ -464,7 +464,7 @@ def __init__(

Used to control the speed of data input into a Flow. 0 disables prefetch (1000 requests is the default)
:param protocol: Communication protocol of the server exposed by the Gateway. This can be a single value or a list of protocols, depending on your chosen Gateway. Choose the convenient protocols from: ['GRPC', 'HTTP', 'WEBSOCKET'].
:param provider: If set, Executor is translated to a custom container compatible with the chosen provider. Choose the convenient providers from: ['NONE', 'SAGEMAKER'].
:param provider: If set, Executor is translated to a custom container compatible with the chosen provider. Choose the convenient providers from: ['NONE', 'SAGEMAKER', 'GCP'].
:param proxy: If set, respect the http_proxy and https_proxy environment variables. otherwise, it will unset these proxy variables before start. gRPC seems to prefer no proxy
:param py_modules: The customized python modules need to be imported before loading the gateway

Expand Down Expand Up @@ -969,7 +969,7 @@ def add(
:param port_monitoring: The port on which the prometheus server is exposed, default is a random port between [49152, 65535]
:param prefer_platform: The preferred target Docker platform. (e.g. "linux/amd64", "linux/arm64")
:param protocol: Communication protocol of the server exposed by the Executor. This can be a single value or a list of protocols, depending on your chosen Gateway. Choose the convenient protocols from: ['GRPC', 'HTTP', 'WEBSOCKET'].
:param provider: If set, Executor is translated to a custom container compatible with the chosen provider. Choose the convenient providers from: ['NONE', 'SAGEMAKER'].
:param provider: If set, Executor is translated to a custom container compatible with the chosen provider. Choose the convenient providers from: ['NONE', 'SAGEMAKER', 'GCP'].
:param py_modules: The customized python modules need to be imported before loading the executor

Note that the recommended way is to only import a single module - a simple python file, if your
Expand Down Expand Up @@ -1132,7 +1132,7 @@ def add(
:param port_monitoring: The port on which the prometheus server is exposed, default is a random port between [49152, 65535]
:param prefer_platform: The preferred target Docker platform. (e.g. "linux/amd64", "linux/arm64")
:param protocol: Communication protocol of the server exposed by the Executor. This can be a single value or a list of protocols, depending on your chosen Gateway. Choose the convenient protocols from: ['GRPC', 'HTTP', 'WEBSOCKET'].
:param provider: If set, Executor is translated to a custom container compatible with the chosen provider. Choose the convenient providers from: ['NONE', 'SAGEMAKER'].
:param provider: If set, Executor is translated to a custom container compatible with the chosen provider. Choose the convenient providers from: ['NONE', 'SAGEMAKER', 'GCP'].
:param py_modules: The customized python modules need to be imported before loading the executor

Note that the recommended way is to only import a single module - a simple python file, if your
Expand Down Expand Up @@ -1396,7 +1396,7 @@ def config_gateway(

Used to control the speed of data input into a Flow. 0 disables prefetch (1000 requests is the default)
:param protocol: Communication protocol of the server exposed by the Gateway. This can be a single value or a list of protocols, depending on your chosen Gateway. Choose the convenient protocols from: ['GRPC', 'HTTP', 'WEBSOCKET'].
:param provider: If set, Executor is translated to a custom container compatible with the chosen provider. Choose the convenient providers from: ['NONE', 'SAGEMAKER'].
:param provider: If set, Executor is translated to a custom container compatible with the chosen provider. Choose the convenient providers from: ['NONE', 'SAGEMAKER', 'GCP'].
:param proxy: If set, respect the http_proxy and https_proxy environment variables. otherwise, it will unset these proxy variables before start. gRPC seems to prefer no proxy
:param py_modules: The customized python modules need to be imported before loading the gateway

Expand Down Expand Up @@ -1496,7 +1496,7 @@ def config_gateway(

Used to control the speed of data input into a Flow. 0 disables prefetch (1000 requests is the default)
:param protocol: Communication protocol of the server exposed by the Gateway. This can be a single value or a list of protocols, depending on your chosen Gateway. Choose the convenient protocols from: ['GRPC', 'HTTP', 'WEBSOCKET'].
:param provider: If set, Executor is translated to a custom container compatible with the chosen provider. Choose the convenient providers from: ['NONE', 'SAGEMAKER'].
:param provider: If set, Executor is translated to a custom container compatible with the chosen provider. Choose the convenient providers from: ['NONE', 'SAGEMAKER', 'GCP'].
:param proxy: If set, respect the http_proxy and https_proxy environment variables. otherwise, it will unset these proxy variables before start. gRPC seems to prefer no proxy
:param py_modules: The customized python modules need to be imported before loading the gateway

Expand Down
11 changes: 6 additions & 5 deletions jina/serve/executors/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -393,7 +393,7 @@ def __init__(
self._add_dynamic_batching(dynamic_batching)
self._add_runtime_args(runtime_args)
self.logger = JinaLogger(self.__class__.__name__, **vars(self.runtime_args))
self._validate_sagemaker()
self._validate_csp()
self._init_instrumentation(runtime_args)
self._init_monitoring()
self._init_workspace = workspace
Expand Down Expand Up @@ -599,14 +599,15 @@ def _add_requests(self, _requests: Optional[Dict]):
f'expect {typename(self)}.{func} to be a function, but receiving {typename(_func)}'
)

def _validate_sagemaker(self):
# sagemaker expects the POST /invocations endpoint to be defined.
def _validate_csp(self):
# csp (sagemaker/azure/gcp) expects the POST /invocations endpoint to be defined.
# if it is not defined, we check if there is only one endpoint defined,
# and if so, we use it as the POST /invocations endpoint, or raise an error
if (
not hasattr(self, 'runtime_args')
or not hasattr(self.runtime_args, 'provider')
or self.runtime_args.provider != ProviderType.SAGEMAKER.value
or self.runtime_args.provider
not in (ProviderType.SAGEMAKER.value, ProviderType.GCP.value)
):
return

Expand Down Expand Up @@ -1084,7 +1085,7 @@ def serve(
:param port_monitoring: The port on which the prometheus server is exposed, default is a random port between [49152, 65535]
:param prefer_platform: The preferred target Docker platform. (e.g. "linux/amd64", "linux/arm64")
:param protocol: Communication protocol of the server exposed by the Executor. This can be a single value or a list of protocols, depending on your chosen Gateway. Choose the convenient protocols from: ['GRPC', 'HTTP', 'WEBSOCKET'].
:param provider: If set, Executor is translated to a custom container compatible with the chosen provider. Choose the convenient providers from: ['NONE', 'SAGEMAKER'].
:param provider: If set, Executor is translated to a custom container compatible with the chosen provider. Choose the convenient providers from: ['NONE', 'SAGEMAKER', 'GCP'].
:param py_modules: The customized python modules need to be imported before loading the executor

Note that the recommended way is to only import a single module - a simple python file, if your
Expand Down
17 changes: 17 additions & 0 deletions jina/serve/runtimes/asyncio.py
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,23 @@ def _get_server(self):
cors=getattr(self.args, 'cors', None),
is_cancel=self.is_cancel,
)
elif (
hasattr(self.args, 'provider')
and self.args.provider == ProviderType.GCP
):
from jina.serve.runtimes.servers.http import GCPHTTPServer

return GCPHTTPServer(
name=self.args.name,
runtime_args=self.args,
req_handler_cls=self.req_handler_cls,
proxy=getattr(self.args, 'proxy', None),
uvicorn_kwargs=getattr(self.args, 'uvicorn_kwargs', None),
ssl_keyfile=getattr(self.args, 'ssl_keyfile', None),
ssl_certfile=getattr(self.args, 'ssl_certfile', None),
cors=getattr(self.args, 'cors', None),
is_cancel=self.is_cancel,
)
elif not hasattr(self.args, 'protocol') or (
len(self.args.protocol) == 1 and self.args.protocol[0] == ProtocolType.GRPC
):
Expand Down
37 changes: 37 additions & 0 deletions jina/serve/runtimes/servers/http.py
Original file line number Diff line number Diff line change
Expand Up @@ -297,3 +297,40 @@ def app(self):
cors=self.cors,
logger=self.logger,
)


class GCPHTTPServer(FastAPIBaseServer):
"""
:class:`GCPHTTPServer` is a FastAPIBaseServer that uses a custom FastAPI app for GCP endpoints

"""

@property
def port(self):
"""Get the port for the GCP server
:return: Return the port for the GCP server, always 8080"""
return 8080

@property
def ports(self):
"""Get the port for the GCP server
:return: Return the port for the GCP server, always 8080"""
return [8080]

@property
def app(self):
"""Get the GCP fastapi app
:return: Return a FastAPI app for the GCP container
"""
return self._request_handler._http_fastapi_gcp_app(
title=self.title,
description=self.description,
no_crud_endpoints=self.no_crud_endpoints,
no_debug_endpoints=self.no_debug_endpoints,
expose_endpoints=self.expose_endpoints,
expose_graphql_endpoint=self.expose_graphql_endpoint,
tracing=self.tracing,
tracer_provider=self.tracer_provider,
cors=self.cors,
logger=self.logger,
)
Loading