Skip to content
Permalink
Browse files

Implement feature uploading bento repo to S3 (#356)

  • Loading branch information...
leonsim authored and parano committed Oct 26, 2019
1 parent 00555cd commit eeb398cfed184e9e45a2f16e59be920ecfa42660
@@ -122,3 +122,6 @@ built-docs

# MacOS X
.DS_Store

# sqlite
storage.db
@@ -100,9 +100,7 @@ def generate_aws_lambda_serverless_config(
"stage": stage,
"name": 'aws',
'runtime': runtime,
"apiGateway": {
"binaryMediaTypes": ['image/*']
}
"apiGateway": {"binaryMediaTypes": ['image/*']},
},
"functions": {
api_name: {
@@ -128,7 +126,7 @@ def generate_aws_lambda_serverless_config(
serverless_project_dir
),
],
},
}
},
}

@@ -22,7 +22,7 @@
package='bentoml',
syntax='proto3',
serialized_options=None,
serialized_pb=_b('\n\x10repository.proto\x12\x07\x62\x65ntoml\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x0cstatus.proto\"\x9c\x01\n\x08\x42\x65ntoUri\x12+\n\x04type\x18\x01 \x01(\x0e\x32\x1d.bentoml.BentoUri.StorageType\x12\x0b\n\x03uri\x18\x02 \x01(\t\"V\n\x0bStorageType\x12\t\n\x05UNSET\x10\x00\x12\t\n\x05LOCAL\x10\x01\x12\x06\n\x02S3\x10\x02\x12\x07\n\x03GCS\x10\x03\x12\x16\n\x12\x41ZURE_BLOB_STORAGE\x10\x04\x12\x08\n\x04HDFS\x10\x05\"\x83\x04\n\x14\x42\x65ntoServiceMetadata\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12.\n\ncreated_at\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x03\x65nv\x18\x04 \x01(\x0b\x32-.bentoml.BentoServiceMetadata.BentoServiceEnv\x12>\n\tartifacts\x18\x05 \x03(\x0b\x32+.bentoml.BentoServiceMetadata.BentoArtifact\x12;\n\x04\x61pis\x18\x06 \x03(\x0b\x32-.bentoml.BentoServiceMetadata.BentoServiceApi\x1ah\n\x0f\x42\x65ntoServiceEnv\x12\x10\n\x08setup_sh\x18\x01 \x01(\t\x12\x11\n\tconda_env\x18\x02 \x01(\t\x12\x18\n\x10pip_dependencies\x18\x03 \x01(\t\x12\x16\n\x0epython_version\x18\x04 \x01(\t\x1a\x34\n\rBentoArtifact\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x15\n\rartifact_type\x18\x02 \x01(\t\x1a\x43\n\x0f\x42\x65ntoServiceApi\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x14\n\x0chandler_type\x18\x02 \x01(\t\x12\x0c\n\x04\x64ocs\x18\x03 \x01(\t\"\xac\x01\n\x05\x42\x65nto\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x1e\n\x03uri\x18\x03 \x01(\x0b\x32\x11.bentoml.BentoUri\x12=\n\x16\x62\x65nto_service_metadata\x18\x04 \x01(\x0b\x32\x1d.bentoml.BentoServiceMetadata\x12%\n\x06status\x18\x05 \x01(\x0b\x32\x15.bentoml.UploadStatus\"<\n\x0f\x41\x64\x64\x42\x65ntoRequest\x12\x12\n\nbento_name\x18\x01 \x01(\t\x12\x15\n\rbento_version\x18\x02 \x01(\t\"S\n\x10\x41\x64\x64\x42\x65ntoResponse\x12\x1f\n\x06status\x18\x01 \x01(\x0b\x32\x0f.bentoml.Status\x12\x1e\n\x03uri\x18\x02 \x01(\x0b\x32\x11.bentoml.BentoUri\"\xe5\x01\n\x0cUploadStatus\x12,\n\x06status\x18\x01 \x01(\x0e\x32\x1c.bentoml.UploadStatus.Status\x12.\n\nupdated_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x12\n\npercentage\x18\x03 \x01(\x05\x12\x15\n\rerror_message\x18\x04 \x01(\t\"L\n\x06Status\x12\x11\n\rUNINITIALIZED\x10\x00\x12\r\n\tUPLOADING\x10\x01\x12\x08\n\x04\x44ONE\x10\x02\x12\t\n\x05\x45RROR\x10\x03\x12\x0b\n\x07TIMEOUT\x10\x04\"\xa6\x01\n\x12UpdateBentoRequest\x12\x12\n\nbento_name\x18\x01 \x01(\t\x12\x15\n\rbento_version\x18\x02 \x01(\t\x12,\n\rupload_status\x18\x03 \x01(\x0b\x32\x15.bentoml.UploadStatus\x12\x37\n\x10service_metadata\x18\x04 \x01(\x0b\x32\x1d.bentoml.BentoServiceMetadata\"6\n\x13UpdateBentoResponse\x12\x1f\n\x06status\x18\x01 \x01(\x0b\x32\x0f.bentoml.Status\"J\n\x1d\x44\x61ngerouslyDeleteBentoRequest\x12\x12\n\nbento_name\x18\x01 \x01(\t\x12\x15\n\rbento_version\x18\x02 \x01(\t\"A\n\x1e\x44\x61ngerouslyDeleteBentoResponse\x12\x1f\n\x06status\x18\x01 \x01(\x0b\x32\x0f.bentoml.Status\"<\n\x0fGetBentoRequest\x12\x12\n\nbento_name\x18\x01 \x01(\t\x12\x15\n\rbento_version\x18\x02 \x01(\t\"R\n\x10GetBentoResponse\x12\x1f\n\x06status\x18\x01 \x01(\x0b\x32\x0f.bentoml.Status\x12\x1d\n\x05\x62\x65nto\x18\x02 \x01(\x0b\x32\x0e.bentoml.Bento\"U\n\x10ListBentoRequest\x12\x12\n\nbento_name\x18\x01 \x01(\t\x12\x0e\n\x06offset\x18\x02 \x01(\x05\x12\r\n\x05limit\x18\x03 \x01(\x05\x12\x0e\n\x06\x66ilter\x18\x04 \x01(\t\"T\n\x11ListBentoResponse\x12\x1f\n\x06status\x18\x01 \x01(\x0b\x32\x0f.bentoml.Status\x12\x1e\n\x06\x62\x65ntos\x18\x02 \x03(\x0b\x32\x0e.bentoml.Bentob\x06proto3')
serialized_pb=_b('\n\x10repository.proto\x12\x07\x62\x65ntoml\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x0cstatus.proto\"\xb7\x01\n\x08\x42\x65ntoUri\x12+\n\x04type\x18\x01 \x01(\x0e\x32\x1d.bentoml.BentoUri.StorageType\x12\x0b\n\x03uri\x18\x02 \x01(\t\x12\x19\n\x11\x61\x64\x64itional_fields\x18\x03 \x01(\t\"V\n\x0bStorageType\x12\t\n\x05UNSET\x10\x00\x12\t\n\x05LOCAL\x10\x01\x12\x06\n\x02S3\x10\x02\x12\x07\n\x03GCS\x10\x03\x12\x16\n\x12\x41ZURE_BLOB_STORAGE\x10\x04\x12\x08\n\x04HDFS\x10\x05\"\x83\x04\n\x14\x42\x65ntoServiceMetadata\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12.\n\ncreated_at\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x03\x65nv\x18\x04 \x01(\x0b\x32-.bentoml.BentoServiceMetadata.BentoServiceEnv\x12>\n\tartifacts\x18\x05 \x03(\x0b\x32+.bentoml.BentoServiceMetadata.BentoArtifact\x12;\n\x04\x61pis\x18\x06 \x03(\x0b\x32-.bentoml.BentoServiceMetadata.BentoServiceApi\x1ah\n\x0f\x42\x65ntoServiceEnv\x12\x10\n\x08setup_sh\x18\x01 \x01(\t\x12\x11\n\tconda_env\x18\x02 \x01(\t\x12\x18\n\x10pip_dependencies\x18\x03 \x01(\t\x12\x16\n\x0epython_version\x18\x04 \x01(\t\x1a\x34\n\rBentoArtifact\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x15\n\rartifact_type\x18\x02 \x01(\t\x1a\x43\n\x0f\x42\x65ntoServiceApi\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x14\n\x0chandler_type\x18\x02 \x01(\t\x12\x0c\n\x04\x64ocs\x18\x03 \x01(\t\"\xac\x01\n\x05\x42\x65nto\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x1e\n\x03uri\x18\x03 \x01(\x0b\x32\x11.bentoml.BentoUri\x12=\n\x16\x62\x65nto_service_metadata\x18\x04 \x01(\x0b\x32\x1d.bentoml.BentoServiceMetadata\x12%\n\x06status\x18\x05 \x01(\x0b\x32\x15.bentoml.UploadStatus\"<\n\x0f\x41\x64\x64\x42\x65ntoRequest\x12\x12\n\nbento_name\x18\x01 \x01(\t\x12\x15\n\rbento_version\x18\x02 \x01(\t\"S\n\x10\x41\x64\x64\x42\x65ntoResponse\x12\x1f\n\x06status\x18\x01 \x01(\x0b\x32\x0f.bentoml.Status\x12\x1e\n\x03uri\x18\x02 \x01(\x0b\x32\x11.bentoml.BentoUri\"\xe5\x01\n\x0cUploadStatus\x12,\n\x06status\x18\x01 \x01(\x0e\x32\x1c.bentoml.UploadStatus.Status\x12.\n\nupdated_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x12\n\npercentage\x18\x03 \x01(\x05\x12\x15\n\rerror_message\x18\x04 \x01(\t\"L\n\x06Status\x12\x11\n\rUNINITIALIZED\x10\x00\x12\r\n\tUPLOADING\x10\x01\x12\x08\n\x04\x44ONE\x10\x02\x12\t\n\x05\x45RROR\x10\x03\x12\x0b\n\x07TIMEOUT\x10\x04\"\xa6\x01\n\x12UpdateBentoRequest\x12\x12\n\nbento_name\x18\x01 \x01(\t\x12\x15\n\rbento_version\x18\x02 \x01(\t\x12,\n\rupload_status\x18\x03 \x01(\x0b\x32\x15.bentoml.UploadStatus\x12\x37\n\x10service_metadata\x18\x04 \x01(\x0b\x32\x1d.bentoml.BentoServiceMetadata\"6\n\x13UpdateBentoResponse\x12\x1f\n\x06status\x18\x01 \x01(\x0b\x32\x0f.bentoml.Status\"J\n\x1d\x44\x61ngerouslyDeleteBentoRequest\x12\x12\n\nbento_name\x18\x01 \x01(\t\x12\x15\n\rbento_version\x18\x02 \x01(\t\"A\n\x1e\x44\x61ngerouslyDeleteBentoResponse\x12\x1f\n\x06status\x18\x01 \x01(\x0b\x32\x0f.bentoml.Status\"<\n\x0fGetBentoRequest\x12\x12\n\nbento_name\x18\x01 \x01(\t\x12\x15\n\rbento_version\x18\x02 \x01(\t\"R\n\x10GetBentoResponse\x12\x1f\n\x06status\x18\x01 \x01(\x0b\x32\x0f.bentoml.Status\x12\x1d\n\x05\x62\x65nto\x18\x02 \x01(\x0b\x32\x0e.bentoml.Bento\"U\n\x10ListBentoRequest\x12\x12\n\nbento_name\x18\x01 \x01(\t\x12\x0e\n\x06offset\x18\x02 \x01(\x05\x12\r\n\x05limit\x18\x03 \x01(\x05\x12\x0e\n\x06\x66ilter\x18\x04 \x01(\t\"T\n\x11ListBentoResponse\x12\x1f\n\x06status\x18\x01 \x01(\x0b\x32\x0f.bentoml.Status\x12\x1e\n\x06\x62\x65ntos\x18\x02 \x03(\x0b\x32\x0e.bentoml.Bentob\x06proto3')
,
dependencies=[google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,status__pb2.DESCRIPTOR,])

@@ -61,8 +61,8 @@
],
containing_type=None,
serialized_options=None,
serialized_start=147,
serialized_end=233,
serialized_start=174,
serialized_end=260,
)
_sym_db.RegisterEnumDescriptor(_BENTOURI_STORAGETYPE)

@@ -95,8 +95,8 @@
],
containing_type=None,
serialized_options=None,
serialized_start=1229,
serialized_end=1305,
serialized_start=1256,
serialized_end=1332,
)
_sym_db.RegisterEnumDescriptor(_UPLOADSTATUS_STATUS)

@@ -122,6 +122,13 @@
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='additional_fields', full_name='bentoml.BentoUri.additional_fields', index=2,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
@@ -136,7 +143,7 @@
oneofs=[
],
serialized_start=77,
serialized_end=233,
serialized_end=260,
)


@@ -187,8 +194,8 @@
extension_ranges=[],
oneofs=[
],
serialized_start=524,
serialized_end=628,
serialized_start=551,
serialized_end=655,
)

_BENTOSERVICEMETADATA_BENTOARTIFACT = _descriptor.Descriptor(
@@ -224,8 +231,8 @@
extension_ranges=[],
oneofs=[
],
serialized_start=630,
serialized_end=682,
serialized_start=657,
serialized_end=709,
)

_BENTOSERVICEMETADATA_BENTOSERVICEAPI = _descriptor.Descriptor(
@@ -268,8 +275,8 @@
extension_ranges=[],
oneofs=[
],
serialized_start=684,
serialized_end=751,
serialized_start=711,
serialized_end=778,
)

_BENTOSERVICEMETADATA = _descriptor.Descriptor(
@@ -333,8 +340,8 @@
extension_ranges=[],
oneofs=[
],
serialized_start=236,
serialized_end=751,
serialized_start=263,
serialized_end=778,
)


@@ -392,8 +399,8 @@
extension_ranges=[],
oneofs=[
],
serialized_start=754,
serialized_end=926,
serialized_start=781,
serialized_end=953,
)


@@ -430,8 +437,8 @@
extension_ranges=[],
oneofs=[
],
serialized_start=928,
serialized_end=988,
serialized_start=955,
serialized_end=1015,
)


@@ -468,8 +475,8 @@
extension_ranges=[],
oneofs=[
],
serialized_start=990,
serialized_end=1073,
serialized_start=1017,
serialized_end=1100,
)


@@ -521,8 +528,8 @@
extension_ranges=[],
oneofs=[
],
serialized_start=1076,
serialized_end=1305,
serialized_start=1103,
serialized_end=1332,
)


@@ -573,8 +580,8 @@
extension_ranges=[],
oneofs=[
],
serialized_start=1308,
serialized_end=1474,
serialized_start=1335,
serialized_end=1501,
)


@@ -604,8 +611,8 @@
extension_ranges=[],
oneofs=[
],
serialized_start=1476,
serialized_end=1530,
serialized_start=1503,
serialized_end=1557,
)


@@ -642,8 +649,8 @@
extension_ranges=[],
oneofs=[
],
serialized_start=1532,
serialized_end=1606,
serialized_start=1559,
serialized_end=1633,
)


@@ -673,8 +680,8 @@
extension_ranges=[],
oneofs=[
],
serialized_start=1608,
serialized_end=1673,
serialized_start=1635,
serialized_end=1700,
)


@@ -711,8 +718,8 @@
extension_ranges=[],
oneofs=[
],
serialized_start=1675,
serialized_end=1735,
serialized_start=1702,
serialized_end=1762,
)


@@ -749,8 +756,8 @@
extension_ranges=[],
oneofs=[
],
serialized_start=1737,
serialized_end=1819,
serialized_start=1764,
serialized_end=1846,
)


@@ -801,8 +808,8 @@
extension_ranges=[],
oneofs=[
],
serialized_start=1821,
serialized_end=1906,
serialized_start=1848,
serialized_end=1933,
)


@@ -839,8 +846,8 @@
extension_ranges=[],
oneofs=[
],
serialized_start=1908,
serialized_end=1992,
serialized_start=1935,
serialized_end=2019,
)

_BENTOURI.fields_by_name['type'].enum_type = _BENTOURI_STORAGETYPE
@@ -17,6 +17,7 @@
from __future__ import print_function

import os
import json
import shutil
import boto3
from six import add_metaclass
@@ -121,7 +122,7 @@ def __init__(self, base_url):

parse_result = urlparse(base_url)
self.bucket = parse_result.netloc
self.base_path = parse_result.path
self.base_path = parse_result.path.lstrip('/')

self.s3_client = boto3.client("s3")

@@ -135,7 +136,7 @@ def _get_object_name(self, bento_name, bento_version):
def add(self, bento_name, bento_version):
# Generate pre-signed s3 path for upload

object_name = self._get_object_name(bento_name, bento_version)
object_name = self._get_object_name(bento_name, bento_version) + '.tar.gz'

try:
response = self.s3_client.generate_presigned_post(
@@ -149,7 +150,12 @@ def add(self, bento_name, bento_version):
raise BentoMLRepositoryException(
"Not able to get pre-signed URL on S3. Error: {}".format(e)
)
return response

return BentoUri(
type=self.uri_type,
uri=response['url'],
additional_fields=json.dumps(response['fields']),
)

def get(self, bento_name, bento_version):
# Return s3 path containing uploaded Bento files
@@ -14,7 +14,14 @@

# List of APIs for accessing remote or local yatai service via Python


import io
import json
import logging
import tarfile
import requests
import tempfile


from bentoml.service import BentoService
from bentoml.exceptions import BentoMLException
@@ -79,21 +86,63 @@ def upload_bento_service(bento_service, base_path=None, version=None):
# Saving directory to path managed by LocalBentoRepository
save_to_dir(bento_service, response.uri.uri)

upload_status = UploadStatus(status=UploadStatus.DONE)
upload_status.updated_at.GetCurrentTime()
update_bento_req = UpdateBentoRequest(
bento_name=bento_service.name,
bento_version=bento_service.version,
upload_status=upload_status,
service_metadata=bento_service._get_bento_service_metadata_pb(),
)
yatai.UpdateBento(update_bento_req)
update_bento_upload_progress(yatai, bento_service)

# Return URI to saved bento in repository storage
return response.uri.uri
elif response.uri.type == BentoUri.S3:
with tempfile.TemporaryDirectory() as tmpdir:

save_to_dir(bento_service, tmpdir)

fileobj = io.BytesIO()
with tarfile.open(mode="w:gz", fileobj=fileobj) as tar:
tar.add(tmpdir, arcname=bento_service.name)
fileobj.seek(0, 0)

files = {'file': ('dummy', fileobj)}
http_response = requests.post(
response.uri.uri,
data=json.loads(response.uri.additional_fields),
files=files,
)

if http_response.status_code != 204:
update_bento_upload_progress(yatai, bento_service, UploadStatus.ERROR)

raise BentoMLException(
"Error saving Bento to S3 with status code {} and error detail "
"is {}".format(http_response.status_code, http_response.text)
)

logger.info(
"Successfully saved Bento '%s:%s' to S3: %s",
bento_service.name,
bento_service.version,
response.uri.uri,
)

update_bento_upload_progress(yatai, bento_service)

return response.uri.uri

else:
raise BentoMLException(
"Error saving Bento to target repository, URI type %s at %s not supported"
% response.uri.type,
response.uri.uri,
)


def update_bento_upload_progress(
yatai, bento_service, status=UploadStatus.DONE, progress=None
):
upload_status = UploadStatus(status=status)
upload_status.updated_at.GetCurrentTime()
update_bento_req = UpdateBentoRequest(
bento_name=bento_service.name,
bento_version=bento_service.version,
upload_status=upload_status,
service_metadata=bento_service._get_bento_service_metadata_pb(),
)
yatai.UpdateBento(update_bento_req)

0 comments on commit eeb398c

Please sign in to comment.
You can’t perform that action at this time.