Skip to content
Permalink
Browse files

Remove unnecessary check for S3 client initialisation and implement f…

…eature: get pre-signed S3 URL. (#326)

* Remove unnecessary check for S3 client initlialization.

* Implement feature: get pre-signed s3 url.

* Make expiration configurable and make object name consistant.
  • Loading branch information...
leonsim authored and parano committed Oct 8, 2019
1 parent 2f28a2e commit 78c6d6a337a6a2d186e24e01c2f12191a2d0d85b
@@ -56,6 +56,9 @@ default_image_handler_accept_file_extensions = .jpg,.png,.jpeg,.tiff,.webp,.bmp
#
#

[yatai]
bento_uri_default_expiration = 3000

[tensorflow]
#
#
@@ -118,28 +118,33 @@ def dangerously_delete(self, bento_name, bento_version):

class _S3BentoRepository(BentoRepositoryBase):
def __init__(self, base_url):
# Ensure bucket exist and server has permission to manage files under base_path
self.base_url = base_url
self.uri_type = BentoUri.S3

parse_result = urlparse(base_url)
bucket = parse_result.netloc
base_path = parse_result.path
self.bucket = parse_result.netloc
self.base_path = parse_result.path

s3_client = boto3.client("s3")
self.s3_client = boto3.client("s3")

def add(self, bento_name, bento_version):
# Generate pre-signed s3 path for upload
expiration = config('yatai').getint('bento_uri_default_expiration')

object_name = "/".join([self.base_path, bento_name, bento_version])

try:
filename = uuid.uuid4().hex
s3_path = os.path.join(base_path, filename)
s3_client.upload_file(Filename=filename, Bucket=bucket, Key=s3_path)
response = self.s3_client.generate_presigned_post(
self.bucket,
object_name,
Fields=None,
Conditions=None,
ExpiresIn=expiration,
)
except Exception as e:
raise BentoMLRepositoryException(
"Bento is not able to access S3 bucket with error {}".format(e)
"Not able to get pre-signed URL on S3. Error: {}".format(e)
)

def add(self, bento_name, bento_version):
# Generate pre-signed s3 path for upload
raise NotImplementedError
return response

def get(self, bento_name, bento_version):
# Return s3 path containing uploaded Bento files
@@ -61,6 +61,7 @@ def predictTorch(self, input_data):
return self.artifacts.model.predictTorch(input_data)

if sys.version_info >= (3, 6):

@bentoml.api(bentoml.handlers.FastaiImageHandler)
def predictFastaiImage(self, input_data):
return self.artifacts.model.predictImage(input_data)
@@ -71,6 +72,7 @@ def predictFastaiImage(self, input_data):
def predictFastaiImages(self, original, compared):
return all(original.data[0, 0] == compared.data[0, 0])


@pytest.fixture()
def bento_service():
"""Create a new TestBentoService
@@ -9,6 +9,7 @@ def test_fastai_image_handler(capsys, tmpdir):
# fast ai is required 3.6 or higher.
assert True
else:

class ImageHandlerModelForFastai(bentoml.BentoService):
@bentoml.api(FastaiImageHandler)
def predict(self, image):
@@ -38,6 +38,7 @@ def test_fastai_image_handler_pip_dependencies():
# fast ai is required 3.6 or higher.
assert True
else:

class TestFastAiImageService(bentoml.BentoService):
@bentoml.api(bentoml.handlers.FastaiImageHandler)
def test(self, image):

0 comments on commit 78c6d6a

Please sign in to comment.
You can’t perform that action at this time.