diff --git a/bentoml/cli/deployment.py b/bentoml/cli/deployment.py index 573c2e3f3f8..ed7a644389a 100644 --- a/bentoml/cli/deployment.py +++ b/bentoml/cli/deployment.py @@ -303,17 +303,24 @@ def create( 'timeout': timeout, } yatai_service = get_yatai_service() - result = create_deployment( - name, - namespace, - bento_name, - bento_version, - platform, - operator_spec, - parse_key_value_pairs(labels), - parse_key_value_pairs(annotations), - yatai_service, - ) + try: + result = create_deployment( + name, + namespace, + bento_name, + bento_version, + platform, + operator_spec, + parse_key_value_pairs(labels), + parse_key_value_pairs(annotations), + yatai_service, + ) + except BentoMLException as e: + _echo( + 'Failed to create deployment {}.: {}'.format(name, str(e)), + CLI_COLOR_ERROR, + ) + return if result.status.status_code != status_pb2.Status.OK: _echo( diff --git a/bentoml/deployment/aws_lambda/__init__.py b/bentoml/deployment/aws_lambda/__init__.py index a77d07f38e0..8a3ef2c2811 100644 --- a/bentoml/deployment/aws_lambda/__init__.py +++ b/bentoml/deployment/aws_lambda/__init__.py @@ -44,7 +44,6 @@ from bentoml.exceptions import BentoMLException from bentoml.proto.deployment_pb2 import ( ApplyDeploymentResponse, - Deployment, DeploymentState, DescribeDeploymentResponse, DeleteDeploymentResponse, @@ -307,31 +306,19 @@ def _apply( ) return ApplyDeploymentResponse(status=Status.INTERNAL(str(e))) - logger.info('Finish deployed lambda project, fetching latest status') - res_deployment_pb = Deployment(state=DeploymentState()) - res_deployment_pb.CopyFrom(deployment_pb) - state = self.describe(res_deployment_pb, yatai_service).state - res_deployment_pb.state.CopyFrom(state) + deployment_pb.state.state = DeploymentState.PENDING + return ApplyDeploymentResponse(status=Status.OK(), deployment=deployment_pb) + except BentoMLException as error: + deployment_pb.state = DeploymentState( + state=DeploymentState.ERROR, error_message='Error: {}'.format(error) + ) return ApplyDeploymentResponse( - status=Status.OK(), deployment=res_deployment_pb + status=exception_to_return_status(error), deployment=deployment_pb ) - except BentoMLException as error: - return ApplyDeploymentResponse(status=exception_to_return_status(error)) - def delete(self, deployment_pb, yatai_service): try: logger.debug('Deleting AWS Lambda deployment') - describe_state_result = self.describe(deployment_pb, yatai_service).state - if describe_state_result.state != DeploymentState.RUNNING: - message = ( - 'Failed to delete, no active deployment {name}. ' - 'The current state is {state}'.format( - name=deployment_pb.name, - state=DeploymentState.State.Name(describe_state_result.state), - ) - ) - return DeleteDeploymentResponse(status=Status.ABORTED(message)) deployment_spec = deployment_pb.spec lambda_deployment_config = deployment_spec.aws_lambda_operator_config @@ -340,10 +327,11 @@ def delete(self, deployment_pb, yatai_service): stack_name = generate_aws_compatible_string( deployment_pb.namespace + '-' + deployment_pb.name ) - deployment_info_json = json.loads(describe_state_result.info_json) - bucket_name = deployment_info_json.get('s3_bucket') - if bucket_name: - _cleanup_s3_bucket(bucket_name, lambda_deployment_config.region) + if deployment_pb.state.info_json: + deployment_info_json = json.loads(deployment_pb.state.info_json) + bucket_name = deployment_info_json.get('s3_bucket') + if bucket_name: + _cleanup_s3_bucket(bucket_name, lambda_deployment_config.region) logger.debug( 'Deleting AWS CloudFormation: %s that includes Lambda function ' diff --git a/bentoml/deployment/aws_lambda/utils.py b/bentoml/deployment/aws_lambda/utils.py index e2a883aa42d..4f518fbebdb 100644 --- a/bentoml/deployment/aws_lambda/utils.py +++ b/bentoml/deployment/aws_lambda/utils.py @@ -33,10 +33,10 @@ def ensure_sam_available_or_raise(): try: import samcli - if list(map(int, samcli.__version__.split('.'))) < [0, 33, 1]: + if samcli.__version__ != '0.33.1': raise BentoMLException( - 'aws-sam-cli package requires version 0.33.1 or ' - 'higher. Update the package with `pip install -U aws-sam-cli`' + 'aws-sam-cli package requires version 0.33.1 ' + 'Install the package with `pip install -U aws-sam-cli==0.33.1`' ) except ImportError: raise ImportError( @@ -172,12 +172,12 @@ def init_sam_project( requirement_txt_path = os.path.join(bento_service_bundle_path, 'requirements.txt') shutil.copy(requirement_txt_path, function_path) - # Copy bundled pip dependencies - logger.debug('Coping bundled_dependencies') bundled_dep_path = os.path.join( bento_service_bundle_path, 'bundled_pip_dependencies' ) if os.path.isdir(bundled_dep_path): + # Copy bundled pip dependencies + logger.debug('Coping bundled_dependencies') shutil.copytree( bundled_dep_path, os.path.join(function_path, 'bundled_pip_dependencies') ) diff --git a/bentoml/deployment/utils.py b/bentoml/deployment/utils.py index b33ac093b3e..3a91c5e2df2 100644 --- a/bentoml/deployment/utils.py +++ b/bentoml/deployment/utils.py @@ -70,7 +70,7 @@ def ensure_docker_available_or_raise(): subprocess.check_output(['docker', 'info']) except subprocess.CalledProcessError as error: raise BentoMLException( - 'Error executing docker command: {}'.format(error.output) + 'Error executing docker command: {}'.format(error.output.decode()) ) except not_found_error: raise BentoMLMissingDependencyException( diff --git a/bentoml/yatai/python_api.py b/bentoml/yatai/python_api.py index f6999340a1b..299c6e3ecd5 100644 --- a/bentoml/yatai/python_api.py +++ b/bentoml/yatai/python_api.py @@ -213,90 +213,98 @@ def create_deployment( yatai_service = get_yatai_service() - try: - # Make sure there is no active deployment with the same deployment name - get_deployment_pb = yatai_service.GetDeployment( - GetDeploymentRequest(deployment_name=deployment_name, namespace=namespace) + # Make sure there is no active deployment with the same deployment name + get_deployment_pb = yatai_service.GetDeployment( + GetDeploymentRequest(deployment_name=deployment_name, namespace=namespace) + ) + if get_deployment_pb.status.status_code == status_pb2.Status.OK: + raise BentoMLDeploymentException( + 'Deployment "{name}" already existed, use Update or Apply for updating ' + 'existing deployment, delete the deployment, or use a different deployment ' + 'name'.format(name=deployment_name) ) - if get_deployment_pb.status.status_code == status_pb2.Status.OK: - raise BentoMLDeploymentException( - 'Deployment "{name}" already existed, use Update or Apply for updating' - 'existing deployment, or create the deployment with a different name or' - 'under a different deployment namespace'.format(name=deployment_name) - ) - if get_deployment_pb.status.status_code != status_pb2.Status.NOT_FOUND: - raise BentoMLDeploymentException( - 'Failed accesing YataiService deployment store. {error_code}:' - '{error_message}'.format( - error_code=Status.Name(get_deployment_pb.status.status_code), - error_message=get_deployment_pb.status.error_message, - ) + if get_deployment_pb.status.status_code != status_pb2.Status.NOT_FOUND: + raise BentoMLDeploymentException( + 'Failed accesing YataiService deployment store. {error_code}:' + '{error_message}'.format( + error_code=Status.Name(get_deployment_pb.status.status_code), + error_message=get_deployment_pb.status.error_message, ) + ) - deployment_dict = { - "name": deployment_name, - "namespace": namespace or config().get('deployment', 'default_namespace'), - "labels": labels, - "annotations": annotations, - "spec": { - "bento_name": bento_name, - "bento_version": bento_version, - "operator": platform, - }, + deployment_dict = { + "name": deployment_name, + "namespace": namespace or config().get('deployment', 'default_namespace'), + "labels": labels, + "annotations": annotations, + "spec": { + "bento_name": bento_name, + "bento_version": bento_version, + "operator": platform, + }, + } + + operator = platform.replace('-', '_').upper() + try: + operator_value = DeploymentSpec.DeploymentOperator.Value(operator) + except ValueError: + return ApplyDeploymentResponse( + status=Status.INVALID_ARGUMENT('Invalid platform "{}"'.format(platform)) + ) + if operator_value == DeploymentSpec.AWS_SAGEMAKER: + deployment_dict['spec']['sagemaker_operator_config'] = { + 'region': operator_spec.get('region') + or config().get('aws', 'default_region'), + 'instance_count': operator_spec.get('instance_count') + or config().getint('sagemaker', 'default_instance_count'), + 'instance_type': operator_spec.get('instance_type') + or config().get('sagemaker', 'default_instance_type'), + 'api_name': operator_spec.get('api_name', ''), + } + elif operator_value == DeploymentSpec.AWS_LAMBDA: + deployment_dict['spec']['aws_lambda_operator_config'] = { + 'region': operator_spec.get('region') + or config().get('aws', 'default_region') + } + for field in ['api_name', 'memory_size', 'timeout']: + if operator_spec.get(field): + deployment_dict['spec']['aws_lambda_operator_config'][ + field + ] = operator_spec[field] + elif operator_value == DeploymentSpec.GCP_FCUNTION: + deployment_dict['spec']['gcp_function_operatorConfig'] = { + 'region': operator_spec.get('region') + or config().get('google-cloud', 'default_region') } + if operator_spec.get('api_name'): + deployment_dict['spec']['gcp_function_operator_config'][ + 'api_name' + ] = operator_spec['api_name'] + elif operator_value == DeploymentSpec.KUBERNETES: + deployment_dict['spec']['kubernetes_operator_config'] = { + 'kube_namespace': operator_spec.get('kube_namespace', ''), + 'replicas': operator_spec.get('replicas', 0), + 'service_name': operator_spec.get('service_name', ''), + 'service_type': operator_spec.get('service_type', ''), + } + else: + raise BentoMLDeploymentException( + 'Platform "{}" is not supported in the current version of ' + 'BentoML'.format(platform) + ) - operator = platform.replace('-', '_').upper() - try: - operator_value = DeploymentSpec.DeploymentOperator.Value(operator) - except ValueError: - return ApplyDeploymentResponse( - status=Status.INVALID_ARGUMENT('Invalid platform "{}"'.format(platform)) - ) - if operator_value == DeploymentSpec.AWS_SAGEMAKER: - deployment_dict['spec']['sagemaker_operator_config'] = { - 'region': operator_spec.get('region') - or config().get('aws', 'default_region'), - 'instance_count': operator_spec.get('instance_count') - or config().getint('sagemaker', 'default_instance_count'), - 'instance_type': operator_spec.get('instance_type') - or config().get('sagemaker', 'default_instance_type'), - 'api_name': operator_spec.get('api_name', ''), - } - elif operator_value == DeploymentSpec.AWS_LAMBDA: - deployment_dict['spec']['aws_lambda_operator_config'] = { - 'region': operator_spec.get('region') - or config().get('aws', 'default_region') - } - for field in ['api_name', 'memory_size', 'timeout']: - if operator_spec.get(field): - deployment_dict['spec']['aws_lambda_operator_config'][ - field - ] = operator_spec[field] - elif operator_value == DeploymentSpec.GCP_FCUNTION: - deployment_dict['spec']['gcp_function_operatorConfig'] = { - 'region': operator_spec.get('region') - or config().get('google-cloud', 'default_region') - } - if operator_spec.get('api_name'): - deployment_dict['spec']['gcp_function_operator_config'][ - 'api_name' - ] = operator_spec['api_name'] - elif operator_value == DeploymentSpec.KUBERNETES: - deployment_dict['spec']['kubernetes_operator_config'] = { - 'kube_namespace': operator_spec.get('kube_namespace', ''), - 'replicas': operator_spec.get('replicas', 0), - 'service_name': operator_spec.get('service_name', ''), - 'service_type': operator_spec.get('service_type', ''), - } - else: - raise BentoMLDeploymentException( - 'Platform "{}" is not supported in the current version of ' - 'BentoML'.format(platform) - ) + apply_response = apply_deployment(deployment_dict, yatai_service) - return apply_deployment(deployment_dict, yatai_service) - except BentoMLException as error: - return ApplyDeploymentResponse(status=Status.INTERNAL(str(error))) + if apply_response.status.status_code == status_pb2.Status.OK: + describe_response = describe_deployment( + deployment_name, namespace, yatai_service + ) + if describe_response.status.status_code == status_pb2.Status.OK: + deployment_state = describe_response.state + apply_response.deployment.state.CopyFrom(deployment_state) + return apply_response + + return apply_response # TODO update_deployment is not finished. It will be working on along with cli command diff --git a/bentoml/yatai/yatai_service_impl.py b/bentoml/yatai/yatai_service_impl.py index 7974c05cb2c..9c1644463dc 100644 --- a/bentoml/yatai/yatai_service_impl.py +++ b/bentoml/yatai/yatai_service_impl.py @@ -117,8 +117,32 @@ def ApplyDeployment(self, request, context=None): # deploying to target platform response = operator.apply(request.deployment, self, previous_deployment) - # update deployment state - self.deployment_store.insert_or_update(response.deployment) + if response.status.status_code == status_pb2.Status.OK: + # update deployment state + if response and response.deployment: + self.deployment_store.insert_or_update(response.deployment) + else: + raise BentoMLException( + "DeploymentOperator Internal Error: Invalid Response" + ) + logger.info( + "ApplyDeployment (%s, namespace %s) succeeded", + request.deployment.name, + request.deployment.namespace, + ) + else: + if not previous_deployment: + # When failed to create the deployment, delete it from active + # deployments records + self.deployment_store.delete( + request.deployment.name, request.deployment.namespace + ) + logger.debug( + "ApplyDeployment (%s, namespace %s) failed: %s", + request.deployment.name, + request.deployment.namespace, + response.status.error_message, + ) return response @@ -299,6 +323,7 @@ def DangerouslyDeleteBento(self, request, context=None): self.bento_metadata_store.dangerously_delete( request.bento_name, request.bento_version ) + self.repo.dangerously_delete(request.bento_name, request.bento_version) except BentoMLException as e: logger.error("INTERNAL ERROR: %s", e) return DangerouslyDeleteBentoResponse(status=Status.INTERNAL(str(e))) diff --git a/scripts/e2e_lambda_deployment.py b/scripts/e2e_lambda_deployment.py index e347e1faf76..8091d214251 100644 --- a/scripts/e2e_lambda_deployment.py +++ b/scripts/e2e_lambda_deployment.py @@ -1,4 +1,3 @@ -import shutil import subprocess import logging import uuid @@ -10,8 +9,6 @@ from bentoml import BentoService, load, api, env, artifacts from bentoml.artifact import PickleArtifact from bentoml.handlers import DataframeHandler -from bentoml.proto.repository_pb2 import DangerouslyDeleteBentoRequest -from bentoml.yatai import get_yatai_service logger = logging.getLogger('bentoml.test') @@ -26,16 +23,15 @@ def predict(self, df): if __name__ == '__main__': - logger.info('E2E DEPLOYMENT TEST FOR LAMBDA:::: Training iris classifier') + logger.info('Training iris classifier') clf = svm.SVC(gamma='scale') iris = datasets.load_iris() X, y = iris.data, iris.target clf.fit(X, y) - logger.info( - 'E2E DEPLOYMENT TEST FOR LAMBDA:::: Bundling iris classifier with BentoML' - ) - iris_clf_service = IrisClassifier.pack(clf=clf) + logger.info('Bundling iris classifier with BentoML') + iris_clf_service = IrisClassifier() + iris_clf_service.pack('clf', clf) saved_path = iris_clf_service.save() loaded_service = load(saved_path) @@ -43,14 +39,14 @@ def predict(self, df): deployment_failed = False logger.info( - 'E2E DEPLOYMENT TEST FOR LAMBDA:::: Creating AWS Lambda test deployment ' - 'for iris classifier with BentoML CLI' + 'Creating AWS Lambda test deployment for iris classifier with BentoML CLI' ) bento_name = '{}:{}'.format(loaded_service.name, loaded_service.version) - random_hash = uuid.uuid4().hex[:6].upper() + random_hash = uuid.uuid4().hex[:6] deployment_name = 'tests-lambda-e2e-{}'.format(random_hash) create_deployment_command = [ 'bentoml', + '--verbose', 'deploy', 'create', deployment_name, @@ -61,15 +57,12 @@ def predict(self, df): '--region', 'us-west-2', ] - logger.info( - 'E2E DEPLOYMENT TEST FOR LAMBDA:::: Deploy ' - 'command: {}'.format(create_deployment_command) - ) + logger.info('Deploy command: {}'.format(create_deployment_command)) with subprocess.Popen( create_deployment_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) as proc: create_deployment_stdout = proc.stdout.read().decode('utf-8') - logger.info('E2E DEPLOYMENT TEST FOR LAMBDA:::: Finish deploying to AWS Lambda') + logger.info('Finish deploying to AWS Lambda') logger.info(create_deployment_stdout) if create_deployment_stdout.startswith('Failed to create deployment'): deployment_failed = True @@ -82,9 +75,7 @@ def predict(self, df): ) if not deployment_failed: - logger.info( - 'E2E DEPLOYMENT TEST FOR LAMBDA:::: Test deployment with sample request' - ) + logger.info('Test deployment with sample request') try: request_result = requests.post( deployment_endpoint, @@ -99,9 +90,7 @@ def predict(self, df): logger.error(str(e)) deployment_failed = True - logger.info( - 'E2E DEPLOYMENT TEST FOR LAMBDA:::: Delete test deployment with BentoML CLI' - ) + logger.info('Delete test deployment with BentoML CLI') delete_deployment_command = [ 'bentoml', 'deploy', @@ -109,30 +98,14 @@ def predict(self, df): deployment_name, '--force', ] - logger.info( - 'E2E DEPLOYMENT TEST FOR LAMBDA:::: Delete command: {}'.format( - delete_deployment_command - ) - ) + logger.info('Delete command: {}'.format(delete_deployment_command)) with subprocess.Popen( delete_deployment_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) as proc: delete_deployment_stdout = proc.stdout.read().decode('utf-8') logger.info(delete_deployment_stdout) - logger.info('E2E DEPLOYMENT TEST FOR LAMBDA:::: Cleaning up bento service') - yatai_service = get_yatai_service() - delete_request = DangerouslyDeleteBentoRequest( - bento_name=loaded_service.name, bento_version=loaded_service.version - ) - yatai_service.DangerouslyDeleteBento(delete_request) - logger.info( - 'E2E DEPLOYMENT TEST FOR LAMBDA:::: Delete bento bundle on ' - 'file system {}'.format(saved_path) - ) - shutil.rmtree(saved_path) - - logger.info('E2E DEPLOYMENT TEST FOR LAMBDA:::: Finished') + logger.info('Finished') if deployment_failed: logger.info('E2E deployment failed, fix the issues before releasing') else: diff --git a/setup.py b/setup.py index 5d707f74cbf..08be18c5aef 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ xgboost = ["xgboost"] h2o = ["h2o"] api_server = ["gunicorn", "prometheus_client"] -aws_sam_cli = ["aws-sam-cli"] +aws_sam_cli = ["aws-sam-cli==0.33.1"] optional_requires = ( api_server + imageio + pytorch + tensorflow + fastai + xgboost + h2o + aws_sam_cli diff --git a/tests/deployment/aws_lambda/test_aws_lambda_deployment_operator.py b/tests/deployment/aws_lambda/test_aws_lambda_deployment_operator.py index a0649c9eb43..ba2f83e702d 100644 --- a/tests/deployment/aws_lambda/test_aws_lambda_deployment_operator.py +++ b/tests/deployment/aws_lambda/test_aws_lambda_deployment_operator.py @@ -13,7 +13,6 @@ from bentoml.proto.deployment_pb2 import ( Deployment, DeploymentState, - DescribeDeploymentResponse, ) from bentoml.proto.repository_pb2 import ( Bento, @@ -22,9 +21,8 @@ GetBentoResponse, ) from bentoml.proto import status_pb2 -from bentoml.yatai.status import Status -mock_s3_bucket_name = 'fake_deployment_bucket' +mock_s3_bucket_name = 'test_deployment_bucket' mock_s3_prefix = 'prefix' mock_s3_path = 's3://{}/{}'.format(mock_s3_bucket_name, mock_s3_prefix) @@ -32,7 +30,7 @@ def create_yatai_service_mock(repo_storage_type=BentoUri.LOCAL): bento_pb = Bento(name='bento_test_name', version='version1.1.1') if repo_storage_type == BentoUri.LOCAL: - bento_pb.uri.uri = '/fake/path/to/bundle' + bento_pb.uri.uri = '/tmp/path/to/bundle' bento_pb.uri.type = repo_storage_type api = BentoServiceMetadata.BentoServiceApi(name='predict') bento_pb.bento_service_metadata.apis.extend([api]) @@ -85,9 +83,9 @@ def mock_lambda_app(func): def mock_wrapper(*args, **kwargs): conn = boto3.client('s3', region_name='us-west-2') conn.create_bucket(Bucket=mock_s3_bucket_name) - fake_artifact_key = 'mock_artifact_prefix/model.pkl' + mock_artifact_key = 'mock_artifact_prefix/model.pkl' conn.put_object( - Bucket=mock_s3_bucket_name, Key=fake_artifact_key, Body='mock_body' + Bucket=mock_s3_bucket_name, Key=mock_artifact_key, Body='mock_body' ) return func(*args, **kwargs) @@ -139,7 +137,7 @@ def test_init_sam_project(tmpdir): def test_generate_aws_lambda_template_yaml(tmpdir): deployment_name = 'deployment_name' api_names = ['predict', 'classify'] - s3_bucket_name = 'fake_bucket' + s3_bucket_name = 'test_bucket' py_runtime = 'python3.7' memory_size = 3008 timeout = 6 @@ -188,14 +186,11 @@ def test_aws_lambda_apply_success(): yatai_service_mock = create_yatai_service_mock() test_deployment_pb = generate_lambda_deployment_pb() deployment_operator = AwsLambdaDeploymentOperator() - deployment_operator.describe = MagicMock() - deployment_operator.describe.return_value = DescribeDeploymentResponse( - state=DeploymentState(state=DeploymentState.RUNNING), status=Status.OK() - ) + result_pb = deployment_operator.apply(test_deployment_pb, yatai_service_mock, None) assert result_pb.status.status_code == status_pb2.Status.OK - assert result_pb.deployment.state.state == DeploymentState.RUNNING + assert result_pb.deployment.state.state == DeploymentState.PENDING def test_aws_lambda_describe_still_in_progress(): diff --git a/tox.ini b/tox.ini index d93448e78da..f0ac58b61af 100644 --- a/tox.ini +++ b/tox.ini @@ -2,14 +2,9 @@ envlist = py36,py37 [testenv] -deps = +commands = + pip install .[test] pytest - mock - docker - pip - imageio - fastai -commands = pytest [pytest] addopts = -p no:warnings