Merge branch 'vayu/cp-upload' into 'develop'

Vayu/cp upload

See merge request core/sevenbridges-python!62

.gitlab-ci.yml

@@ -5,31 +5,35 @@ variables:
   REQUIREMENTS2: "requirements2.txt"
 
 stages:
-- test
+  - test
+  - sonar
 
 .test: &test
   stage: test
   before_script:
-  - infinity config build-info
-  - python --version
-  - pip install -U -r $REQUIREMENTS
-  - export PYTHONPATH=$(pwd):$PYTHONPATH
+    - infinity config build-info
+    - python --version
+    - pip install -U -r $REQUIREMENTS
+    - export PYTHONPATH=$(pwd):$PYTHONPATH
   script:
-  - flake8 sevenbridges
-  - py.test -v --cov=sevenbridges
+    - flake8 sevenbridges
+    - py.test -v --cov-config=setup.cfg
+  artifacts:
+    paths:
+      - coverage_report/*
 
 
 # Test with python 2.7
 test:2.7:
   stage: test
   before_script:
-  - infinity config build-info
-  - python --version
-  - pip install -U -r $REQUIREMENTS2
-  - export PYTHONPATH=$(pwd):$PYTHONPATH
+    - infinity config build-info
+    - python --version
+    - pip install -U -r $REQUIREMENTS2
+    - export PYTHONPATH=$(pwd):$PYTHONPATH
   script:
-  - flake8 sevenbridges
-  - py.test -v --cov=sevenbridges
+    - flake8 sevenbridges
+    - py.test -v --cov=sevenbridges
   image: python:2.7
 
 # Test with python 3.5
@@ -46,3 +50,11 @@ test:3.6:
 test:3.7:
   <<: *test
   image: python:3.7-stretch
+
+sonar:
+  stage: sonar
+  image: emeraldsquad/sonar-scanner:1.0.0
+  script: infinity sonar scanner
+  allow_failure: true
+  dependencies:
+    - test:3.7

docs/quickstart.rst

@@ -1813,6 +1813,9 @@ Examples
     # Create automation package from uploaded code package
     automation_package = api.automation_packages.create('automation_id', 'version', 'location_id'):
 
+    # Add a code package to automation from local file
+    automation.add_package('version', 'file_path', schema={})
+
     # Get automation package details
     automation_package = api.automation_packages.get('package_id')
 

setup.cfg

@@ -7,3 +7,9 @@ description-file =
 [flake8]
 per-file-ignores =
     sevenbridges/__init__.py:E402
+
+[tool:pytest]
+addopts =
+    --cov=sevenbridges
+    --cov-report=xml:coverage_report/coverage.xml
+    --cov-report=term

sevenbridges/models/automation.py

@@ -18,6 +18,7 @@
 from sevenbridges.models.enums import AutomationRunActions
 from sevenbridges.models.file import File
 from sevenbridges.models.member import Permissions
+from sevenbridges.transfer.upload import CodePackageUpload
 
 logger = logging.getLogger(__name__)
 
@@ -39,6 +40,7 @@ class AutomationPackage(Resource):
     automation = UuidField(read_only=True)
     version = StringField(read_only=True)
     location = StringField(read_only=True)
+    schema = DictField(read_only=True)
     created_by = StringField(read_only=True)
     created_on = DateTimeField(read_only=True)
     archived = BooleanField(read_only=True)
@@ -78,12 +80,13 @@ def query(cls, automation, offset=None, limit=None, api=None):
         )
 
     @classmethod
-    def create(cls, automation, version, location, api=None):
+    def create(cls, automation, version, location, schema, api=None):
         """
         Create a code package.
         :param automation: Automation id.
         :param version: File ID of the uploaded code package.
         :param location: The code package version.
+        :param schema: IO schema for main step of execution.
         :param api: Api instance.
         :return:
         """
@@ -97,9 +100,13 @@ def create(cls, automation, version, location, api=None):
         if location is None:
             raise SbgError('Code package location is required!')
 
+        if schema is None:
+            raise SbgError('Schema is required!')
+
         data = {
             'version': version,
-            'location': location
+            'location': location,
+            'schema': schema
         }
 
         extra = {
@@ -516,17 +523,48 @@ def get_package(cls, package, api=None):
             id=package_id, api=api
         )
 
-    def add_package(self, version, location, api=None):
+    def add_package(self, version, file_path, schema, file_name=None,
+                    retry_count=5, timeout=60, part_size=None, api=None):
         """
         Add a code package to automation template.
-        :param version: File ID of the uploaded code package.
-        :param location: The code package version.
+        :param version: The code package version.
+        :param file_path: Path to the code package file to be uploaded.
+        :param schema: IO schema for main step of execution.
+        :param part_size: Size of upload part in bytes.
+        :param file_name: Optional file name.
+        :param retry_count: Upload retry count.
+        :param timeout: Timeout for s3/google session.
         :param api: sevenbridges Api instance.
         :return: AutomationPackage
         """
         api = api or self._API
+        if version is None:
+            raise SbgError('Code package version is required!')
+
+        if file_path is None:
+            raise SbgError('Code package file path is required!')
+
+        # Multipart upload the code package:
+        upload = CodePackageUpload(
+            file_path,
+            self.id,
+            api=api,
+            part_size=part_size,
+            file_name=file_name,
+            retry_count=retry_count,
+            timeout=timeout
+        )
+        upload.start()
+        upload.wait()
+        package_file = upload.result()
+
+        # Create the automation package:
         return AutomationPackage.create(
-            self.id, version=version, location=location, api=api
+            self.id,
+            version=version,
+            location=package_file.id,
+            schema=schema,
+            api=api
         )
 
     def get_member(self, username, api=None):
@@ -623,6 +661,7 @@ class AutomationRun(Resource):
     automation = CompoundField(Automation, read_only=True)
     package = CompoundField(AutomationPackage, read_only=True)
     inputs = DictField()
+    outputs = DictField(read_only=True)
     settings = DictField()
     created_on = DateTimeField(read_only=True)
     start_time = DateTimeField(read_only=True)
@@ -709,6 +748,8 @@ def create(cls, package, inputs=None, settings=None, resume_from=None,
         data = {'package': package}
         if inputs:
             data['inputs'] = inputs
+        else:
+            data['inputs'] = dict()
         if settings:
             data['settings'] = settings
         if resume_from:

sevenbridges/tests/conftest.py

@@ -57,6 +57,10 @@ def __init__(self, request_mocker, base_url):
             request_mocker, base_url
         )
         self.async_jobs = providers.AsyncJobProvider(request_mocker, base_url)
+        self.cp_uploads = providers.CodePackageUploadProvider(
+            request_mocker, base_url
+        )
+        self.uploads = providers.FileUploadProvider(request_mocker, base_url)
 
 
 class Verifier(object):

sevenbridges/tests/providers.py

@@ -1,3 +1,4 @@
+import re
 import time
 
 import faker
@@ -1490,9 +1491,11 @@ def has_package(self, package_id):
         )
         self.request_mocker.get(href, json=package)
 
-    def can_add_package(self, automation_id, package_id, location, version):
+    def can_add_package(self, automation_id, package_id,
+                        location, version, schema):
         package = self.package_provider.default_automation_package(
-            package_id=package_id, location=location, version=version
+            package_id=package_id, location=location, version=version,
+            schema=schema
         )
         href = self.base_url + '/automation/automations/{}/packages'.format(
             automation_id
@@ -1538,17 +1541,19 @@ def __init__(self, request_mocker, base_url):
 
     @staticmethod
     def default_automation_package(
-            package_id=None, version=None, location=None
+            package_id=None, version=None, location=None, schema=None
     ):
         package_id = package_id or generator.uuid4()
         version = version or generator.slug()
         location = location or generator.slug()
+        schema = schema or {}
 
         return {
             'id': package_id,
             'automation': generator.uuid4(),
             'version': version,
             'location': location,
+            'schema': schema,
             'created_by': generator.user_name(),
             'created_on': generator.date(),
             'archived': False,
@@ -1665,6 +1670,7 @@ def default_automation_run(self, automation=None):
             'automation': automation,
             'package': generator.uuid4(),
             'inputs': {},
+            'outputs': None,
             'settings': {},
             'created_on': generator.date(),
             'start_time': generator.date(),
@@ -1810,3 +1816,127 @@ def can_move_files(self, files):
         self.request_mocker.post(
             '/async/files/move', json=async_job
         )
+
+
+class CodePackageUploadProvider(object):
+    def __init__(self, request_mocker, base_url):
+        self.request_mocker = request_mocker
+        self.base_url = base_url
+        self.file_provider = FileProvider(request_mocker, base_url)
+
+    def initialized_upload(self, upload_id, part_size):
+        response = {
+            "upload_id": upload_id,
+            "part_size": part_size
+        }
+        self.request_mocker.post(
+            '/automation/upload', json=response
+        )
+
+    def got_file_part(self, url):
+        regx = '^{}/automation/upload/.*/part/.*$'.format(self.base_url)
+        matcher = re.compile(regx)
+        self.request_mocker.get(
+            matcher, json={"url": url}
+        )
+
+    def got_etag(self, url):
+        self.request_mocker.put(
+            url,
+            json={},
+            headers={'etag': generator.uuid4()}
+        )
+
+    def reported_part(self):
+        regx = '^{}/automation/upload/.*/part/'.format(
+            self.base_url
+        )
+        matcher = re.compile(regx)
+        self.request_mocker.post(
+            matcher, json={}
+        )
+
+    def finalized_upload(self, package_file_id):
+        regx = '^{}/automation/upload/.*/complete'.format(
+            self.base_url
+        )
+        matcher = re.compile(regx)
+        self.request_mocker.post(
+            matcher, json={"name": "dummy_file_name", "id": package_file_id}
+        )
+
+    def deleted(self):
+        regx = '^{}/automation/upload/.*'.format(
+            self.base_url
+        )
+        matcher = re.compile(regx)
+        self.request_mocker.delete(matcher)
+
+
+class FileUploadProvider(object):
+    def __init__(self, request_mocker, base_url):
+        self.request_mocker = request_mocker
+        self.base_url = base_url
+        self.file_provider = FileProvider(request_mocker, base_url)
+
+    def initialized_upload(self, upload_id, part_size, failed=False):
+        status_code = 500 if failed else 200
+        response = {
+            "upload_id": upload_id,
+            "part_size": part_size
+        }
+        self.request_mocker.post(
+            '/upload/multipart', json=response, status_code=status_code
+        )
+
+    def got_file_part(self, url, failed=False):
+        status_code = 500 if failed else 200
+        regx = '^{}/upload/multipart/.*/part/.*$'.format(self.base_url)
+        matcher = re.compile(regx)
+        self.request_mocker.get(
+            matcher, json={"url": url},
+            status_code=status_code
+        )
+
+    def got_etag(self, url, failed=False):
+        status_code = 500 if failed else 200
+        headers = {} if failed else {'etag': generator.uuid4()}
+
+        self.request_mocker.put(
+            url,
+            json={},
+            headers=headers,
+            status_code=status_code
+        )
+
+    def reported_part(self, failed=False):
+        status_code = 500 if failed else 200
+        regx = '^{}/upload/multipart/.*/part/'.format(
+            self.base_url
+        )
+        matcher = re.compile(regx)
+        self.request_mocker.post(
+            matcher, json={}, status_code=status_code
+        )
+
+    def finalized_upload(self, package_file_id, failed=False):
+        status_code = 500 if failed else 200
+
+        regx = '^{}/upload/multipart/.*/complete'.format(
+            self.base_url
+        )
+        matcher = re.compile(regx)
+        self.request_mocker.post(
+            matcher,
+            json={"name": "dummy_file_name", "id": package_file_id},
+            status_code=status_code
+        )
+
+    def deleted(self, failed=False):
+        status_code = 500 if failed else 200
+
+        regx = '^{}/upload/multipart/.*'.format(
+            self.base_url
+        )
+        matcher = re.compile(regx)
+        self.request_mocker.delete(matcher, status_code=status_code)