Feature remote import (#158)

* improvement: 优化从配置中创建包源和与 API 创建的包源包名冲突时的提示

* feature: 添加 S3 及 FileSystem importer

* minor: 添加 S3Importer 及 FSImporter 的单元测试

* feature: s3 与 fs 类型的 ExternalSource 返回对应的 importer

* feature: 允许用户通过配置控制每个源的安全限制

* bugfix: 修复获取组件实例时没有处理组件可能不存在的问题

* feature: 添加 form_is_embbeded 方法来判断组件是否包含嵌入式的表单

* feature: component 相关接口添加 form_is_embedded 字段

* minor: sync pipeline code

gcloud/webservice3/resources.py

@@ -354,6 +354,7 @@ def alter_list_data_to_serialize(self, request, data):
             bundle.data['output'] = component.outputs_format()
             bundle.data['form'] = component.form
             bundle.data['desc'] = component.desc
+            bundle.data['form_is_embedded'] = component.form_is_embedded()
             # 国际化
             name = bundle.data['name'].split('-')
             bundle.data['group_name'] = _(name[0])
@@ -367,6 +368,7 @@ def alter_detail_data_to_serialize(self, request, data):
         bundle.data['output'] = component.outputs_format()
         bundle.data['form'] = component.form
         bundle.data['desc'] = component.desc
+        bundle.data['form_is_embedded'] = component.form_is_embedded()
         # 国际化
         name = bundle.data['name'].split('-')
         bundle.data['group_name'] = _(name[0])

pipeline/component_framework/component.py

@@ -29,6 +29,10 @@ def outputs_format(cls):
         outputs = map(lambda oi: oi._asdict(), outputs)
         return outputs
 
+    @classmethod
+    def form_is_embedded(cls):
+        return getattr(cls, 'embedded_form', False)
+
     def clean_execute_data(self, context):
         return self.data_dict
 

pipeline/component_framework/library.py

@@ -25,8 +25,7 @@ def __new__(cls, *args, **kwargs):
             raise ValueError('please pass a component_code in args or kwargs: '
                              'ComponentLibrary(\'code\') or ComponentLibrary(component_code=\'code\')')
         if component_code not in cls.components:
-            raise ComponentNotExistException('component %s does not exist.' %
-                                             component_code)
+            raise ComponentNotExistException('component %s does not exist.' % component_code)
         return cls.components[component_code]
 
     @classmethod
@@ -35,4 +34,7 @@ def get_component_class(cls, component_code):
 
     @classmethod
     def get_component(cls, component_code, data_dict):
-        return cls.get_component_class(component_code)(data_dict)
+        component_cls = cls.get_component_class(component_code)
+        if component_cls is None:
+            raise ComponentNotExistException('component %s does not exist.' % component_code)
+        return component_cls(data_dict)

pipeline/component_framework/test.py

@@ -0,0 +1,204 @@
+# -*- coding: utf-8 -*-
+"""
+Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community
+Edition) available.
+Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+
+import logging
+
+from abc import abstractproperty
+
+from pipeline.core.data.base import DataObject
+
+logger = logging.getLogger(__name__)
+
+
+class ComponentTestMixin(object):
+
+    @abstractproperty
+    def component_cls(self):
+        raise NotImplementedError()
+
+    @abstractproperty
+    def cases(self):
+        raise NotImplementedError()
+
+    @property
+    def patchers(self):
+        return []
+
+    @property
+    def _component_cls_name(self):
+        return self.component_cls.__name__
+
+    def _format_failure_message(self, no, name, msg):
+        return '{component_cls} case {no}:{name} fail: {msg}'.format(
+            component_cls=self._component_cls_name,
+            no=no + 1,
+            name=name,
+            msg=msg
+        )
+
+    def _do_case_assert(self, service, method, assertion, no, name, args=None, kwargs=None):
+
+        do_continue = False
+        args = args or [service]
+        kwargs = kwargs or {}
+
+        data = kwargs.get('data') or args[0]
+
+        if assertion.exc:
+            # raise assertion
+
+            try:
+                getattr(service, method)(*args, **kwargs)
+            except Exception as e:
+                assert e.__class__ == assertion.exc, self._format_failure_message(
+                    no=no,
+                    name=name,
+                    msg='{method} raise assertion failed,\nexcept: {e}\nactual: {a}'.format(
+                        method=method,
+                        e=assertion.exc,
+                        a=e.__class__
+                    ))
+                do_continue = True
+            else:
+                self.assertTrue(False, msg=self._format_failure_message(
+                    no=no,
+                    name=name,
+                    msg='{method} raise assertion failed, {method} not raise any exception'.format(
+                        method=method
+                    )
+                ))
+        else:
+
+            result = getattr(service, method)(*args, **kwargs)
+
+            if result is None or result is True:
+                self.assertTrue(assertion.success, msg=self._format_failure_message(
+                    no=no,
+                    name=name,
+                    msg='{method} success assertion failed, {method} execute success'.format(
+                        method=method
+                    )
+                ))
+
+                self.assertDictEqual(data.outputs, assertion.outputs, msg=self._format_failure_message(
+                    no=no,
+                    name=name,
+                    msg='{method} outputs assertion failed,\nexcept: {e}\nactual: {a}'.format(
+                        method=method,
+                        e=data.outputs,
+                        a=assertion.outputs
+                    )
+                ))
+
+            else:
+                self.assertFalse(assertion.success, msg=self._format_failure_message(
+                    no=no,
+                    name=name,
+                    msg='{method} success assertion failed, {method} execute failed'.format(
+                        method=method
+                    )
+                ))
+
+                do_continue = True
+
+        return do_continue
+
+    def test_component(self):
+        patchers = self.patchers
+        for patcher in patchers:
+            patcher.start()
+
+        component = self.component_cls({})
+
+        bound_service = component.service()
+
+        for no, case in enumerate(self.cases):
+            data = DataObject(inputs=case.inputs)
+            parent_data = DataObject(inputs=case.parent_data)
+
+            # execute result check
+            do_continue = self._do_case_assert(service=bound_service,
+                                               method='execute',
+                                               args=(data, parent_data),
+                                               assertion=case.execute_assertion,
+                                               no=no,
+                                               name=case.name)
+
+            if do_continue:
+                continue
+
+            if bound_service.need_schedule():
+
+                if bound_service.interval is None:
+                    # callback case
+                    self._do_case_assert(service=bound_service,
+                                         method='schedule',
+                                         args=(data, parent_data, case.schedule_assertion.callback_data),
+                                         assertion=case.schedule_assertion,
+                                         no=no,
+                                         name=case.name)
+
+                else:
+                    # schedule case
+                    assertions = case.schedule_assertion
+                    assertions = assertions if isinstance(assertions, list) else [assertions]
+
+                    for assertion in assertions:
+                        do_continue = self._do_case_assert(service=bound_service,
+                                                           method='schedule',
+                                                           args=(data, parent_data),
+                                                           assertion=assertion,
+                                                           no=no,
+                                                           name=case.name)
+
+                        if do_continue:
+                            break
+
+        logger.info('{component} paas {num} cases.'.format(
+            component=self._component_cls_name,
+            num=len(self.cases)
+        ))
+
+        for patcher in patchers:
+            patcher.stop()
+
+
+class ComponentTestCase(object):
+    def __init__(self,
+                 inputs,
+                 parent_data,
+                 execute_assertion,
+                 schedule_assertion,
+                 name=''):
+        self.inputs = inputs
+        self.parent_data = parent_data
+        self.execute_assertion = execute_assertion
+        self.schedule_assertion = schedule_assertion
+        self.name = name
+
+
+class Assertion(object):
+    def __init__(self, success, outputs, exc=None):
+        self.success = success
+        self.outputs = outputs
+        self.exc = exc
+
+
+class ExecuteAssertion(Assertion):
+    pass
+
+
+class ScheduleAssertion(Assertion):
+    def __init__(self, callback_data, *args, **kwargs):
+        self.callback_data = callback_data
+        super(ScheduleAssertion, self).__init__(*args, **kwargs)

pipeline/contrib/external_plugins/models/base.py

@@ -14,7 +14,7 @@
 from copy import deepcopy
 from abc import abstractmethod
 
-from django.db import models
+from django.db import models, IntegrityError
 from django.utils.translation import ugettext_lazy as _
 
 from pipeline.contrib.external_plugins import exceptions
@@ -69,10 +69,15 @@ def update_source_from_config(self, configs):
             defaults = deepcopy(config['details'])
             defaults['packages'] = config['packages']
 
-            self.update_or_create(
-                name=config['name'],
-                from_config=True,
-                defaults=defaults)
+            try:
+                self.update_or_create(
+                    name=config['name'],
+                    from_config=True,
+                    defaults=defaults)
+            except IntegrityError:
+                raise exceptions.InvalidOperationException(
+                    'There is a external source named "{source_name}" but not create from config, '
+                    'can not do source update operation'.format(source_name=config['name']))
 
 
 class ExternalPackageSource(models.Model):
@@ -107,7 +112,7 @@ def modules(self):
     def update_package_source_from_config(source_configs):
         classified_config = {source_type: [] for source_type in source_cls_factory.keys()}
 
-        for config in source_configs:
+        for config in deepcopy(source_configs):
             classified_config.setdefault(config.pop('type'), []).append(config)
 
         for source_type, configs in classified_config.items():

pipeline/contrib/external_plugins/models/source.py

@@ -14,7 +14,11 @@
 from django.db import models
 from django.conf import settings
 from django.utils.translation import ugettext_lazy as _
-from pipeline.contrib.external_plugins.utils.importer.git import GitRepoModuleImporter
+from pipeline.contrib.external_plugins.utils.importer import (
+    GitRepoModuleImporter,
+    S3ModuleImporter,
+    FSModuleImporter
+)
 
 from pipeline.contrib.external_plugins.models.base import (
     GIT,
@@ -37,7 +41,9 @@ def importer(self):
         return GitRepoModuleImporter(repo_raw_url=self.repo_raw_address,
                                      branch=self.branch,
                                      modules=self.packages.keys(),
-                                     proxy=getattr(settings, 'EXTERNAL_SOURCE_PROXY'))
+                                     proxy=getattr(settings, 'EXTERNAL_SOURCE_PROXY'),
+                                     secure_only=getattr(settings,
+                                                         'EXTERNAL_SOURCE_SECURE_RESTRICT', {}).get(self.name, True))
 
 
 @package_source
@@ -52,7 +58,13 @@ def type():
         return S3
 
     def importer(self):
-        pass
+        return S3ModuleImporter(modules=self.packages.keys(),
+                                service_address=self.service_address,
+                                bucket=self.bucket,
+                                access_key=self.access_key,
+                                secret_key=self.secret_key,
+                                secure_only=getattr(settings,
+                                                    'EXTERNAL_SOURCE_SECURE_RESTRICT', {}).get(self.name, True))
 
 
 @package_source
@@ -64,4 +76,5 @@ def type():
         return FILE_SYSTEM
 
     def importer(self):
-        pass
+        return FSModuleImporter(modules=self.packages.keys(),
+                                path=self.path)

pipeline/contrib/external_plugins/tests/mock.py

@@ -16,6 +16,12 @@
 from mock import MagicMock, patch, call  # noqa
 
 
+def mock_s3_resource(resource, **kwargs):
+    ret = {'resource': resource}
+    ret.update(kwargs)
+    return ret
+
+
 class Object(object):
     pass
 

pipeline/contrib/external_plugins/tests/mock_settings.py

@@ -16,6 +16,8 @@
 IMP_ACQUIRE_LOCK = 'imp.acquire_lock'
 IMP_RELEASE_LOCK = 'imp.release_lock'
 REQUESTS_GET = 'requests.get'
+BOTO3_RESOURCE = 'boto3.resource'
+OS_PATH_EXISTS = 'os.path.exists'
 
 IMPORTLIB_IMPORT_MODULE = 'importlib.import_module'
 
@@ -36,3 +38,19 @@
 UTILS_IMPORTER_GIT_IS_PACKAGE = 'pipeline.contrib.external_plugins.utils.importer.git.GitRepoModuleImporter.is_package'
 UTILS_IMPORTER__SETUP_IMPORTER = 'pipeline.contrib.external_plugins.utils.importer.utils._setup_importer'
 UTILS_IMPORTER__REMOVE_IMPORTER = 'pipeline.contrib.external_plugins.utils.importer.utils._remove_importer'
+
+UTILS_IMPORTER_S3__FETCH_OBJ_CONTENT = \
+    'pipeline.contrib.external_plugins.utils.importer.s3.S3ModuleImporter._fetch_obj_content'
+UTILS_IMPORTER_S3_GET_SOURCE = 'pipeline.contrib.external_plugins.utils.importer.s3.S3ModuleImporter.get_source'
+UTILS_IMPORTER_S3_GET_FILE = 'pipeline.contrib.external_plugins.utils.importer.s3.S3ModuleImporter.get_file'
+UTILS_IMPORTER_S3_IS_PACKAGE = 'pipeline.contrib.external_plugins.utils.importer.s3.S3ModuleImporter.is_package'
+UTILS_IMPORTER_S3__GET_S3_OBJ_CONTENT = \
+    'pipeline.contrib.external_plugins.utils.importer.s3.S3ModuleImporter._get_s3_obj_content'
+
+UTILS_IMPORTER_FS_GET_SOURCE = 'pipeline.contrib.external_plugins.utils.importer.fs.FSModuleImporter.get_source'
+UTILS_IMPORTER_FS_GET_FILE = 'pipeline.contrib.external_plugins.utils.importer.fs.FSModuleImporter.get_file'
+UTILS_IMPORTER_FS_IS_PACKAGE = 'pipeline.contrib.external_plugins.utils.importer.fs.FSModuleImporter.is_package'
+UTILS_IMPORTER_FS__FETCH_FILE_CONTENT = \
+    'pipeline.contrib.external_plugins.utils.importer.fs.FSModuleImporter._fetch_file_content'
+UTILS_IMPORTER_FS__GET_FILE_CONTENT = \
+    'pipeline.contrib.external_plugins.utils.importer.fs.FSModuleImporter._get_file_content'

pipeline/contrib/external_plugins/tests/models/base/test_external_package_source.py

@@ -11,6 +11,8 @@
 specific language governing permissions and limitations under the License.
 """
 
+from copy import deepcopy
+
 from django.test import TestCase
 
 from pipeline.contrib.external_plugins import exceptions
@@ -220,3 +222,12 @@ def test_update_package_source_from_config__unsupported_source_type(self):
             }
         ]
         self.assertRaises(KeyError, ExternalPackageSource.update_package_source_from_config, source_configs)
+
+    def test_update_source_from_config__name_conflict(self):
+        source = deepcopy(SOURCE_1)
+        source['type'] = 'git'
+        ExternalPackageSource.update_package_source_from_config([source])
+        GitRepoSource.objects.filter(name=source['name']).update(from_config=False)
+        self.assertRaises(exceptions.InvalidOperationException,
+                          ExternalPackageSource.update_package_source_from_config,
+                          [source])