From a8e1eb13221c8b49a51c2f585e9da57cf78e3d44 Mon Sep 17 00:00:00 2001 From: homholueng Date: Tue, 9 Apr 2019 11:41:23 +0800 Subject: [PATCH 01/29] =?UTF-8?q?feature:=20=E6=B7=BB=E5=8A=A0=E9=9D=9E?= =?UTF-8?q?=E6=A0=87=E5=87=86=E5=AD=98=E5=82=A8=E4=BB=A3=E7=A0=81=E5=AF=BC?= =?UTF-8?q?=E5=85=A5=E5=99=A8=E5=8F=8A=20git=20=E4=BB=93=E5=BA=93=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E5=AF=BC=E5=85=A5=E5=99=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pipeline/tests/mock.py | 7 +- pipeline/tests/mock_settings.py | 15 +- pipeline/tests/utils/importer/__init__.py | 8 + pipeline/tests/utils/importer/base.py | 207 ++++++++++++++++++++++ pipeline/tests/utils/importer/git.py | 137 ++++++++++++++ pipeline/utils/importer/__init__.py | 8 + pipeline/utils/importer/base.py | 170 ++++++++++++++++++ pipeline/utils/importer/git.py | 83 +++++++++ 8 files changed, 633 insertions(+), 2 deletions(-) create mode 100644 pipeline/tests/utils/importer/__init__.py create mode 100644 pipeline/tests/utils/importer/base.py create mode 100644 pipeline/tests/utils/importer/git.py create mode 100644 pipeline/utils/importer/__init__.py create mode 100644 pipeline/utils/importer/base.py create mode 100644 pipeline/utils/importer/git.py diff --git a/pipeline/tests/mock.py b/pipeline/tests/mock.py index 3275dab701..d92d08a574 100644 --- a/pipeline/tests/mock.py +++ b/pipeline/tests/mock.py @@ -5,7 +5,7 @@ Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://opensource.org/licenses/MIT Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -""" # noqa +""" # noqa from __future__ import absolute_import import mock @@ -27,6 +27,11 @@ class Object(object): pass +class ReadableObject(object): + def __init__(self, **kwargs): + self.read = MagicMock(return_value=kwargs.get('read_return')) + + class ContextObject(object): def __init__(self, variables): self.variables = variables diff --git a/pipeline/tests/mock_settings.py b/pipeline/tests/mock_settings.py index 3c58247665..2f293dbb82 100644 --- a/pipeline/tests/mock_settings.py +++ b/pipeline/tests/mock_settings.py @@ -5,7 +5,13 @@ Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://opensource.org/licenses/MIT Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -""" # noqa +""" # noqa + +SYS_MODULES = 'sys.modules' +IMP_ACQUIRE_LOCK = 'imp.acquire_lock' +IMP_RELEASE_LOCK = 'imp.release_lock' +URLLIB2_URLOPEN = 'urllib2.urlopen' + PIPELINE_CORE_GATEWAY_DEFORMAT = 'pipeline.core.flow.gateway.deformat_constant_key' PIPELINE_CORE_CONSTANT_RESOLVE = 'pipeline.core.data.expression.ConstantTemplate.resolve_data' @@ -110,3 +116,10 @@ EXG_HYDRATE_DATA = 'pipeline.engine.core.handlers.exclusive_gateway.hydrate_data' CPG_HYDRATE_DATA = 'pipeline.engine.core.handlers.conditional_parallel.hydrate_data' + +UTILS_IMPORTER_BASE_EXECUTE_SRC_CODE = 'pipeline.utils.importer.base.NonstandardModuleImporter._execute_src_code' +UTILS_IMPORTER_GIT__FETCH_REPO_FILE = 'pipeline.utils.importer.git.GitRepoModuleImporter._fetch_repo_file' +UTILS_IMPORTER_GIT__FILE_URL = 'pipeline.utils.importer.git.GitRepoModuleImporter._file_url' +UTILS_IMPORTER_GIT_GET_SOURCE = 'pipeline.utils.importer.git.GitRepoModuleImporter.get_source' +UTILS_IMPORTER_GIT_GET_FILE = 'pipeline.utils.importer.git.GitRepoModuleImporter.get_file' +UTILS_IMPORTER_GIT_IS_PACKAGE = 'pipeline.utils.importer.git.GitRepoModuleImporter.is_package' diff --git a/pipeline/tests/utils/importer/__init__.py b/pipeline/tests/utils/importer/__init__.py new file mode 100644 index 0000000000..6a09231373 --- /dev/null +++ b/pipeline/tests/utils/importer/__init__.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. +""" # noqa diff --git a/pipeline/tests/utils/importer/base.py b/pipeline/tests/utils/importer/base.py new file mode 100644 index 0000000000..c40799b155 --- /dev/null +++ b/pipeline/tests/utils/importer/base.py @@ -0,0 +1,207 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. +""" # noqa + +import imp +import sys +from django.test import TestCase + +from pipeline.tests.mock import * # noqa +from pipeline.tests.mock_settings import * # noqa +from pipeline.utils.importer.base import NonstandardModuleImporter + + +class DummyImporter(NonstandardModuleImporter): + + def __init__(self, **kwargs): + super(DummyImporter, self).__init__(modules=kwargs.get('modules', [])) + self._is_package = kwargs.get('is_package') + self._get_code = kwargs.get('get_code') + self._get_source = kwargs.get('get_source') + self._get_file = kwargs.get('get_file') + self._get_path = kwargs.get('get_path') + + self._accept_find_module_request_hook = MagicMock() + self._pre_load_module_hook = MagicMock() + self._post_load_module_hook = MagicMock() + self._import_error_hook = MagicMock() + + def is_package(self, fullname): + return self._is_package + + def get_code(self, fullname): + return self._get_code + + def get_source(self, fullname): + return self._get_source + + def get_file(self, fullname): + return self._get_file + + def get_path(self, fullname): + return self._get_path + + def accept_find_module_request_hook(self, fullname, path): + self._accept_find_module_request_hook(fullname=fullname, path=path) + + def pre_load_module_hook(self, fullname, module): + self._pre_load_module_hook(fullname=fullname, module=module) + + def post_load_module_hook(self, fullname, module): + self._post_load_module_hook(fullname=fullname, module=module) + + def import_error_hook(self, fullname): + self._import_error_hook(fullname=fullname) + + +class NonstandardModuleImporterTestCase(TestCase): + + def setUp(self): + self.imp_acquire_lock_patcher = patch(IMP_ACQUIRE_LOCK, MagicMock()) + self.imp_release_lock_patcher = patch(IMP_RELEASE_LOCK, MagicMock()) + self.importer_exec_src_code_patcher = patch(UTILS_IMPORTER_BASE_EXECUTE_SRC_CODE, MagicMock()) + + self.imp_acquire_lock_patcher.start() + self.imp_release_lock_patcher.start() + self.importer_exec_src_code_patcher.start() + + def tearDown(self): + self.imp_acquire_lock_patcher.stop() + self.imp_release_lock_patcher.stop() + self.importer_exec_src_code_patcher.stop() + + def test_find_module__module_not_in_self_modules(self): + importer = DummyImporter() + + self.assertIsNone(importer.find_module('django')) + importer._accept_find_module_request_hook.assert_not_called() + + self.assertIsNone(importer.find_module('django.test')) + importer._accept_find_module_request_hook.assert_not_called() + + self.assertIsNone(importer.find_module('django.test.utils')) + importer._accept_find_module_request_hook.assert_not_called() + + def test_find_module__module_in_built_in(self): + importer = DummyImporter() + + self.assertIsNone(importer.find_module('math')) + importer._accept_find_module_request_hook.assert_not_called() + + def test_find_module__module_has_name_repetition(self): + importer = DummyImporter(modules=['magic_module']) + + self.assertIsNone(importer.find_module('magic_module.magic_sub_module.magic_module')) + importer._accept_find_module_request_hook.assert_not_called() + + def test_find_module__accept(self): + importer = DummyImporter(modules=['magic_module']) + + fullname = 'magic_module' + self.assertIs(importer, importer.find_module(fullname)) + importer._accept_find_module_request_hook.assert_called_once_with(fullname=fullname, path=None) + importer._accept_find_module_request_hook.reset_mock() + + fullname = 'magic_module.magic_sub_module_1' + self.assertIs(importer, importer.find_module(fullname)) + importer._accept_find_module_request_hook.assert_called_once_with(fullname=fullname, path=None) + importer._accept_find_module_request_hook.reset_mock() + + fullname = 'magic_module.magic_sub_module_1.magic_sub_module_2' + self.assertIs(importer, importer.find_module(fullname)) + importer._accept_find_module_request_hook.assert_called_once_with(fullname=fullname, path=None) + importer._accept_find_module_request_hook.reset_mock() + + def test_load_module__module_already_in_sys_modules(self): + fullname = 'exist_module' + mod = Object() + importer = DummyImporter() + + with patch(SYS_MODULES, {fullname: mod}): + self.assertEqual(importer.load_module(fullname=fullname), mod) + imp.acquire_lock.assert_called_once() + imp.release_lock.assert_called_once() + + def test_load_module__get_source_raise_import_error(self): + sub_module = 'sub_module' + fullname = 'exist_module.sub_module' + mod = Object() + importer = DummyImporter() + importer.get_source = MagicMock(side_effect=ImportError) + + with patch(SYS_MODULES, {sub_module: mod}): + self.assertIsNone(importer.load_module(fullname=fullname)) + imp.acquire_lock.assert_called_once() + imp.release_lock.assert_called_once() + + def test_load_module__is_package(self): + src_code = 'src_code' + fullname = 'magic_module' + file = 'file' + path = 'path' + importer = DummyImporter(is_package=True, get_source=src_code, get_file=file, get_path=path) + + with patch(SYS_MODULES, {}): + mod = importer.load_module(fullname=fullname) + + self.assertIs(sys.modules[fullname], mod) + self.assertEqual(mod.__file__, file) + self.assertIs(mod.__loader__, importer) + self.assertEqual(mod.__path__, path) + self.assertEqual(mod.__package__, fullname) + + imp.acquire_lock.assert_called_once() + importer._pre_load_module_hook.assert_called_once_with(fullname=fullname, module=mod) + importer._execute_src_code.assert_called_once_with(src_code=src_code, module=mod) + importer._post_load_module_hook.assert_called_once_with(fullname=fullname, module=mod) + imp.release_lock.assert_called_once() + + def test_load_module__is_not_package(self): + src_code = 'src_code' + fullname = 'magic_module.sub_module' + file = 'file' + importer = DummyImporter(is_package=False, get_source=src_code, get_file=file) + + with patch(SYS_MODULES, {}): + mod = importer.load_module(fullname=fullname) + + self.assertIs(sys.modules[fullname], mod) + self.assertEqual(mod.__file__, file) + self.assertIs(mod.__loader__, importer) + self.assertEqual(mod.__package__, fullname.rpartition('.')[0]) + + imp.acquire_lock.assert_called_once() + importer._pre_load_module_hook.assert_called_once_with(fullname=fullname, module=mod) + importer._execute_src_code.assert_called_once_with(src_code=src_code, module=mod) + importer._post_load_module_hook.assert_called_once_with(fullname=fullname, module=mod) + imp.release_lock.assert_called_once() + + def test_load_module__raise_exception_before_add_module(self): + fullname = 'magic_module.sub_module' + importer = DummyImporter(is_package=False) + importer.get_source = MagicMock(side_effect=Exception()) + importer._import_error_hook = MagicMock(side_effect=Exception()) + + with patch(SYS_MODULES, {}): + self.assertRaises(ImportError, importer.load_module, fullname) + self.assertNotIn(fullname, sys.modules) + + importer._import_error_hook.assert_called_once() + imp.release_lock.assert_called_once() + + def test_load_module__raise_exception_after_add_module(self): + fullname = 'magic_module.sub_module' + importer = DummyImporter(is_package=False) + importer.get_file = MagicMock(side_effect=Exception()) + + with patch(SYS_MODULES, {}): + self.assertRaises(ImportError, importer.load_module, fullname) + self.assertNotIn(fullname, sys.modules) + + importer._import_error_hook.assert_called_once() + imp.release_lock.assert_called_once() diff --git a/pipeline/tests/utils/importer/git.py b/pipeline/tests/utils/importer/git.py new file mode 100644 index 0000000000..0d0d91b81f --- /dev/null +++ b/pipeline/tests/utils/importer/git.py @@ -0,0 +1,137 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. +""" # noqa + +import urllib2 + +from django.test import TestCase + +from pipeline.tests.mock import * # noqa +from pipeline.tests.mock_settings import * # noqa +from pipeline.utils.importer.git import GitRepoModuleImporter + +GET_FILE_RETURN = 'GET_FILE_RETURN' +GET_SOURCE_RETURN = 'a=1' +IS_PACKAGE_RETURN = False +_FILE_URL_RETURN = '_FILE_URL_RETURN' +_FETCH_REPO_FILE_RETURN = '_FETCH_REPO_FILE_RETURN' + + +class GitRepoModuleImporterTestCase(TestCase): + def setUp(self): + self.repo_raw_url = 'https://test-git-repo-raw/' + self.repo_raw_url_without_slash = 'https://test-git-repo-raw' + self.branch = 'master' + self.fullname = 'module1.module2.module3' + self.module_url = 'https://test-git-repo-raw/master/module1/module2/module3.py' + self.package_url = 'https://test-git-repo-raw/master/module1/module2/module3/__init__.py' + + def test__init__(self): + importer = GitRepoModuleImporter(modules=[], repo_raw_url=self.repo_raw_url, branch=self.branch) + self.assertEqual(importer.repo_raw_url, self.repo_raw_url) + + importer = GitRepoModuleImporter(modules=[], repo_raw_url=self.repo_raw_url_without_slash, branch=self.branch) + self.assertEqual(importer.repo_raw_url, self.repo_raw_url) + + def test__file_url(self): + importer = GitRepoModuleImporter(modules=[], repo_raw_url=self.repo_raw_url, branch=self.branch) + self.assertEqual(importer._file_url(self.fullname, is_pkg=True), self.package_url) + self.assertEqual(importer._file_url(self.fullname, is_pkg=False), self.module_url) + + def test__fetch_repo_file__no_cache(self): + importer = GitRepoModuleImporter(modules=[], + repo_raw_url=self.repo_raw_url, + branch=self.branch, + use_cache=False) + first_request_content = ReadableObject(read_return='first_request_content') + second_request_content = ReadableObject(read_return='second_request_content') + + with patch(URLLIB2_URLOPEN, MagicMock(return_value=first_request_content)): + self.assertEqual(importer._fetch_repo_file(self.module_url), first_request_content.read()) + self.assertEqual(importer.file_cache, {}) + + with patch(URLLIB2_URLOPEN, MagicMock(return_value=second_request_content)): + self.assertEqual(importer._fetch_repo_file(self.module_url), second_request_content.read()) + self.assertEqual(importer.file_cache, {}) + + with patch(URLLIB2_URLOPEN, MagicMock(side_effect=IOError())): + self.assertRaises(IOError, importer._fetch_repo_file, self.module_url) + self.assertEqual(importer.error_cache, {}) + + def test__fetch_repo_file__use_cache(self): + importer = GitRepoModuleImporter(modules=[], repo_raw_url=self.repo_raw_url, branch=self.branch) + first_request_content = ReadableObject(read_return='first_request_content') + second_request_content = ReadableObject(read_return='second_request_content') + io_error = IOError() + + with patch(URLLIB2_URLOPEN, MagicMock(return_value=first_request_content)): + self.assertEqual(importer._fetch_repo_file(self.module_url), first_request_content.read()) + self.assertEqual(importer.file_cache[self.module_url], first_request_content.read()) + + with patch(URLLIB2_URLOPEN, MagicMock(return_value=second_request_content)): + self.assertEqual(importer._fetch_repo_file(self.module_url), first_request_content.read()) + self.assertEqual(importer.file_cache[self.module_url], first_request_content.read()) + + with patch(URLLIB2_URLOPEN, MagicMock(side_effect=io_error)): + self.assertRaises(IOError, importer._fetch_repo_file, self.package_url) + self.assertIs(importer.error_cache[self.package_url], io_error) + + with patch(URLLIB2_URLOPEN, MagicMock(side_effect=NotImplementedError())): + self.assertRaises(IOError, importer._fetch_repo_file, self.package_url) + self.assertIs(importer.error_cache[self.package_url], io_error) + + def test_is_package(self): + importer = GitRepoModuleImporter(modules=[], repo_raw_url=self.repo_raw_url, branch=self.branch) + + with patch(UTILS_IMPORTER_GIT__FETCH_REPO_FILE, MagicMock()): + self.assertTrue(importer.is_package(self.fullname)) + importer._fetch_repo_file.assert_called_once_with(importer._file_url(self.fullname, is_pkg=True)) + + with patch(UTILS_IMPORTER_GIT__FETCH_REPO_FILE, MagicMock(side_effect=IOError)): + self.assertFalse(importer.is_package(self.fullname)) + importer._fetch_repo_file.assert_called_once_with(importer._file_url(self.fullname, is_pkg=True)) + + @patch(UTILS_IMPORTER_GIT_GET_FILE, MagicMock(return_value=GET_FILE_RETURN)) + @patch(UTILS_IMPORTER_GIT_GET_SOURCE, MagicMock(return_value=GET_SOURCE_RETURN)) + def test_get_code(self): + expect_code = compile(GET_SOURCE_RETURN, GET_FILE_RETURN, 'exec') + importer = GitRepoModuleImporter(modules=[], repo_raw_url=self.repo_raw_url, branch=self.branch) + + self.assertEqual(expect_code, importer.get_code(self.fullname)) + + @patch(UTILS_IMPORTER_GIT_IS_PACKAGE, MagicMock(return_value=IS_PACKAGE_RETURN)) + @patch(UTILS_IMPORTER_GIT__FETCH_REPO_FILE, MagicMock(return_value=_FETCH_REPO_FILE_RETURN)) + def test_get_source(self): + importer = GitRepoModuleImporter(modules=[], repo_raw_url=self.repo_raw_url, branch=self.branch) + + source = importer.get_source(self.fullname) + + self.assertEqual(source, _FETCH_REPO_FILE_RETURN) + importer._fetch_repo_file.assert_called_once_with(importer._file_url(self.fullname, is_pkg=IS_PACKAGE_RETURN)) + + @patch(UTILS_IMPORTER_GIT_IS_PACKAGE, MagicMock(return_value=IS_PACKAGE_RETURN)) + @patch(UTILS_IMPORTER_GIT__FETCH_REPO_FILE, MagicMock(side_effect=IOError)) + def test_get_source__catch_io_error(self): + importer = GitRepoModuleImporter(modules=[], repo_raw_url=self.repo_raw_url, branch=self.branch) + + self.assertRaises(ImportError, importer.get_source, self.fullname) + importer._fetch_repo_file.assert_called_once_with(importer._file_url(self.fullname, is_pkg=IS_PACKAGE_RETURN)) + + def test_get_path(self): + importer = GitRepoModuleImporter(modules=[], repo_raw_url=self.repo_raw_url, branch=self.branch) + self.assertEqual(importer.get_path(self.fullname), ['https://test-git-repo-raw/master/module1/module2/module3']) + + def test_get_file(self): + importer = GitRepoModuleImporter(modules=[], repo_raw_url=self.repo_raw_url, branch=self.branch) + + with patch(UTILS_IMPORTER_GIT_IS_PACKAGE, MagicMock(return_value=False)): + self.assertEqual(importer.get_file(self.fullname), self.module_url) + + with patch(UTILS_IMPORTER_GIT_IS_PACKAGE, MagicMock(return_value=True)): + self.assertEqual(importer.get_file(self.fullname), self.package_url) + diff --git a/pipeline/utils/importer/__init__.py b/pipeline/utils/importer/__init__.py new file mode 100644 index 0000000000..6a09231373 --- /dev/null +++ b/pipeline/utils/importer/__init__.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. +""" # noqa diff --git a/pipeline/utils/importer/base.py b/pipeline/utils/importer/base.py new file mode 100644 index 0000000000..46bbc6460b --- /dev/null +++ b/pipeline/utils/importer/base.py @@ -0,0 +1,170 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. +""" # noqa + +import imp +import sys +import logging +import traceback + +from contextlib import contextmanager +from abc import ABCMeta, abstractmethod + +logger = logging.getLogger(__name__) + + +@contextmanager +def hook_sandbox(hook, fullname): + hook_name = hook.__func__.func_name + try: + logger.info('Execute {hook_name} for {module}'.format(module=fullname, hook_name=hook_name)) + yield + except Exception: + logger.error('{module} {hook_name} raise exception: {traceback}'.format( + module=fullname, + hook_name=hook_name, + traceback=traceback.format_exc() + )) + + +class NonstandardModuleImporter(object): + __metaclass__ = ABCMeta + + def __init__(self, modules): + self.modules = modules + + def find_module(self, fullname, path=None): + logger.info('=============FINDER: {cls}'.format(cls=self.__class__.__name__)) + logger.info('Try to find module: {module} in path: {path}'.format(module=fullname, + path=path)) + + logger.info('Check if in declared nonstandard modules: {modules}'.format(modules=self.modules)) + root_parent = fullname.split('.')[0] + if root_parent not in self.modules: + logger.info('Root module({module}) are not find in nonstandard modules'.format(module=root_parent)) + return None + + logger.info('Check if is built-in module') + try: + loader = imp.find_module(fullname, path) + if loader: + logger.info('Found {module} locally'.format(module=fullname)) + return None + except ImportError: + pass + + logger.info('Checking if is name repetition') + if fullname.split('.').count(fullname.split('.')[-1]) > 1: + logger.info('Found {module} locally'.format(module=fullname)) + return None + + with hook_sandbox(fullname=fullname, hook=self.accept_find_module_request_hook): + self.accept_find_module_request_hook(fullname=fullname, path=path) + + return self + + def load_module(self, fullname): + try: + imp.acquire_lock() + + logger.info('=============LOADER: {cls}'.format(cls=self.__class__.__name__)) + logger.info('Try to load module: {module}'.format(module=fullname)) + + if fullname in sys.modules: + logger.info('Module {module} already loaded'.format(module=fullname)) + return sys.modules[fullname] + + is_pkg = self.is_package(fullname) + + try: + src_code = self.get_source(fullname) + except ImportError as e: + logger.info('Get source code for {module} error: {message}'.format(module=fullname, + message=e.message)) + return None + + logger.info('Importing {module}'.format(module=fullname)) + mod = sys.modules.setdefault(fullname, imp.new_module(fullname)) + + with hook_sandbox(fullname=fullname, hook=self.pre_load_module_hook): + self.pre_load_module_hook(fullname=fullname, module=mod) + + mod.__file__ = self.get_file(fullname) + mod.__loader__ = self + mod.__name__ = fullname + if is_pkg: + mod.__path__ = self.get_path(fullname) + mod.__package__ = fullname + else: + mod.__package__ = fullname.rpartition('.')[0] + + logger.info('Module prepared, ready to execute source code for {module}'.format(module=fullname)) + logger.info('Source code for {module}:\n{src_code}'.format(module=fullname, + src_code=src_code)) + + self._execute_src_code(src_code=src_code, module=mod) + + with hook_sandbox(fullname=fullname, hook=self.post_load_module_hook): + self.post_load_module_hook(fullname=fullname, module=mod) + + return mod + + except Exception: + + with hook_sandbox(fullname=fullname, hook=self.import_error_hook): + self.import_error_hook(fullname) + + err_msg = '{module} import raise exception: {traceback}'.format( + module=fullname, + traceback=traceback.format_exc() + ) + logger.error(err_msg) + + if fullname in sys.modules: + logger.info('Remove module {module} from sys.modules'.format(module=fullname)) + del sys.modules[fullname] + + raise ImportError(err_msg) + + finally: + imp.release_lock() + + def _execute_src_code(self, src_code, module): + exec src_code in module.__dict__ + + @abstractmethod + def is_package(self, fullname): + raise NotImplementedError() + + @abstractmethod + def get_code(self, fullname): + raise NotImplementedError() + + @abstractmethod + def get_source(self, fullname): + raise NotImplementedError() + + @abstractmethod + def get_file(self, fullname): + return NotImplementedError() + + @abstractmethod + def get_path(self, fullname): + return NotImplementedError() + + def accept_find_module_request_hook(self, fullname, path): + pass + + def pre_load_module_hook(self, fullname, module): + pass + + def post_load_module_hook(self, fullname, module): + pass + + def import_error_hook(self, fullname): + pass diff --git a/pipeline/utils/importer/git.py b/pipeline/utils/importer/git.py new file mode 100644 index 0000000000..3caa5ddf8d --- /dev/null +++ b/pipeline/utils/importer/git.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. +""" # noqa + +import logging +import urlparse +import urllib2 + +from pipeline.utils.importer.base import NonstandardModuleImporter + +logger = logging.getLogger(__name__) + + +class GitRepoModuleImporter(NonstandardModuleImporter): + + def __init__(self, modules, repo_raw_url, branch, use_cache=True): + super(GitRepoModuleImporter, self).__init__(modules=modules) + self.repo_raw_url = repo_raw_url if repo_raw_url.endswith('/') else '%s/' % repo_raw_url + self.branch = branch + self.use_cache = use_cache + self.file_cache = {} + self.error_cache = {} + + def is_package(self, fullname): + try: + self._fetch_repo_file(self._file_url(fullname, is_pkg=True)) + except IOError: + return False + + return True + + def get_code(self, fullname): + return compile(self.get_source(fullname), self.get_file(fullname), 'exec') + + def get_source(self, fullname): + try: + return self._fetch_repo_file(self._file_url(fullname, is_pkg=self.is_package(fullname))) + except IOError: + raise ImportError('Can not find {module} in {repo}/{branch}'.format(module=fullname, + repo=self.repo_raw_url, + branch=self.branch)) + + def get_path(self, fullname): + return [self._file_url(fullname, is_pkg=True).rpartition('/')[0]] + + def get_file(self, fullname): + return self._file_url(fullname, is_pkg=self.is_package(fullname)) + + def _file_url(self, fullname, is_pkg=False): + base_url = '%s/' % urlparse.urljoin(self.repo_raw_url, self.branch) + path = fullname.replace('.', '/') + file_name = '%s/__init__.py' % path if is_pkg else '%s.py' % path + return urlparse.urljoin(base_url, file_name) + + def _fetch_repo_file(self, file_url): + logger.info('Try to fetch git file: {file_url}'.format(file_url=file_url)) + if self.use_cache: + + if file_url in self.file_cache: + logger.info('Content in cache for git file: {file_url} found'.format(file_url=file_url)) + return self.file_cache[file_url] + + if file_url in self.error_cache: + logger.info('Error in cache for git file: {file_url} found'.format(file_url=file_url)) + raise self.error_cache[file_url] + + try: + file_content = urllib2.urlopen(file_url).read() + except IOError as e: + logger.info('Error cached for git file: {file_url}'.format(file_url=file_url)) + self.error_cache[file_url] = e + raise e + + self.file_cache[file_url] = file_content + logger.info('Content cached for git file: {file_url}'.format(file_url=file_url)) + return file_content + + return urllib2.urlopen(file_url).read() From c9a811df9a6e9904821c6b3fe20d2044a95f6f41 Mon Sep 17 00:00:00 2001 From: homholueng Date: Tue, 9 Apr 2019 15:45:45 +0800 Subject: [PATCH 02/29] =?UTF-8?q?minor:=20=E5=B0=86=20urllib2=20=E7=9A=84?= =?UTF-8?q?=E4=BD=BF=E7=94=A8=E6=94=B9=E4=B8=BA=20requests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pipeline/tests/mock.py | 5 ++- pipeline/tests/mock_settings.py | 2 +- pipeline/tests/utils/importer/git.py | 58 ++++++++++++++-------------- pipeline/utils/importer/git.py | 45 ++++++++------------- 4 files changed, 48 insertions(+), 62 deletions(-) diff --git a/pipeline/tests/mock.py b/pipeline/tests/mock.py index 4ce353c15e..9c59d885bd 100644 --- a/pipeline/tests/mock.py +++ b/pipeline/tests/mock.py @@ -31,9 +31,10 @@ class Object(object): pass -class ReadableObject(object): +class MockResponse(object): def __init__(self, **kwargs): - self.read = MagicMock(return_value=kwargs.get('read_return')) + self.content = kwargs.get('content') + self.ok = kwargs.get('ok', True) class ContextObject(object): diff --git a/pipeline/tests/mock_settings.py b/pipeline/tests/mock_settings.py index fcfbb14487..d9d68760dd 100644 --- a/pipeline/tests/mock_settings.py +++ b/pipeline/tests/mock_settings.py @@ -14,7 +14,7 @@ SYS_MODULES = 'sys.modules' IMP_ACQUIRE_LOCK = 'imp.acquire_lock' IMP_RELEASE_LOCK = 'imp.release_lock' -URLLIB2_URLOPEN = 'urllib2.urlopen' +REQUESTS_GET = 'requests.get' PIPELINE_CORE_GATEWAY_DEFORMAT = 'pipeline.core.flow.gateway.deformat_constant_key' PIPELINE_CORE_CONSTANT_RESOLVE = 'pipeline.core.data.expression.ConstantTemplate.resolve_data' diff --git a/pipeline/tests/utils/importer/git.py b/pipeline/tests/utils/importer/git.py index a83c5f4fff..c5fac7c0f5 100644 --- a/pipeline/tests/utils/importer/git.py +++ b/pipeline/tests/utils/importer/git.py @@ -50,52 +50,50 @@ def test__fetch_repo_file__no_cache(self): repo_raw_url=self.repo_raw_url, branch=self.branch, use_cache=False) - first_request_content = ReadableObject(read_return='first_request_content') - second_request_content = ReadableObject(read_return='second_request_content') + first_resp = MockResponse(content='first_request_content') + second_resp = MockResponse(content='second_request_content') - with patch(URLLIB2_URLOPEN, MagicMock(return_value=first_request_content)): - self.assertEqual(importer._fetch_repo_file(self.module_url), first_request_content.read()) + with patch(REQUESTS_GET, MagicMock(return_value=first_resp)): + self.assertEqual(importer._fetch_repo_file(self.module_url), first_resp.content) self.assertEqual(importer.file_cache, {}) - with patch(URLLIB2_URLOPEN, MagicMock(return_value=second_request_content)): - self.assertEqual(importer._fetch_repo_file(self.module_url), second_request_content.read()) + with patch(REQUESTS_GET, MagicMock(return_value=second_resp)): + self.assertEqual(importer._fetch_repo_file(self.module_url), second_resp.content) self.assertEqual(importer.file_cache, {}) - with patch(URLLIB2_URLOPEN, MagicMock(side_effect=IOError())): - self.assertRaises(IOError, importer._fetch_repo_file, self.module_url) - self.assertEqual(importer.error_cache, {}) + with patch(REQUESTS_GET, MagicMock(return_value=MockResponse(ok=False))): + self.assertIsNone(importer._fetch_repo_file(self.module_url)) def test__fetch_repo_file__use_cache(self): importer = GitRepoModuleImporter(modules=[], repo_raw_url=self.repo_raw_url, branch=self.branch) - first_request_content = ReadableObject(read_return='first_request_content') - second_request_content = ReadableObject(read_return='second_request_content') - io_error = IOError() + first_resp = MockResponse(content='first_request_content') + second_resp = MockResponse(content='second_request_content') - with patch(URLLIB2_URLOPEN, MagicMock(return_value=first_request_content)): - self.assertEqual(importer._fetch_repo_file(self.module_url), first_request_content.read()) - self.assertEqual(importer.file_cache[self.module_url], first_request_content.read()) + with patch(REQUESTS_GET, MagicMock(return_value=first_resp)): + self.assertEqual(importer._fetch_repo_file(self.module_url), first_resp.content) + self.assertEqual(importer.file_cache[self.module_url], first_resp.content) - with patch(URLLIB2_URLOPEN, MagicMock(return_value=second_request_content)): - self.assertEqual(importer._fetch_repo_file(self.module_url), first_request_content.read()) - self.assertEqual(importer.file_cache[self.module_url], first_request_content.read()) + with patch(REQUESTS_GET, MagicMock(return_value=second_resp)): + self.assertEqual(importer._fetch_repo_file(self.module_url), first_resp.content) + self.assertEqual(importer.file_cache[self.module_url], first_resp.content) - with patch(URLLIB2_URLOPEN, MagicMock(side_effect=io_error)): - self.assertRaises(IOError, importer._fetch_repo_file, self.package_url) - self.assertIs(importer.error_cache[self.package_url], io_error) + with patch(REQUESTS_GET, MagicMock(return_value=MockResponse(ok=False))): + self.assertIsNone(importer._fetch_repo_file(self.package_url)) + self.assertIsNone(importer.file_cache[self.package_url]) - with patch(URLLIB2_URLOPEN, MagicMock(side_effect=NotImplementedError())): - self.assertRaises(IOError, importer._fetch_repo_file, self.package_url) - self.assertIs(importer.error_cache[self.package_url], io_error) + with patch(REQUESTS_GET, MagicMock(return_value=second_resp)): + self.assertIsNone(importer._fetch_repo_file(self.package_url)) + self.assertIsNone(importer.file_cache[self.package_url]) def test_is_package(self): importer = GitRepoModuleImporter(modules=[], repo_raw_url=self.repo_raw_url, branch=self.branch) - with patch(UTILS_IMPORTER_GIT__FETCH_REPO_FILE, MagicMock()): - self.assertTrue(importer.is_package(self.fullname)) + with patch(UTILS_IMPORTER_GIT__FETCH_REPO_FILE, MagicMock(return_value=None)): + self.assertFalse(importer.is_package(self.fullname)) importer._fetch_repo_file.assert_called_once_with(importer._file_url(self.fullname, is_pkg=True)) - with patch(UTILS_IMPORTER_GIT__FETCH_REPO_FILE, MagicMock(side_effect=IOError)): - self.assertFalse(importer.is_package(self.fullname)) + with patch(UTILS_IMPORTER_GIT__FETCH_REPO_FILE, MagicMock(return_value='')): + self.assertTrue(importer.is_package(self.fullname)) importer._fetch_repo_file.assert_called_once_with(importer._file_url(self.fullname, is_pkg=True)) @patch(UTILS_IMPORTER_GIT_GET_FILE, MagicMock(return_value=GET_FILE_RETURN)) @@ -117,8 +115,8 @@ def test_get_source(self): importer._fetch_repo_file.assert_called_once_with(importer._file_url(self.fullname, is_pkg=IS_PACKAGE_RETURN)) @patch(UTILS_IMPORTER_GIT_IS_PACKAGE, MagicMock(return_value=IS_PACKAGE_RETURN)) - @patch(UTILS_IMPORTER_GIT__FETCH_REPO_FILE, MagicMock(side_effect=IOError)) - def test_get_source__catch_io_error(self): + @patch(UTILS_IMPORTER_GIT__FETCH_REPO_FILE, MagicMock(return_value=None)) + def test_get_source__fetch_none(self): importer = GitRepoModuleImporter(modules=[], repo_raw_url=self.repo_raw_url, branch=self.branch) self.assertRaises(ImportError, importer.get_source, self.fullname) diff --git a/pipeline/utils/importer/git.py b/pipeline/utils/importer/git.py index 86a86e2b48..834565681b 100644 --- a/pipeline/utils/importer/git.py +++ b/pipeline/utils/importer/git.py @@ -13,7 +13,7 @@ import logging import urlparse -import urllib2 +import requests from pipeline.utils.importer.base import NonstandardModuleImporter @@ -28,26 +28,21 @@ def __init__(self, modules, repo_raw_url, branch, use_cache=True): self.branch = branch self.use_cache = use_cache self.file_cache = {} - self.error_cache = {} def is_package(self, fullname): - try: - self._fetch_repo_file(self._file_url(fullname, is_pkg=True)) - except IOError: - return False - - return True + return self._fetch_repo_file(self._file_url(fullname, is_pkg=True)) is not None def get_code(self, fullname): return compile(self.get_source(fullname), self.get_file(fullname), 'exec') def get_source(self, fullname): - try: - return self._fetch_repo_file(self._file_url(fullname, is_pkg=self.is_package(fullname))) - except IOError: - raise ImportError('Can not find {module} in {repo}/{branch}'.format(module=fullname, - repo=self.repo_raw_url, - branch=self.branch)) + source_code = self._fetch_repo_file(self._file_url(fullname, is_pkg=self.is_package(fullname))) + + if source_code is None: + raise ImportError('Can not find {module} in {repo}{branch}'.format(module=fullname, + repo=self.repo_raw_url, + branch=self.branch)) + return source_code def get_path(self, fullname): return [self._file_url(fullname, is_pkg=True).rpartition('/')[0]] @@ -63,25 +58,17 @@ def _file_url(self, fullname, is_pkg=False): def _fetch_repo_file(self, file_url): logger.info('Try to fetch git file: {file_url}'.format(file_url=file_url)) - if self.use_cache: - if file_url in self.file_cache: - logger.info('Content in cache for git file: {file_url} found'.format(file_url=file_url)) - return self.file_cache[file_url] + if self.use_cache and file_url in self.file_cache: + logger.info('Use content in cache for git file: {file_url}'.format(file_url=file_url)) + return self.file_cache[file_url] - if file_url in self.error_cache: - logger.info('Error in cache for git file: {file_url} found'.format(file_url=file_url)) - raise self.error_cache[file_url] + resp = requests.get(file_url, timeout=10) - try: - file_content = urllib2.urlopen(file_url).read() - except IOError as e: - logger.info('Error cached for git file: {file_url}'.format(file_url=file_url)) - self.error_cache[file_url] = e - raise e + file_content = resp.content if resp.ok else None + if self.use_cache: self.file_cache[file_url] = file_content logger.info('Content cached for git file: {file_url}'.format(file_url=file_url)) - return file_content - return urllib2.urlopen(file_url).read() + return file_content From 27a1df40bf760f78529f39290785c0fe1d8a666e Mon Sep 17 00:00:00 2001 From: homholueng Date: Tue, 9 Apr 2019 15:46:21 +0800 Subject: [PATCH 03/29] =?UTF-8?q?minor:=20=E7=A7=BB=E9=99=A4=E6=B5=8B?= =?UTF-8?q?=E8=AF=95=E4=BB=A3=E7=A0=81=E4=B8=AD=E7=9A=84=E5=85=B3=E9=94=AE?= =?UTF-8?q?=E5=AD=97=E4=BD=BF=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pipeline/tests/utils/importer/base.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pipeline/tests/utils/importer/base.py b/pipeline/tests/utils/importer/base.py index d1eef07f34..8f318dbf8c 100644 --- a/pipeline/tests/utils/importer/base.py +++ b/pipeline/tests/utils/importer/base.py @@ -146,15 +146,15 @@ def test_load_module__get_source_raise_import_error(self): def test_load_module__is_package(self): src_code = 'src_code' fullname = 'magic_module' - file = 'file' + _file = 'file' path = 'path' - importer = DummyImporter(is_package=True, get_source=src_code, get_file=file, get_path=path) + importer = DummyImporter(is_package=True, get_source=src_code, get_file=_file, get_path=path) with patch(SYS_MODULES, {}): mod = importer.load_module(fullname=fullname) self.assertIs(sys.modules[fullname], mod) - self.assertEqual(mod.__file__, file) + self.assertEqual(mod.__file__, _file) self.assertIs(mod.__loader__, importer) self.assertEqual(mod.__path__, path) self.assertEqual(mod.__package__, fullname) @@ -168,14 +168,14 @@ def test_load_module__is_package(self): def test_load_module__is_not_package(self): src_code = 'src_code' fullname = 'magic_module.sub_module' - file = 'file' - importer = DummyImporter(is_package=False, get_source=src_code, get_file=file) + _file = 'file' + importer = DummyImporter(is_package=False, get_source=src_code, get_file=_file) with patch(SYS_MODULES, {}): mod = importer.load_module(fullname=fullname) self.assertIs(sys.modules[fullname], mod) - self.assertEqual(mod.__file__, file) + self.assertEqual(mod.__file__, _file) self.assertIs(mod.__loader__, importer) self.assertEqual(mod.__package__, fullname.rpartition('.')[0]) From f9c42471e68fa815f144f49188d37e670f1b3d9c Mon Sep 17 00:00:00 2001 From: homholueng Date: Tue, 9 Apr 2019 15:58:41 +0800 Subject: [PATCH 04/29] =?UTF-8?q?minor:=20=E5=86=85=E9=83=A8=E5=8C=85?= =?UTF-8?q?=E4=B8=8E=E7=AC=AC=E4=B8=89=E6=96=B9=E5=8C=85=E4=B9=8B=E9=97=B4?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E7=A9=BA=E6=A0=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pipeline/utils/importer/git.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pipeline/utils/importer/git.py b/pipeline/utils/importer/git.py index 834565681b..cec7fadd37 100644 --- a/pipeline/utils/importer/git.py +++ b/pipeline/utils/importer/git.py @@ -12,6 +12,7 @@ """ import logging + import urlparse import requests From a45eada1d854a608a9bee202c9dfdb13b1c261cd Mon Sep 17 00:00:00 2001 From: homholueng Date: Wed, 10 Apr 2019 10:02:45 +0800 Subject: [PATCH 05/29] =?UTF-8?q?minor:=20=E5=88=A0=E9=99=A4=E5=86=97?= =?UTF-8?q?=E4=BD=99=E7=BC=96=E7=A0=81=E5=A3=B0=E6=98=8E;=20=E7=AC=AC?= =?UTF-8?q?=E4=B8=89=E6=96=B9=E5=8C=85=E5=BC=95=E7=94=A8=E9=97=B4=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E7=A9=BA=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pipeline/component_framework/models.py | 1 - pipeline/tests/utils/importer/base.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/component_framework/models.py b/pipeline/component_framework/models.py index fc457bc07c..fcdd2ee9ad 100644 --- a/pipeline/component_framework/models.py +++ b/pipeline/component_framework/models.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 # -*- coding: utf-8 -*- """ Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community diff --git a/pipeline/tests/utils/importer/base.py b/pipeline/tests/utils/importer/base.py index 8f318dbf8c..af170a6289 100644 --- a/pipeline/tests/utils/importer/base.py +++ b/pipeline/tests/utils/importer/base.py @@ -13,6 +13,7 @@ import imp import sys + from django.test import TestCase from pipeline.tests.mock import * # noqa From ef671895aa756442975ae900e32c760b2850f643 Mon Sep 17 00:00:00 2001 From: homholueng Date: Wed, 10 Apr 2019 17:52:29 +0800 Subject: [PATCH 06/29] =?UTF-8?q?feature:=20=E6=B7=BB=E5=8A=A0=E8=BF=9C?= =?UTF-8?q?=E7=A8=8B=E5=8C=85=E6=BA=90=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pipeline/contrib/external_plugins/__init__.py | 12 ++ pipeline/contrib/external_plugins/admin.py | 40 +++++ pipeline/contrib/external_plugins/apps.py | 20 +++ .../contrib/external_plugins/exceptions.py | 16 ++ .../migrations/0001_initial.py | 71 ++++++++ .../external_plugins/migrations/__init__.py | 12 ++ .../external_plugins/models/__init__.py | 15 ++ .../contrib/external_plugins/models/base.py | 98 +++++++++++ .../contrib/external_plugins/models/fields.py | 30 ++++ .../contrib/external_plugins/models/source.py | 66 ++++++++ .../external_plugins/tests/__init__.py | 12 ++ .../external_plugins/tests/models/__init__.py | 12 ++ .../tests/models/test_base.py | 153 ++++++++++++++++++ .../tests/models/test_source.py | 37 +++++ 14 files changed, 594 insertions(+) create mode 100644 pipeline/contrib/external_plugins/__init__.py create mode 100644 pipeline/contrib/external_plugins/admin.py create mode 100644 pipeline/contrib/external_plugins/apps.py create mode 100644 pipeline/contrib/external_plugins/exceptions.py create mode 100644 pipeline/contrib/external_plugins/migrations/0001_initial.py create mode 100644 pipeline/contrib/external_plugins/migrations/__init__.py create mode 100644 pipeline/contrib/external_plugins/models/__init__.py create mode 100644 pipeline/contrib/external_plugins/models/base.py create mode 100644 pipeline/contrib/external_plugins/models/fields.py create mode 100644 pipeline/contrib/external_plugins/models/source.py create mode 100644 pipeline/contrib/external_plugins/tests/__init__.py create mode 100644 pipeline/contrib/external_plugins/tests/models/__init__.py create mode 100644 pipeline/contrib/external_plugins/tests/models/test_base.py create mode 100644 pipeline/contrib/external_plugins/tests/models/test_source.py diff --git a/pipeline/contrib/external_plugins/__init__.py b/pipeline/contrib/external_plugins/__init__.py new file mode 100644 index 0000000000..90524bb0e7 --- /dev/null +++ b/pipeline/contrib/external_plugins/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/pipeline/contrib/external_plugins/admin.py b/pipeline/contrib/external_plugins/admin.py new file mode 100644 index 0000000000..5d477bcf16 --- /dev/null +++ b/pipeline/contrib/external_plugins/admin.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.contrib import admin + +from pipeline.contrib.external_plugins.models import ( + GitRepoSource, + S3Source, + FileSystemSource +) + + +# Register your models here. + +@admin.register(GitRepoSource) +class GitRepoSourceAdmin(admin.ModelAdmin): + list_display = ['name', 'from_config', 'repo_raw_address', 'branch'] + search_fields = ['name', 'branch', 'repo_raw_address'] + + +@admin.register(S3Source) +class S3SourceAdmin(admin.ModelAdmin): + list_display = ['name', 'from_config', 'service_address', 'bucket'] + search_fields = ['name', 'bucket', 'service_address'] + + +@admin.register(FileSystemSource) +class FileSystemSourceAdmin(admin.ModelAdmin): + list_display = ['name', 'from_config', 'path'] + search_fields = ['name', 'path'] diff --git a/pipeline/contrib/external_plugins/apps.py b/pipeline/contrib/external_plugins/apps.py new file mode 100644 index 0000000000..03a13e0713 --- /dev/null +++ b/pipeline/contrib/external_plugins/apps.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from __future__ import unicode_literals + +from django.apps import AppConfig + + +class ExternalPluginsConfig(AppConfig): + name = 'external_plugins' diff --git a/pipeline/contrib/external_plugins/exceptions.py b/pipeline/contrib/external_plugins/exceptions.py new file mode 100644 index 0000000000..ddbdaa6685 --- /dev/null +++ b/pipeline/contrib/external_plugins/exceptions.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + + +class InvalidOperationException(Exception): + pass diff --git a/pipeline/contrib/external_plugins/migrations/0001_initial.py b/pipeline/contrib/external_plugins/migrations/0001_initial.py new file mode 100644 index 0000000000..4e429c49fe --- /dev/null +++ b/pipeline/contrib/external_plugins/migrations/0001_initial.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from __future__ import unicode_literals + +from django.db import migrations, models +import pipeline.contrib.external_plugins.models.fields + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='FileSystemSource', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=128, unique=True, verbose_name='\u5305\u6e90\u540d')), + ('from_config', models.BooleanField(default=False, verbose_name='\u662f\u5426\u662f\u4ece\u914d\u7f6e\u6587\u4ef6\u4e2d\u8bfb\u53d6\u7684')), + ('packages', pipeline.contrib.external_plugins.models.fields.JSONTextField(verbose_name='\u6a21\u5757\u914d\u7f6e')), + ('path', models.TextField(verbose_name='\u6587\u4ef6\u7cfb\u7edf\u8def\u5f84')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='GitRepoSource', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=128, unique=True, verbose_name='\u5305\u6e90\u540d')), + ('from_config', models.BooleanField(default=False, verbose_name='\u662f\u5426\u662f\u4ece\u914d\u7f6e\u6587\u4ef6\u4e2d\u8bfb\u53d6\u7684')), + ('packages', pipeline.contrib.external_plugins.models.fields.JSONTextField(verbose_name='\u6a21\u5757\u914d\u7f6e')), + ('repo_raw_address', models.TextField(verbose_name='\u6587\u4ef6\u6258\u7ba1\u4ed3\u5e93\u94fe\u63a5')), + ('branch', models.CharField(max_length=128, verbose_name='\u5206\u652f\u540d')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='S3Source', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=128, unique=True, verbose_name='\u5305\u6e90\u540d')), + ('from_config', models.BooleanField(default=False, verbose_name='\u662f\u5426\u662f\u4ece\u914d\u7f6e\u6587\u4ef6\u4e2d\u8bfb\u53d6\u7684')), + ('packages', pipeline.contrib.external_plugins.models.fields.JSONTextField(verbose_name='\u6a21\u5757\u914d\u7f6e')), + ('service_address', models.TextField(verbose_name='\u5bf9\u8c61\u5b58\u50a8\u670d\u52a1\u5730\u5740')), + ('bucket', models.TextField(verbose_name='bucket \u540d')), + ('access_key', models.TextField(verbose_name='access key')), + ('secret_key', models.TextField(verbose_name='secret key')), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/pipeline/contrib/external_plugins/migrations/__init__.py b/pipeline/contrib/external_plugins/migrations/__init__.py new file mode 100644 index 0000000000..90524bb0e7 --- /dev/null +++ b/pipeline/contrib/external_plugins/migrations/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/pipeline/contrib/external_plugins/models/__init__.py b/pipeline/contrib/external_plugins/models/__init__.py new file mode 100644 index 0000000000..50910cf4a8 --- /dev/null +++ b/pipeline/contrib/external_plugins/models/__init__.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from pipeline.contrib.external_plugins.models.base import source_cls_factory # noqa +from pipeline.contrib.external_plugins.models.source import * # noqa diff --git a/pipeline/contrib/external_plugins/models/base.py b/pipeline/contrib/external_plugins/models/base.py new file mode 100644 index 0000000000..df3df60fd5 --- /dev/null +++ b/pipeline/contrib/external_plugins/models/base.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from copy import deepcopy +from abc import abstractmethod + +from django.db import models +from django.utils.translation import ugettext_lazy as _ + +from pipeline.contrib.external_plugins import exceptions +from pipeline.contrib.external_plugins.models.fields import JSONTextField + +GIT = 'git' +S3 = 's3' +FILE_SYSTEM = 'fs' + +package_source_types = frozenset({ + GIT, + S3, + FILE_SYSTEM +}) + +source_cls_factory = { + +} + + +def package_source(cls): + source_cls_factory[cls.type()] = cls + return cls + + +class SourceManager(models.Manager): + + def create_source(self, name, packages, from_config, **kwargs): + create_kwargs = deepcopy(kwargs) + create_kwargs['name'] = name + create_kwargs['packages'] = packages + create_kwargs['from_config'] = from_config + return self.create(**create_kwargs) + + def remove_source(self, source_id): + source = self.get(id=source_id) + + if source.from_config: + raise exceptions.InvalidOperationException('Can not remove source create from config') + + source.delete() + + def update_source_from_config(self, configs): + + sources_from_config = self.filter(from_config=True).all() + existing_source_names = {source.name for source in sources_from_config} + source_name_in_config = {config['name'] for config in configs} + + invalid_source_names = existing_source_names - source_name_in_config + + # remove invalid source + self.filter(name__in=invalid_source_names).delete() + + # update and create source + for config in configs: + defaults = deepcopy(config['details']) + defaults['packages'] = config['packages'] + + self.update_or_create( + name=config['name'], + from_config=True, + defaults=defaults) + + +class ExternalPackageSource(models.Model): + name = models.CharField(_(u'包源名'), max_length=128, unique=True) + from_config = models.BooleanField(_(u"是否是从配置文件中读取的"), default=False) + packages = JSONTextField(_(u"模块配置")) + + objects = SourceManager() + + class Meta: + abstract = True + + @staticmethod + @abstractmethod + def type(): + raise NotImplementedError() + + def importer(self): + raise NotImplementedError() diff --git a/pipeline/contrib/external_plugins/models/fields.py b/pipeline/contrib/external_plugins/models/fields.py new file mode 100644 index 0000000000..455b19eb1f --- /dev/null +++ b/pipeline/contrib/external_plugins/models/fields.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import ujson as json +from django.db import models + + +class JSONTextField(models.TextField): + def __init__(self, *args, **kwargs): + super(JSONTextField, self).__init__(*args, **kwargs) + + def get_prep_value(self, value): + return json.dumps(value) + + def to_python(self, value): + value = super(JSONTextField, self).to_python(value) + return json.loads(value) + + def from_db_value(self, value, expression, connection, context): + return self.to_python(value) diff --git a/pipeline/contrib/external_plugins/models/source.py b/pipeline/contrib/external_plugins/models/source.py new file mode 100644 index 0000000000..61dcd5e0e6 --- /dev/null +++ b/pipeline/contrib/external_plugins/models/source.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.db import models +from django.utils.translation import ugettext_lazy as _ + +from pipeline.utils.importer.git import GitRepoModuleImporter + +from pipeline.contrib.external_plugins.models.base import ( + GIT, + S3, + FILE_SYSTEM, + package_source, + ExternalPackageSource) + + +@package_source +class GitRepoSource(ExternalPackageSource): + repo_raw_address = models.TextField(_(u"文件托管仓库链接")) + branch = models.CharField(_(u"分支名"), max_length=128) + + @staticmethod + def type(): + return GIT + + def importer(self): + return GitRepoModuleImporter(repo_raw_url=self.repo_raw_address, + branch=self.branch, + modules=self.packages.keys()) + + +@package_source +class S3Source(ExternalPackageSource): + service_address = models.TextField(_(u"对象存储服务地址")) + bucket = models.TextField(_(u"bucket 名")) + access_key = models.TextField(_(u"access key")) + secret_key = models.TextField(_(u"secret key")) + + @staticmethod + def type(): + return S3 + + def importer(self): + pass + + +@package_source +class FileSystemSource(ExternalPackageSource): + path = models.TextField(_(u"文件系统路径")) + + @staticmethod + def type(): + return FILE_SYSTEM + + def importer(self): + pass diff --git a/pipeline/contrib/external_plugins/tests/__init__.py b/pipeline/contrib/external_plugins/tests/__init__.py new file mode 100644 index 0000000000..90524bb0e7 --- /dev/null +++ b/pipeline/contrib/external_plugins/tests/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/pipeline/contrib/external_plugins/tests/models/__init__.py b/pipeline/contrib/external_plugins/tests/models/__init__.py new file mode 100644 index 0000000000..90524bb0e7 --- /dev/null +++ b/pipeline/contrib/external_plugins/tests/models/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/pipeline/contrib/external_plugins/tests/models/test_base.py b/pipeline/contrib/external_plugins/tests/models/test_base.py new file mode 100644 index 0000000000..0900da3c50 --- /dev/null +++ b/pipeline/contrib/external_plugins/tests/models/test_base.py @@ -0,0 +1,153 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.test import TestCase + +from pipeline.contrib.external_plugins import exceptions +from pipeline.contrib.external_plugins.models import GitRepoSource + +SOURCE_NAME = 'source_name' +PACKAGES = [{'1': 1}, {'2': 2}] +FROM_CONFIG = True +REPO_RAW_ADDRESS = 'REPO_RAW_ADDRESS' +BRANCH = 'master' + +OLD_SOURCE_1 = { + 'name': 'source_1', + 'details': { + 'repo_raw_address': 'old_address', + 'branch': 'stage', + }, + 'packages': { + 'root_package': { + 'version': '', + 'modules': ['test1'] + } + } +} + +OLD_SOURCE_3 = { + 'name': 'source_3', + 'details': { + 'repo_raw_address': 'old_address_3', + 'branch': 'master', + }, + 'packages': { + 'root_package': { + 'version': '', + 'modules': ['test5'] + } + } +} + +SOURCE_1 = { + 'name': 'source_1', + 'details': { + 'repo_raw_address': 'https://github.com/homholueng/plugins_example_1', + 'branch': 'master', + }, + 'packages': { + 'root_package': { + 'version': '', + 'modules': ['test1', 'test2'] + } + } +} + +SOURCE_2 = { + 'name': 'source_2', + 'details': { + 'repo_raw_address': 'https://github.com/homholueng/plugins_example_2', + 'branch': 'master', + }, + 'packages': { + 'root_package': { + 'version': '', + 'modules': ['test3', 'test4'] + } + } +} + +SOURCE_4 = { + 'name': 'source_4', + 'details': { + 'repo_raw_address': 'https://github.com/homholueng/plugins_example_4', + 'branch': 'master', + }, + 'packages': { + 'root_package': { + 'version': '', + 'modules': ['test5', 'test6'] + } + } +} + +GIT_SOURCE_CONFIGS = [SOURCE_1, SOURCE_2, SOURCE_4] + + +class ModelsBaseTestCase(TestCase): + + def setUp(self): + GitRepoSource.objects.create_source(name=SOURCE_NAME, + packages=PACKAGES, + from_config=FROM_CONFIG, + repo_raw_address=REPO_RAW_ADDRESS, + branch=BRANCH) + + def tearDown(self): + GitRepoSource.objects.all().delete() + + def test_create_source(self): + source_1 = GitRepoSource.objects.get(name=SOURCE_NAME) + self.assertEqual(source_1.name, SOURCE_NAME) + self.assertEqual(source_1.packages, PACKAGES) + self.assertEqual(source_1.from_config, FROM_CONFIG) + self.assertEqual(source_1.repo_raw_address, REPO_RAW_ADDRESS) + self.assertEqual(source_1.branch, BRANCH) + + def test_remove_source(self): + source_1 = GitRepoSource.objects.get(name=SOURCE_NAME) + + self.assertRaises(exceptions.InvalidOperationException, GitRepoSource.objects.remove_source, source_1.id) + + source_1.from_config = False + source_1.save() + + GitRepoSource.objects.remove_source(source_1.id) + + self.assertFalse(GitRepoSource.objects.filter(id=source_1.id).exists()) + + def _assert_source_equals_config(self, source, config): + self.assertEqual(source.name, config['name']) + self.assertEqual(source.packages, config['packages']) + self.assertEqual(source.repo_raw_address, config['details']['repo_raw_address']) + self.assertEqual(source.branch, config['details']['branch']) + + def test_update_source_from_config(self): + GitRepoSource.objects.all().delete() + + for source in [OLD_SOURCE_1, OLD_SOURCE_3]: + GitRepoSource.objects.create_source(name=source['name'], + packages=source['packages'], + from_config=True, + repo_raw_address=source['details']['repo_raw_address'], + branch=source['details']['branch']) + + GitRepoSource.objects.update_source_from_config(GIT_SOURCE_CONFIGS) + + self.assertFalse(GitRepoSource.objects.filter(name=OLD_SOURCE_3['name']).exists()) + + for config in GIT_SOURCE_CONFIGS: + source = GitRepoSource.objects.get(name=config['name']) + self.assertTrue(source.from_config) + self._assert_source_equals_config(source, config) diff --git a/pipeline/contrib/external_plugins/tests/models/test_source.py b/pipeline/contrib/external_plugins/tests/models/test_source.py new file mode 100644 index 0000000000..4528991dce --- /dev/null +++ b/pipeline/contrib/external_plugins/tests/models/test_source.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.test import TestCase + +from pipeline.contrib.external_plugins.models.base import ( + ExternalPackageSource, + GIT, + S3, + FILE_SYSTEM) +from pipeline.contrib.external_plugins.models import ( + GitRepoSource, + S3Source, + FileSystemSource) + + +class SourceTestCase(TestCase): + + def test_source_cls(self): + self.assertTrue(issubclass(GitRepoSource, ExternalPackageSource)) + self.assertTrue(issubclass(S3Source, ExternalPackageSource)) + self.assertTrue(issubclass(FileSystemSource, ExternalPackageSource)) + + def test_source_type(self): + self.assertEqual(GitRepoSource.type(), GIT) + self.assertEqual(S3Source.type(), S3) + self.assertEqual(FileSystemSource.type(), FILE_SYSTEM) From fc8aa51362486c6fe307ff6137bc8802232c5076 Mon Sep 17 00:00:00 2001 From: homholueng Date: Wed, 10 Apr 2019 20:40:19 +0800 Subject: [PATCH 07/29] =?UTF-8?q?feature:=20=E6=B7=BB=E5=8A=A0=E5=8A=A0?= =?UTF-8?q?=E8=BD=BD=E8=BF=9C=E7=A8=8B=E6=A8=A1=E5=9D=97=E7=9B=B8=E5=85=B3?= =?UTF-8?q?=E9=80=BB=E8=BE=91=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pipeline/contrib/external_plugins/__init__.py | 2 + pipeline/contrib/external_plugins/apps.py | 16 +- pipeline/contrib/external_plugins/loader.py | 35 ++++ .../contrib/external_plugins/models/base.py | 23 +++ .../tests/models/base/__init__.py | 0 .../base/test_external_package_source.py | 153 ++++++++++++++++++ .../tests/models/test_base.py | 2 +- pipeline/utils/importer/__init__.py | 3 + pipeline/utils/importer/utils.py | 33 ++++ 9 files changed, 264 insertions(+), 3 deletions(-) create mode 100644 pipeline/contrib/external_plugins/loader.py create mode 100644 pipeline/contrib/external_plugins/tests/models/base/__init__.py create mode 100644 pipeline/contrib/external_plugins/tests/models/base/test_external_package_source.py create mode 100644 pipeline/utils/importer/utils.py diff --git a/pipeline/contrib/external_plugins/__init__.py b/pipeline/contrib/external_plugins/__init__.py index 90524bb0e7..6bb83a65ef 100644 --- a/pipeline/contrib/external_plugins/__init__.py +++ b/pipeline/contrib/external_plugins/__init__.py @@ -10,3 +10,5 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ + +default_app_config = 'pipeline.contrib.external_plugins.apps.ExternalPluginsConfig' diff --git a/pipeline/contrib/external_plugins/apps.py b/pipeline/contrib/external_plugins/apps.py index 03a13e0713..d91dc055de 100644 --- a/pipeline/contrib/external_plugins/apps.py +++ b/pipeline/contrib/external_plugins/apps.py @@ -11,10 +11,22 @@ specific language governing permissions and limitations under the License. """ -from __future__ import unicode_literals - from django.apps import AppConfig +from django.db.utils import ProgrammingError + +from pipeline.conf import settings +from pipeline.contrib.external_plugins import loader +from pipeline.contrib.external_plugins.models import ExternalPackageSource class ExternalPluginsConfig(AppConfig): name = 'external_plugins' + + def ready(self): + try: + ExternalPackageSource.update_package_source_from_config(getattr(settings, 'COMPONENTS_PACKAGE_SOURCES', {})) + except ProgrammingError: + # first migrate + return + + loader.load_external_modules() diff --git a/pipeline/contrib/external_plugins/loader.py b/pipeline/contrib/external_plugins/loader.py new file mode 100644 index 0000000000..67eb0328f4 --- /dev/null +++ b/pipeline/contrib/external_plugins/loader.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import importlib + +from pipeline.utils.importer import importer_context +from pipeline.contrib.external_plugins.models import source_cls_factory + + +def load_external_modules(): + for source_type, source_model_cls in source_cls_factory.items(): + # get all external source + sources = source_model_cls.objects.all() + + # get importer for source + for source in sources: + _import_modules_in_source(source) + + +def _import_modules_in_source(source): + importer = source.importer() + + with importer_context(importer): + for module in source.modules: + importlib.import_module(module) diff --git a/pipeline/contrib/external_plugins/models/base.py b/pipeline/contrib/external_plugins/models/base.py index df3df60fd5..5877432bbd 100644 --- a/pipeline/contrib/external_plugins/models/base.py +++ b/pipeline/contrib/external_plugins/models/base.py @@ -11,12 +11,14 @@ specific language governing permissions and limitations under the License. """ +import importlib from copy import deepcopy from abc import abstractmethod from django.db import models from django.utils.translation import ugettext_lazy as _ +from pipeline.utils.importer.utils import importer_context from pipeline.contrib.external_plugins import exceptions from pipeline.contrib.external_plugins.models.fields import JSONTextField @@ -94,5 +96,26 @@ class Meta: def type(): raise NotImplementedError() + @abstractmethod def importer(self): raise NotImplementedError() + + @property + def modules(self): + modules = [] + + for _, package_info in self.packages: + modules.extend(package_info['modules']) + + return modules + + @staticmethod + def update_package_source_from_config(source_configs): + classified_config = {} + + for config in source_configs: + classified_config.setdefault(config.pop('type'), []).append(config) + + for source_type, config in classified_config.items(): + source_model_cls = source_cls_factory[source_type] + source_model_cls.objects.update_source_from_config(config=config) diff --git a/pipeline/contrib/external_plugins/tests/models/base/__init__.py b/pipeline/contrib/external_plugins/tests/models/base/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pipeline/contrib/external_plugins/tests/models/base/test_external_package_source.py b/pipeline/contrib/external_plugins/tests/models/base/test_external_package_source.py new file mode 100644 index 0000000000..6e46c2a34d --- /dev/null +++ b/pipeline/contrib/external_plugins/tests/models/base/test_external_package_source.py @@ -0,0 +1,153 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.test import TestCase + +from pipeline.contrib.external_plugins import exceptions +from pipeline.contrib.external_plugins.models import GitRepoSource + +SOURCE_NAME = 'source_name' +PACKAGES = [{'1': 1}, {'2': 2}] +FROM_CONFIG = True +REPO_RAW_ADDRESS = 'REPO_RAW_ADDRESS' +BRANCH = 'master' + +OLD_SOURCE_1 = { + 'name': 'source_1', + 'details': { + 'repo_raw_address': 'old_address', + 'branch': 'stage', + }, + 'packages': { + 'root_package': { + 'version': '', + 'modules': ['test1'] + } + } +} + +OLD_SOURCE_3 = { + 'name': 'source_3', + 'details': { + 'repo_raw_address': 'old_address_3', + 'branch': 'master', + }, + 'packages': { + 'root_package': { + 'version': '', + 'modules': ['test5'] + } + } +} + +SOURCE_1 = { + 'name': 'source_1', + 'details': { + 'repo_raw_address': 'https://github.com/homholueng/plugins_example_1', + 'branch': 'master', + }, + 'packages': { + 'root_package': { + 'version': '', + 'modules': ['test1', 'test2'] + } + } +} + +SOURCE_2 = { + 'name': 'source_2', + 'details': { + 'repo_raw_address': 'https://github.com/homholueng/plugins_example_2', + 'branch': 'master', + }, + 'packages': { + 'root_package': { + 'version': '', + 'modules': ['test3', 'test4'] + } + } +} + +SOURCE_4 = { + 'name': 'source_4', + 'details': { + 'repo_raw_address': 'https://github.com/homholueng/plugins_example_4', + 'branch': 'master', + }, + 'packages': { + 'root_package': { + 'version': '', + 'modules': ['test5', 'test6'] + } + } +} + +GIT_SOURCE_CONFIGS = [SOURCE_1, SOURCE_2, SOURCE_4] + + +class ExternalPackageSourceTestCase(TestCase): + + def setUp(self): + GitRepoSource.objects.create_source(name=SOURCE_NAME, + packages=PACKAGES, + from_config=FROM_CONFIG, + repo_raw_address=REPO_RAW_ADDRESS, + branch=BRANCH) + + def tearDown(self): + GitRepoSource.objects.all().delete() + + def test_create_source(self): + source_1 = GitRepoSource.objects.get(name=SOURCE_NAME) + self.assertEqual(source_1.name, SOURCE_NAME) + self.assertEqual(source_1.packages, PACKAGES) + self.assertEqual(source_1.from_config, FROM_CONFIG) + self.assertEqual(source_1.repo_raw_address, REPO_RAW_ADDRESS) + self.assertEqual(source_1.branch, BRANCH) + + def test_remove_source(self): + source_1 = GitRepoSource.objects.get(name=SOURCE_NAME) + + self.assertRaises(exceptions.InvalidOperationException, GitRepoSource.objects.remove_source, source_1.id) + + source_1.from_config = False + source_1.save() + + GitRepoSource.objects.remove_source(source_1.id) + + self.assertFalse(GitRepoSource.objects.filter(id=source_1.id).exists()) + + def _assert_source_equals_config(self, source, config): + self.assertEqual(source.name, config['name']) + self.assertEqual(source.packages, config['packages']) + self.assertEqual(source.repo_raw_address, config['details']['repo_raw_address']) + self.assertEqual(source.branch, config['details']['branch']) + + def test_update_source_from_config(self): + GitRepoSource.objects.all().delete() + + for source in [OLD_SOURCE_1, OLD_SOURCE_3]: + GitRepoSource.objects.create_source(name=source['name'], + packages=source['packages'], + from_config=True, + repo_raw_address=source['details']['repo_raw_address'], + branch=source['details']['branch']) + + GitRepoSource.objects.update_source_from_config(GIT_SOURCE_CONFIGS) + + self.assertFalse(GitRepoSource.objects.filter(name=OLD_SOURCE_3['name']).exists()) + + for config in GIT_SOURCE_CONFIGS: + source = GitRepoSource.objects.get(name=config['name']) + self.assertTrue(source.from_config) + self._assert_source_equals_config(source, config) diff --git a/pipeline/contrib/external_plugins/tests/models/test_base.py b/pipeline/contrib/external_plugins/tests/models/test_base.py index 0900da3c50..6e46c2a34d 100644 --- a/pipeline/contrib/external_plugins/tests/models/test_base.py +++ b/pipeline/contrib/external_plugins/tests/models/test_base.py @@ -95,7 +95,7 @@ GIT_SOURCE_CONFIGS = [SOURCE_1, SOURCE_2, SOURCE_4] -class ModelsBaseTestCase(TestCase): +class ExternalPackageSourceTestCase(TestCase): def setUp(self): GitRepoSource.objects.create_source(name=SOURCE_NAME, diff --git a/pipeline/utils/importer/__init__.py b/pipeline/utils/importer/__init__.py index 90524bb0e7..c4afa9b00e 100644 --- a/pipeline/utils/importer/__init__.py +++ b/pipeline/utils/importer/__init__.py @@ -10,3 +10,6 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ + +from pipeline.utils.importer.utils import importer_context # noqa +from pipeline.utils.importer.git import GitRepoModuleImporter # noqa diff --git a/pipeline/utils/importer/utils.py b/pipeline/utils/importer/utils.py new file mode 100644 index 0000000000..5a439e3b7e --- /dev/null +++ b/pipeline/utils/importer/utils.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import sys +from contextlib import contextmanager + + +@contextmanager +def importer_context(importer): + _setup_importer(importer) + yield + _remove_importer(importer) + + +def _setup_importer(importer): + sys.meta_path.insert(0, importer) + + +def _remove_importer(importer): + for hooked_importer in sys.meta_path: + if hooked_importer is importer: + sys.meta_path.remove(hooked_importer) + return From e0473dedb6fd7ae2b14fe0fab8834d3bfed49676 Mon Sep 17 00:00:00 2001 From: homholueng Date: Wed, 10 Apr 2019 20:40:49 +0800 Subject: [PATCH 08/29] =?UTF-8?q?minor:=20=E6=B7=BB=E5=8A=A0=E5=8A=A0?= =?UTF-8?q?=E8=BD=BD=E8=BF=9C=E7=A8=8B=E6=8F=92=E4=BB=B6=20APP?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/default.py | 1 + 1 file changed, 1 insertion(+) diff --git a/config/default.py b/config/default.py index cbfce85c0a..8cf457ebe5 100644 --- a/config/default.py +++ b/config/default.py @@ -62,6 +62,7 @@ 'pipeline.log', 'pipeline.contrib.statistics', 'pipeline.contrib.periodic_task', + 'pipeline.contrib.external_plugins', 'django_signal_valve', 'pipeline_plugins', 'pipeline_plugins.components', From bd8c079b924d9b1729e658b4a990e44c23440a4e Mon Sep 17 00:00:00 2001 From: homholueng Date: Thu, 11 Apr 2019 10:18:09 +0800 Subject: [PATCH 09/29] =?UTF-8?q?feature:=20pipeline.utils.importer=20?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=AF=BC=E5=85=A5=E5=99=A8=E4=B8=8A=E4=B8=8B?= =?UTF-8?q?=E6=96=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pipeline/tests/mock_settings.py | 3 + .../utils/importer/{base.py => test_base.py} | 0 .../utils/importer/{git.py => test_git.py} | 0 pipeline/tests/utils/importer/test_utils.py | 74 +++++++++++++++++++ pipeline/utils.py | 62 ---------------- pipeline/utils/importer/utils.py | 8 +- 6 files changed, 83 insertions(+), 64 deletions(-) rename pipeline/tests/utils/importer/{base.py => test_base.py} (100%) rename pipeline/tests/utils/importer/{git.py => test_git.py} (100%) create mode 100644 pipeline/tests/utils/importer/test_utils.py delete mode 100644 pipeline/utils.py diff --git a/pipeline/tests/mock_settings.py b/pipeline/tests/mock_settings.py index d9d68760dd..4aa97d534e 100644 --- a/pipeline/tests/mock_settings.py +++ b/pipeline/tests/mock_settings.py @@ -12,6 +12,7 @@ """ SYS_MODULES = 'sys.modules' +SYS_META_PATH = 'sys.meta_path' IMP_ACQUIRE_LOCK = 'imp.acquire_lock' IMP_RELEASE_LOCK = 'imp.release_lock' REQUESTS_GET = 'requests.get' @@ -127,3 +128,5 @@ UTILS_IMPORTER_GIT_GET_SOURCE = 'pipeline.utils.importer.git.GitRepoModuleImporter.get_source' UTILS_IMPORTER_GIT_GET_FILE = 'pipeline.utils.importer.git.GitRepoModuleImporter.get_file' UTILS_IMPORTER_GIT_IS_PACKAGE = 'pipeline.utils.importer.git.GitRepoModuleImporter.is_package' +UTILS_IMPORTER__SETUP_IMPORTER = 'pipeline.utils.importer.utils._setup_importer' +UTILS_IMPORTER__REMOVE_IMPORTER = 'pipeline.utils.importer.utils._remove_importer' diff --git a/pipeline/tests/utils/importer/base.py b/pipeline/tests/utils/importer/test_base.py similarity index 100% rename from pipeline/tests/utils/importer/base.py rename to pipeline/tests/utils/importer/test_base.py diff --git a/pipeline/tests/utils/importer/git.py b/pipeline/tests/utils/importer/test_git.py similarity index 100% rename from pipeline/tests/utils/importer/git.py rename to pipeline/tests/utils/importer/test_git.py diff --git a/pipeline/tests/utils/importer/test_utils.py b/pipeline/tests/utils/importer/test_utils.py new file mode 100644 index 0000000000..fcec52fc8b --- /dev/null +++ b/pipeline/tests/utils/importer/test_utils.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import sys + +from django.test import TestCase + +from pipeline.utils.importer import utils +from pipeline.utils.importer import GitRepoModuleImporter + +from pipeline.tests.mock import * # noqa +from pipeline.tests.mock_settings import * # noqa + + +class UtilsTestCase(TestCase): + + @patch(SYS_META_PATH, []) + def test__set_up_importer(self): + utils._setup_importer('1') + utils._setup_importer('2') + + self.assertEqual(sys.meta_path, ['2', '1']) + + def test__remove_importer(self): + importer_1 = GitRepoModuleImporter(modules=['module_1'], repo_raw_url='url_1', branch='master') + importer_2 = GitRepoModuleImporter(modules=['module_2'], repo_raw_url='url_2', branch='master') + importer_3 = GitRepoModuleImporter(modules=['module_3'], repo_raw_url='url_3', branch='master') + importer_4 = GitRepoModuleImporter(modules=['module_4'], repo_raw_url='url_4', branch='master') + + with patch(SYS_META_PATH, [importer_1, importer_2, importer_3]): + utils._remove_importer(importer_4) + self.assertEqual(sys.meta_path, [importer_1, importer_2, importer_3]) + utils._remove_importer(importer_1) + self.assertEqual(sys.meta_path, [importer_2, importer_3]) + utils._remove_importer(importer_3) + self.assertEqual(sys.meta_path, [importer_2]) + utils._remove_importer(importer_2) + self.assertEqual(sys.meta_path, []) + + @patch(UTILS_IMPORTER__SETUP_IMPORTER, MagicMock()) + @patch(UTILS_IMPORTER__REMOVE_IMPORTER, MagicMock()) + def test_importer_context__normal(self): + importer = 'importer' + with utils.importer_context(importer): + pass + utils._setup_importer.assert_called_once_with(importer) + utils._remove_importer.assert_called_once_with(importer) + + @patch(UTILS_IMPORTER__SETUP_IMPORTER, MagicMock()) + @patch(UTILS_IMPORTER__REMOVE_IMPORTER, MagicMock()) + def test_importer_context__raise_exception(self): + importer = 'importer' + + class CustomException(Exception): + pass + + try: + with utils.importer_context(importer): + raise CustomException() + except CustomException: + pass + + utils._setup_importer.assert_called_once_with(importer) + utils._remove_importer.assert_called_once_with(importer) diff --git a/pipeline/utils.py b/pipeline/utils.py deleted file mode 100644 index eb588b724a..0000000000 --- a/pipeline/utils.py +++ /dev/null @@ -1,62 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community -Edition) available. -Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. -Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. -You may obtain a copy of the License at -http://opensource.org/licenses/MIT -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on -an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. -""" - -from __future__ import absolute_import - - -ITERATED = 1 -NEW = 0 -ITERATING = -1 - - -def has_circle(graph): - # init marks - marks = {} - for node in graph: - # marks as not iterated - marks[node] = NEW - - # dfs every node - for cur_node in graph: - trace = [cur_node] - for node in graph[cur_node]: - if marks[node] == ITERATED: - continue - trace.append(node) - # return immediately when circle be detected - if _has_circle(graph, node, marks, trace): - return True, trace - trace.pop() - # mark as iterated - marks[cur_node] = ITERATED - - return False, [] - - -def _has_circle(graph, cur_node, marks, trace): - # detect circle when iterate to a node which been marked as -1 - if marks[cur_node] == ITERATING: - return True - # mark as iterating - marks[cur_node] = ITERATING - # dfs - for node in graph[cur_node]: - # return immediately when circle be detected - trace.append(node) - if _has_circle(graph, node, marks, trace): - return True - trace.pop() - # mark as iterated - marks[cur_node] = ITERATED - - return False diff --git a/pipeline/utils/importer/utils.py b/pipeline/utils/importer/utils.py index 5a439e3b7e..de2b8b2f80 100644 --- a/pipeline/utils/importer/utils.py +++ b/pipeline/utils/importer/utils.py @@ -18,8 +18,12 @@ @contextmanager def importer_context(importer): _setup_importer(importer) - yield - _remove_importer(importer) + try: + yield + except Exception as e: + raise e + finally: + _remove_importer(importer) def _setup_importer(importer): From 430cc3c13e01f9b00ed151e316175ca3fffeeab5 Mon Sep 17 00:00:00 2001 From: homholueng Date: Thu, 11 Apr 2019 10:19:56 +0800 Subject: [PATCH 10/29] =?UTF-8?q?minors:=20=E5=AE=8C=E5=96=84=20external?= =?UTF-8?q?=5Fplugins=20app=20=E7=9A=84=E5=8D=95=E5=85=83=E6=B5=8B?= =?UTF-8?q?=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pipeline/contrib/external_plugins/apps.py | 7 +- .../contrib/external_plugins/models/base.py | 2 +- .../contrib/external_plugins/tests/mock.py | 37 +++++ .../external_plugins/tests/mock_settings.py | 19 +++ .../tests/models/base/__init__.py | 12 ++ .../tests/models/base/test_base.py | 36 +++++ .../base/test_external_package_source.py | 24 ++- .../tests/models/test_base.py | 153 ------------------ .../external_plugins/tests/test_loader.py | 50 ++++++ 9 files changed, 182 insertions(+), 158 deletions(-) create mode 100644 pipeline/contrib/external_plugins/tests/mock.py create mode 100644 pipeline/contrib/external_plugins/tests/mock_settings.py create mode 100644 pipeline/contrib/external_plugins/tests/models/base/test_base.py delete mode 100644 pipeline/contrib/external_plugins/tests/models/test_base.py create mode 100644 pipeline/contrib/external_plugins/tests/test_loader.py diff --git a/pipeline/contrib/external_plugins/apps.py b/pipeline/contrib/external_plugins/apps.py index d91dc055de..45c6ca9c11 100644 --- a/pipeline/contrib/external_plugins/apps.py +++ b/pipeline/contrib/external_plugins/apps.py @@ -15,14 +15,15 @@ from django.db.utils import ProgrammingError from pipeline.conf import settings -from pipeline.contrib.external_plugins import loader -from pipeline.contrib.external_plugins.models import ExternalPackageSource class ExternalPluginsConfig(AppConfig): - name = 'external_plugins' + name = 'pipeline.contrib.external_plugins' def ready(self): + from pipeline.contrib.external_plugins import loader # noqa + from pipeline.contrib.external_plugins.models import ExternalPackageSource # noqa + try: ExternalPackageSource.update_package_source_from_config(getattr(settings, 'COMPONENTS_PACKAGE_SOURCES', {})) except ProgrammingError: diff --git a/pipeline/contrib/external_plugins/models/base.py b/pipeline/contrib/external_plugins/models/base.py index 5877432bbd..b7be748cfe 100644 --- a/pipeline/contrib/external_plugins/models/base.py +++ b/pipeline/contrib/external_plugins/models/base.py @@ -104,7 +104,7 @@ def importer(self): def modules(self): modules = [] - for _, package_info in self.packages: + for _, package_info in self.packages.items(): modules.extend(package_info['modules']) return modules diff --git a/pipeline/contrib/external_plugins/tests/mock.py b/pipeline/contrib/external_plugins/tests/mock.py new file mode 100644 index 0000000000..126429889f --- /dev/null +++ b/pipeline/contrib/external_plugins/tests/mock.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from __future__ import absolute_import + +from mock import MagicMock, patch, call # noqa + + +class Object(object): + pass + + +class MockPackageSourceManager(object): + def __init__(self, **kwargs): + self.all = MagicMock(return_value=kwargs.get('all')) + + +class MockPackageSourceClass(object): + def __init__(self, **kwargs): + self.objects = MockPackageSourceManager(all=kwargs.get('all')) + + +class MockPackageSource(object): + def __init__(self, **kwargs): + self.type = MagicMock(return_value=kwargs.get('type')) + self.importer = MagicMock(return_value=kwargs.get('importer')) + self.modules = kwargs.get('modules', []) diff --git a/pipeline/contrib/external_plugins/tests/mock_settings.py b/pipeline/contrib/external_plugins/tests/mock_settings.py new file mode 100644 index 0000000000..54bb396c24 --- /dev/null +++ b/pipeline/contrib/external_plugins/tests/mock_settings.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +IMPORTLIB_IMPORT_MODULE = 'importlib.import_module' + +MODELS_BASE_SOURCE_CLS_FACTORY = 'pipeline.contrib.external_plugins.models.base.source_cls_factory' + +LOADER_SOURCE_CLS_FACTORY = 'pipeline.contrib.external_plugins.loader.source_cls_factory' +LOADER__IMPORT_MODULES_IN_SOURCE = 'pipeline.contrib.external_plugins.loader._import_modules_in_source' diff --git a/pipeline/contrib/external_plugins/tests/models/base/__init__.py b/pipeline/contrib/external_plugins/tests/models/base/__init__.py index e69de29bb2..90524bb0e7 100644 --- a/pipeline/contrib/external_plugins/tests/models/base/__init__.py +++ b/pipeline/contrib/external_plugins/tests/models/base/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/pipeline/contrib/external_plugins/tests/models/base/test_base.py b/pipeline/contrib/external_plugins/tests/models/base/test_base.py new file mode 100644 index 0000000000..f797396402 --- /dev/null +++ b/pipeline/contrib/external_plugins/tests/models/base/test_base.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.test import TestCase + +from pipeline.contrib.external_plugins.tests.mock import * # noqa +from pipeline.contrib.external_plugins.tests.mock_settings import * # noqa +from pipeline.contrib.external_plugins.models import base + + +class BaseModuleTestCase(TestCase): + + def test_package_source(self): + source_type = 'source_type' + + cls_factory = {} + + with patch(MODELS_BASE_SOURCE_CLS_FACTORY, cls_factory): + @base.package_source + class APackageSource(object): + + @staticmethod + def type(): + return source_type + + self.assertIs(cls_factory[source_type], APackageSource) diff --git a/pipeline/contrib/external_plugins/tests/models/base/test_external_package_source.py b/pipeline/contrib/external_plugins/tests/models/base/test_external_package_source.py index 6e46c2a34d..3d910b5b98 100644 --- a/pipeline/contrib/external_plugins/tests/models/base/test_external_package_source.py +++ b/pipeline/contrib/external_plugins/tests/models/base/test_external_package_source.py @@ -17,7 +17,20 @@ from pipeline.contrib.external_plugins.models import GitRepoSource SOURCE_NAME = 'source_name' -PACKAGES = [{'1': 1}, {'2': 2}] +PACKAGES = { + 'root_package_1': { + 'version': '', + 'modules': ['test1', 'test2'] + }, + 'root_package_2': { + 'version': '', + 'modules': ['test3', 'test4'] + }, + 'root_package_3': { + 'version': '', + 'modules': ['test5', 'test6'] + } +} FROM_CONFIG = True REPO_RAW_ADDRESS = 'REPO_RAW_ADDRESS' BRANCH = 'master' @@ -151,3 +164,12 @@ def test_update_source_from_config(self): source = GitRepoSource.objects.get(name=config['name']) self.assertTrue(source.from_config) self._assert_source_equals_config(source, config) + + def test_modules(self): + source = GitRepoSource.objects.get(name=SOURCE_NAME) + + modules = [] + for _, package_info in PACKAGES.items(): + modules.extend(package_info['modules']) + + self.assertEqual(source.modules, modules) diff --git a/pipeline/contrib/external_plugins/tests/models/test_base.py b/pipeline/contrib/external_plugins/tests/models/test_base.py deleted file mode 100644 index 6e46c2a34d..0000000000 --- a/pipeline/contrib/external_plugins/tests/models/test_base.py +++ /dev/null @@ -1,153 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community -Edition) available. -Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. -Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. -You may obtain a copy of the License at -http://opensource.org/licenses/MIT -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on -an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. -""" - -from django.test import TestCase - -from pipeline.contrib.external_plugins import exceptions -from pipeline.contrib.external_plugins.models import GitRepoSource - -SOURCE_NAME = 'source_name' -PACKAGES = [{'1': 1}, {'2': 2}] -FROM_CONFIG = True -REPO_RAW_ADDRESS = 'REPO_RAW_ADDRESS' -BRANCH = 'master' - -OLD_SOURCE_1 = { - 'name': 'source_1', - 'details': { - 'repo_raw_address': 'old_address', - 'branch': 'stage', - }, - 'packages': { - 'root_package': { - 'version': '', - 'modules': ['test1'] - } - } -} - -OLD_SOURCE_3 = { - 'name': 'source_3', - 'details': { - 'repo_raw_address': 'old_address_3', - 'branch': 'master', - }, - 'packages': { - 'root_package': { - 'version': '', - 'modules': ['test5'] - } - } -} - -SOURCE_1 = { - 'name': 'source_1', - 'details': { - 'repo_raw_address': 'https://github.com/homholueng/plugins_example_1', - 'branch': 'master', - }, - 'packages': { - 'root_package': { - 'version': '', - 'modules': ['test1', 'test2'] - } - } -} - -SOURCE_2 = { - 'name': 'source_2', - 'details': { - 'repo_raw_address': 'https://github.com/homholueng/plugins_example_2', - 'branch': 'master', - }, - 'packages': { - 'root_package': { - 'version': '', - 'modules': ['test3', 'test4'] - } - } -} - -SOURCE_4 = { - 'name': 'source_4', - 'details': { - 'repo_raw_address': 'https://github.com/homholueng/plugins_example_4', - 'branch': 'master', - }, - 'packages': { - 'root_package': { - 'version': '', - 'modules': ['test5', 'test6'] - } - } -} - -GIT_SOURCE_CONFIGS = [SOURCE_1, SOURCE_2, SOURCE_4] - - -class ExternalPackageSourceTestCase(TestCase): - - def setUp(self): - GitRepoSource.objects.create_source(name=SOURCE_NAME, - packages=PACKAGES, - from_config=FROM_CONFIG, - repo_raw_address=REPO_RAW_ADDRESS, - branch=BRANCH) - - def tearDown(self): - GitRepoSource.objects.all().delete() - - def test_create_source(self): - source_1 = GitRepoSource.objects.get(name=SOURCE_NAME) - self.assertEqual(source_1.name, SOURCE_NAME) - self.assertEqual(source_1.packages, PACKAGES) - self.assertEqual(source_1.from_config, FROM_CONFIG) - self.assertEqual(source_1.repo_raw_address, REPO_RAW_ADDRESS) - self.assertEqual(source_1.branch, BRANCH) - - def test_remove_source(self): - source_1 = GitRepoSource.objects.get(name=SOURCE_NAME) - - self.assertRaises(exceptions.InvalidOperationException, GitRepoSource.objects.remove_source, source_1.id) - - source_1.from_config = False - source_1.save() - - GitRepoSource.objects.remove_source(source_1.id) - - self.assertFalse(GitRepoSource.objects.filter(id=source_1.id).exists()) - - def _assert_source_equals_config(self, source, config): - self.assertEqual(source.name, config['name']) - self.assertEqual(source.packages, config['packages']) - self.assertEqual(source.repo_raw_address, config['details']['repo_raw_address']) - self.assertEqual(source.branch, config['details']['branch']) - - def test_update_source_from_config(self): - GitRepoSource.objects.all().delete() - - for source in [OLD_SOURCE_1, OLD_SOURCE_3]: - GitRepoSource.objects.create_source(name=source['name'], - packages=source['packages'], - from_config=True, - repo_raw_address=source['details']['repo_raw_address'], - branch=source['details']['branch']) - - GitRepoSource.objects.update_source_from_config(GIT_SOURCE_CONFIGS) - - self.assertFalse(GitRepoSource.objects.filter(name=OLD_SOURCE_3['name']).exists()) - - for config in GIT_SOURCE_CONFIGS: - source = GitRepoSource.objects.get(name=config['name']) - self.assertTrue(source.from_config) - self._assert_source_equals_config(source, config) diff --git a/pipeline/contrib/external_plugins/tests/test_loader.py b/pipeline/contrib/external_plugins/tests/test_loader.py new file mode 100644 index 0000000000..d9c29e3dca --- /dev/null +++ b/pipeline/contrib/external_plugins/tests/test_loader.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.test import TestCase + +from pipeline.contrib.external_plugins import loader +from pipeline.contrib.external_plugins.tests.mock import * # noqa +from pipeline.contrib.external_plugins.tests.mock_settings import * # noqa + + +class LoaderTestCase(TestCase): + def test__import_modules_in_source(self): + import_module = MagicMock() + + with patch(IMPORTLIB_IMPORT_MODULE, import_module): + modules = [1, 2, 3, 4] + source = MockPackageSource(importer='importer', modules=modules) + loader._import_modules_in_source(source) + import_module.assert_has_calls(calls=[ + call(modules[0]), + call(modules[1]), + call(modules[2]), + call(modules[3]) + ]) + + @patch(LOADER__IMPORT_MODULES_IN_SOURCE, MagicMock()) + def test_load_external_modules(self): + cls_factory = Object() + setattr(cls_factory, 'items', MagicMock(return_value=[ + ('type_1', MockPackageSourceClass(all=['source_1', 'source_2'])), + ('type_2', MockPackageSourceClass(all=['source_3', 'source_4'])) + ])) + with patch(LOADER_SOURCE_CLS_FACTORY, cls_factory): + loader.load_external_modules() + loader._import_modules_in_source.assert_has_calls(calls=[ + call('source_1'), + call('source_2'), + call('source_3'), + call('source_4'), + ]) From e347cde39de5b7869afa7ee0f0d0154d639533fb Mon Sep 17 00:00:00 2001 From: homholueng Date: Thu, 11 Apr 2019 10:28:02 +0800 Subject: [PATCH 11/29] =?UTF-8?q?feature:=20GitRepoImporter=20=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E5=BC=BA=E5=88=B6=20HTTPS=20=E5=BC=80=E5=85=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pipeline/tests/utils/importer/test_git.py | 11 +++++++++++ pipeline/tests/utils/importer/test_utils.py | 8 ++++---- pipeline/utils/importer/git.py | 8 +++++++- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/pipeline/tests/utils/importer/test_git.py b/pipeline/tests/utils/importer/test_git.py index c5fac7c0f5..95308eb380 100644 --- a/pipeline/tests/utils/importer/test_git.py +++ b/pipeline/tests/utils/importer/test_git.py @@ -40,6 +40,17 @@ def test__init__(self): importer = GitRepoModuleImporter(modules=[], repo_raw_url=self.repo_raw_url_without_slash, branch=self.branch) self.assertEqual(importer.repo_raw_url, self.repo_raw_url) + self.assertRaises(ValueError, + GitRepoModuleImporter, + modules=[], + repo_raw_url='http://repo-addr/', + branch=self.branch) + + GitRepoModuleImporter(modules=[], + repo_raw_url='http://repo-addr/', + branch=self.branch, + secure_only=False) + def test__file_url(self): importer = GitRepoModuleImporter(modules=[], repo_raw_url=self.repo_raw_url, branch=self.branch) self.assertEqual(importer._file_url(self.fullname, is_pkg=True), self.package_url) diff --git a/pipeline/tests/utils/importer/test_utils.py b/pipeline/tests/utils/importer/test_utils.py index fcec52fc8b..0d4473d7e0 100644 --- a/pipeline/tests/utils/importer/test_utils.py +++ b/pipeline/tests/utils/importer/test_utils.py @@ -32,10 +32,10 @@ def test__set_up_importer(self): self.assertEqual(sys.meta_path, ['2', '1']) def test__remove_importer(self): - importer_1 = GitRepoModuleImporter(modules=['module_1'], repo_raw_url='url_1', branch='master') - importer_2 = GitRepoModuleImporter(modules=['module_2'], repo_raw_url='url_2', branch='master') - importer_3 = GitRepoModuleImporter(modules=['module_3'], repo_raw_url='url_3', branch='master') - importer_4 = GitRepoModuleImporter(modules=['module_4'], repo_raw_url='url_4', branch='master') + importer_1 = GitRepoModuleImporter(modules=['module_1'], repo_raw_url='https://url_1', branch='master') + importer_2 = GitRepoModuleImporter(modules=['module_2'], repo_raw_url='https://url_2', branch='master') + importer_3 = GitRepoModuleImporter(modules=['module_3'], repo_raw_url='https://url_3', branch='master') + importer_4 = GitRepoModuleImporter(modules=['module_4'], repo_raw_url='https://url_4', branch='master') with patch(SYS_META_PATH, [importer_1, importer_2, importer_3]): utils._remove_importer(importer_4) diff --git a/pipeline/utils/importer/git.py b/pipeline/utils/importer/git.py index cec7fadd37..99695c096f 100644 --- a/pipeline/utils/importer/git.py +++ b/pipeline/utils/importer/git.py @@ -23,8 +23,14 @@ class GitRepoModuleImporter(NonstandardModuleImporter): - def __init__(self, modules, repo_raw_url, branch, use_cache=True): + def __init__(self, modules, repo_raw_url, branch, use_cache=True, secure_only=True): super(GitRepoModuleImporter, self).__init__(modules=modules) + + if secure_only and not repo_raw_url.startswith('https'): + raise ValueError('Only accept https when secure_only it True.') + elif not secure_only: + logger.warning('Using not secure protocol is extremely dangerous!!') + self.repo_raw_url = repo_raw_url if repo_raw_url.endswith('/') else '%s/' % repo_raw_url self.branch = branch self.use_cache = use_cache From 6bf3464ec7849509ae0ea3b9ec42815486d737d7 Mon Sep 17 00:00:00 2001 From: homholueng Date: Thu, 11 Apr 2019 20:36:54 +0800 Subject: [PATCH 12/29] =?UTF-8?q?minor:=20=E6=B7=BB=E5=8A=A0=E8=BF=9C?= =?UTF-8?q?=E7=A8=8B=E5=8C=85=E6=BA=90=E9=85=8D=E7=BD=AE=E8=8E=B7=E5=8F=96?= =?UTF-8?q?=E8=AF=AD=E5=8F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pipeline/conf/default_settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pipeline/conf/default_settings.py b/pipeline/conf/default_settings.py index f58bfeace2..976135788a 100644 --- a/pipeline/conf/default_settings.py +++ b/pipeline/conf/default_settings.py @@ -51,3 +51,5 @@ PIPELINE_PARSER_CLASS = getattr(settings, 'PIPELINE_PARSER_CLASS', 'pipeline.parser.pipeline_parser.PipelineParser') ENABLE_EXAMPLE_COMPONENTS = getattr(settings, 'ENABLE_EXAMPLE_COMPONENTS', False) + +COMPONENTS_PACKAGE_SOURCES = getattr(settings, 'COMPONENTS_PACKAGE_SOURCES', {}) From da7225681d1ad403a7574d7570518b6f8e208736 Mon Sep 17 00:00:00 2001 From: homholueng Date: Thu, 11 Apr 2019 20:38:14 +0800 Subject: [PATCH 13/29] =?UTF-8?q?minor:=20=E5=8D=95=E5=85=83=E6=B5=8B?= =?UTF-8?q?=E8=AF=95=E5=AE=8C=E5=96=84=EF=BC=8CImporter=20Logger=20?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E4=BF=AE=E6=94=B9=EF=BC=8CGitImporter=20?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E9=85=8D=E7=BD=AE=E4=BB=A3=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/default.py | 8 +++- pipeline/contrib/external_plugins/apps.py | 17 +++++--- .../contrib/external_plugins/models/base.py | 4 +- .../contrib/external_plugins/models/source.py | 4 +- .../external_plugins/tests/mock_settings.py | 2 + .../base/test_external_package_source.py | 40 ++++++++++++++++++- pipeline/utils/importer/base.py | 2 +- pipeline/utils/importer/git.py | 13 ++++-- 8 files changed, 75 insertions(+), 15 deletions(-) diff --git a/config/default.py b/config/default.py index 8cf457ebe5..b69c3cafb7 100644 --- a/config/default.py +++ b/config/default.py @@ -104,7 +104,7 @@ ) # 所有环境的日志级别可以在这里配置 -# LOG_LEVEL = 'INFO' +LOG_LEVEL = 'INFO' # 静态资源文件(js,css等）在APP上线更新后, 由于浏览器有缓存, # 可能会造成没更新的情况. 所以在引用静态资源的地方，都把这个加上 @@ -133,6 +133,12 @@ # load logging settings LOGGING = get_logging_config_dict(locals()) +LOGGING['loggers']['pipeline'] = { + 'handlers': ['root'], + 'level': LOG_LEVEL, + 'propagate': True, +} + # 初始化管理员列表，列表中的人员将拥有预发布环境和正式环境的管理员权限 # 注意：请在首次提测和上线前修改，之后的修改将不会生效 INIT_SUPERUSER = [] diff --git a/pipeline/contrib/external_plugins/apps.py b/pipeline/contrib/external_plugins/apps.py index 45c6ca9c11..e3f20811f4 100644 --- a/pipeline/contrib/external_plugins/apps.py +++ b/pipeline/contrib/external_plugins/apps.py @@ -11,6 +11,8 @@ specific language governing permissions and limitations under the License. """ +import sys + from django.apps import AppConfig from django.db.utils import ProgrammingError @@ -24,10 +26,13 @@ def ready(self): from pipeline.contrib.external_plugins import loader # noqa from pipeline.contrib.external_plugins.models import ExternalPackageSource # noqa - try: - ExternalPackageSource.update_package_source_from_config(getattr(settings, 'COMPONENTS_PACKAGE_SOURCES', {})) - except ProgrammingError: - # first migrate - return + if not sys.argv[1:2] == ['test']: + try: + ExternalPackageSource.update_package_source_from_config(getattr(settings, + 'COMPONENTS_PACKAGE_SOURCES', + {})) + except ProgrammingError: + # first migrate + return - loader.load_external_modules() + loader.load_external_modules() diff --git a/pipeline/contrib/external_plugins/models/base.py b/pipeline/contrib/external_plugins/models/base.py index b7be748cfe..81e2a1fb9c 100644 --- a/pipeline/contrib/external_plugins/models/base.py +++ b/pipeline/contrib/external_plugins/models/base.py @@ -116,6 +116,6 @@ def update_package_source_from_config(source_configs): for config in source_configs: classified_config.setdefault(config.pop('type'), []).append(config) - for source_type, config in classified_config.items(): + for source_type, configs in classified_config.items(): source_model_cls = source_cls_factory[source_type] - source_model_cls.objects.update_source_from_config(config=config) + source_model_cls.objects.update_source_from_config(configs=configs) diff --git a/pipeline/contrib/external_plugins/models/source.py b/pipeline/contrib/external_plugins/models/source.py index 61dcd5e0e6..3ea206de0c 100644 --- a/pipeline/contrib/external_plugins/models/source.py +++ b/pipeline/contrib/external_plugins/models/source.py @@ -12,6 +12,7 @@ """ from django.db import models +from django.conf import settings from django.utils.translation import ugettext_lazy as _ from pipeline.utils.importer.git import GitRepoModuleImporter @@ -36,7 +37,8 @@ def type(): def importer(self): return GitRepoModuleImporter(repo_raw_url=self.repo_raw_address, branch=self.branch, - modules=self.packages.keys()) + modules=self.packages.keys(), + proxy=getattr(settings, 'EXTERNAL_SOURCE_PROXY')) @package_source diff --git a/pipeline/contrib/external_plugins/tests/mock_settings.py b/pipeline/contrib/external_plugins/tests/mock_settings.py index 54bb396c24..8e6cf45650 100644 --- a/pipeline/contrib/external_plugins/tests/mock_settings.py +++ b/pipeline/contrib/external_plugins/tests/mock_settings.py @@ -14,6 +14,8 @@ IMPORTLIB_IMPORT_MODULE = 'importlib.import_module' MODELS_BASE_SOURCE_CLS_FACTORY = 'pipeline.contrib.external_plugins.models.base.source_cls_factory' +MODELS_SOURCE_MANAGER_UPDATE_SOURCE_FROM_CONFIG = \ + 'pipeline.contrib.external_plugins.models.base.SourceManager.update_source_from_config' LOADER_SOURCE_CLS_FACTORY = 'pipeline.contrib.external_plugins.loader.source_cls_factory' LOADER__IMPORT_MODULES_IN_SOURCE = 'pipeline.contrib.external_plugins.loader._import_modules_in_source' diff --git a/pipeline/contrib/external_plugins/tests/models/base/test_external_package_source.py b/pipeline/contrib/external_plugins/tests/models/base/test_external_package_source.py index 3d910b5b98..3392dd72b8 100644 --- a/pipeline/contrib/external_plugins/tests/models/base/test_external_package_source.py +++ b/pipeline/contrib/external_plugins/tests/models/base/test_external_package_source.py @@ -14,7 +14,12 @@ from django.test import TestCase from pipeline.contrib.external_plugins import exceptions -from pipeline.contrib.external_plugins.models import GitRepoSource +from pipeline.contrib.external_plugins.models import ( + source_cls_factory, + GitRepoSource, + ExternalPackageSource) +from pipeline.contrib.external_plugins.tests.mock import * # noqa +from pipeline.contrib.external_plugins.tests.mock_settings import * # noqa SOURCE_NAME = 'source_name' PACKAGES = { @@ -173,3 +178,36 @@ def test_modules(self): modules.extend(package_info['modules']) self.assertEqual(source.modules, modules) + + @patch(MODELS_SOURCE_MANAGER_UPDATE_SOURCE_FROM_CONFIG, MagicMock()) + def test_update_package_source_from_config__empty_configs(self): + ExternalPackageSource.update_package_source_from_config([]) + for source_model_cls in source_cls_factory.values(): + source_model_cls.objects.update_source_from_config.assert_not_called() + + @patch(MODELS_SOURCE_MANAGER_UPDATE_SOURCE_FROM_CONFIG, MagicMock()) + def test_update_package_source_from_config__normal_case(self): + source_configs = [ + { + 'name': '1', + 'type': 'git' + }, + { + 'name': '2', + 'type': 'git' + }, + { + 'name': '3', + 'type': 's3' + }, + { + 'name': '4', + 'type': 'fs' + } + ] + ExternalPackageSource.update_package_source_from_config(source_configs) + GitRepoSource.objects.update_source_from_config.assert_has_calls([ + call(configs=[{'name': '3'}]), + call(configs=[{'name': '1'}, {'name': '2'}]), + call(configs=[{'name': '4'}]) + ]) diff --git a/pipeline/utils/importer/base.py b/pipeline/utils/importer/base.py index 12ee6a158c..2d81361fc6 100644 --- a/pipeline/utils/importer/base.py +++ b/pipeline/utils/importer/base.py @@ -19,7 +19,7 @@ from contextlib import contextmanager from abc import ABCMeta, abstractmethod -logger = logging.getLogger(__name__) +logger = logging.getLogger('root') @contextmanager diff --git a/pipeline/utils/importer/git.py b/pipeline/utils/importer/git.py index 99695c096f..a144abc938 100644 --- a/pipeline/utils/importer/git.py +++ b/pipeline/utils/importer/git.py @@ -18,12 +18,18 @@ from pipeline.utils.importer.base import NonstandardModuleImporter -logger = logging.getLogger(__name__) +logger = logging.getLogger('root') class GitRepoModuleImporter(NonstandardModuleImporter): - def __init__(self, modules, repo_raw_url, branch, use_cache=True, secure_only=True): + def __init__(self, + modules, + repo_raw_url, + branch, + use_cache=True, + secure_only=True, + proxy=None): super(GitRepoModuleImporter, self).__init__(modules=modules) if secure_only and not repo_raw_url.startswith('https'): @@ -35,6 +41,7 @@ def __init__(self, modules, repo_raw_url, branch, use_cache=True, secure_only=Tr self.branch = branch self.use_cache = use_cache self.file_cache = {} + self.proxy = proxy or {} def is_package(self, fullname): return self._fetch_repo_file(self._file_url(fullname, is_pkg=True)) is not None @@ -70,7 +77,7 @@ def _fetch_repo_file(self, file_url): logger.info('Use content in cache for git file: {file_url}'.format(file_url=file_url)) return self.file_cache[file_url] - resp = requests.get(file_url, timeout=10) + resp = requests.get(file_url, timeout=10, proxies=self.proxy) file_content = resp.content if resp.ok else None From 9ca8c23f428d85ff9f96c52d923b8252ebb63b64 Mon Sep 17 00:00:00 2001 From: homholueng Date: Thu, 11 Apr 2019 21:45:56 +0800 Subject: [PATCH 14/29] =?UTF-8?q?minor:=20=E5=85=B3=E9=94=AE=E8=8A=82?= =?UTF-8?q?=E7=82=B9=E6=B7=BB=E5=8A=A0=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pipeline/contrib/external_plugins/apps.py | 11 ++++++++++- pipeline/contrib/external_plugins/loader.py | 10 +++++++++- pipeline/utils/importer/utils.py | 5 +++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/pipeline/contrib/external_plugins/apps.py b/pipeline/contrib/external_plugins/apps.py index e3f20811f4..072ca43cf4 100644 --- a/pipeline/contrib/external_plugins/apps.py +++ b/pipeline/contrib/external_plugins/apps.py @@ -12,12 +12,16 @@ """ import sys +import logging +import traceback from django.apps import AppConfig from django.db.utils import ProgrammingError from pipeline.conf import settings +logger = logging.getLogger('root') + class ExternalPluginsConfig(AppConfig): name = 'pipeline.contrib.external_plugins' @@ -28,11 +32,16 @@ def ready(self): if not sys.argv[1:2] == ['test']: try: + logger.info('Start to update package source from config file...') ExternalPackageSource.update_package_source_from_config(getattr(settings, 'COMPONENTS_PACKAGE_SOURCES', {})) - except ProgrammingError: + except ProgrammingError as e: + logger.warning('update package source failed, maybe first migration? exception: %s' % + traceback.format_exc()) # first migrate return + logger.info('Start to load external modules...') + loader.load_external_modules() diff --git a/pipeline/contrib/external_plugins/loader.py b/pipeline/contrib/external_plugins/loader.py index 67eb0328f4..4d718f0a3d 100644 --- a/pipeline/contrib/external_plugins/loader.py +++ b/pipeline/contrib/external_plugins/loader.py @@ -11,11 +11,15 @@ specific language governing permissions and limitations under the License. """ +import logging import importlib +import traceback from pipeline.utils.importer import importer_context from pipeline.contrib.external_plugins.models import source_cls_factory +logger = logging.getLogger('root') + def load_external_modules(): for source_type, source_model_cls in source_cls_factory.items(): @@ -32,4 +36,8 @@ def _import_modules_in_source(source): with importer_context(importer): for module in source.modules: - importlib.import_module(module) + try: + importlib.import_module(module) + except Exception as e: + logger.error('An error occurred when loading {%s}: %s' % (module, traceback.format_exc())) + raise e diff --git a/pipeline/utils/importer/utils.py b/pipeline/utils/importer/utils.py index de2b8b2f80..b468d1c2a2 100644 --- a/pipeline/utils/importer/utils.py +++ b/pipeline/utils/importer/utils.py @@ -12,8 +12,11 @@ """ import sys +import logging from contextlib import contextmanager +logger = logging.getLogger('root') + @contextmanager def importer_context(importer): @@ -27,11 +30,13 @@ def importer_context(importer): def _setup_importer(importer): + logger.info('========== setup importer: %s' % importer) sys.meta_path.insert(0, importer) def _remove_importer(importer): for hooked_importer in sys.meta_path: if hooked_importer is importer: + logger.info('========== remove importer: %s' % importer) sys.meta_path.remove(hooked_importer) return From c2a10d79ef1a94c24ed9d7dce2bbfff664164d16 Mon Sep 17 00:00:00 2001 From: homholueng Date: Thu, 11 Apr 2019 22:41:58 +0800 Subject: [PATCH 15/29] =?UTF-8?q?improvement:=20=E5=B0=86=20importer=20?= =?UTF-8?q?=E5=B7=A5=E5=85=B7=E7=B1=BB=E7=A7=BB=E5=8A=A8=E5=88=B0=20extern?= =?UTF-8?q?al=5Fplugins=20app=20=E4=B8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pipeline/contrib/external_plugins/apps.py | 6 +++++- pipeline/contrib/external_plugins/loader.py | 2 +- .../contrib/external_plugins/models/base.py | 2 -- .../contrib/external_plugins/models/source.py | 2 +- pipeline/contrib/external_plugins/tests/mock.py | 6 ++++++ .../external_plugins/tests/mock_settings.py | 17 +++++++++++++++++ .../external_plugins/tests/utils}/__init__.py | 0 .../tests}/utils/importer/__init__.py | 3 --- .../tests/utils/importer/test_base.py | 6 +++--- .../tests/utils/importer/test_git.py | 6 +++--- .../tests/utils/importer/test_utils.py | 8 ++++---- .../contrib/external_plugins/utils/__init__.py | 12 ++++++++++++ .../external_plugins/utils/importer/__init__.py | 15 +++++++++++++++ .../external_plugins}/utils/importer/base.py | 0 .../external_plugins}/utils/importer/git.py | 2 +- .../external_plugins}/utils/importer/utils.py | 0 pipeline/tests/mock.py | 6 ------ pipeline/tests/mock_settings.py | 15 --------------- 18 files changed, 68 insertions(+), 40 deletions(-) rename pipeline/{tests/utils/importer => contrib/external_plugins/tests/utils}/__init__.py (100%) rename pipeline/{ => contrib/external_plugins/tests}/utils/importer/__init__.py (84%) rename pipeline/{ => contrib/external_plugins}/tests/utils/importer/test_base.py (97%) rename pipeline/{ => contrib/external_plugins}/tests/utils/importer/test_git.py (97%) rename pipeline/{ => contrib/external_plugins}/tests/utils/importer/test_utils.py (90%) create mode 100644 pipeline/contrib/external_plugins/utils/__init__.py create mode 100644 pipeline/contrib/external_plugins/utils/importer/__init__.py rename pipeline/{ => contrib/external_plugins}/utils/importer/base.py (100%) rename pipeline/{ => contrib/external_plugins}/utils/importer/git.py (97%) rename pipeline/{ => contrib/external_plugins}/utils/importer/utils.py (100%) diff --git a/pipeline/contrib/external_plugins/apps.py b/pipeline/contrib/external_plugins/apps.py index 072ca43cf4..5e662c29eb 100644 --- a/pipeline/contrib/external_plugins/apps.py +++ b/pipeline/contrib/external_plugins/apps.py @@ -16,6 +16,7 @@ import traceback from django.apps import AppConfig +from django.conf import settings from django.db.utils import ProgrammingError from pipeline.conf import settings @@ -30,7 +31,10 @@ def ready(self): from pipeline.contrib.external_plugins import loader # noqa from pipeline.contrib.external_plugins.models import ExternalPackageSource # noqa - if not sys.argv[1:2] == ['test']: + triggers = getattr(settings, 'EXTERNAL_COMPONENTS_LOAD_TRIGGER', {'runserver', 'celery', 'worker'}) + command = sys.argv[1] + + if command in triggers: try: logger.info('Start to update package source from config file...') ExternalPackageSource.update_package_source_from_config(getattr(settings, diff --git a/pipeline/contrib/external_plugins/loader.py b/pipeline/contrib/external_plugins/loader.py index 4d718f0a3d..6e0aaacb62 100644 --- a/pipeline/contrib/external_plugins/loader.py +++ b/pipeline/contrib/external_plugins/loader.py @@ -15,8 +15,8 @@ import importlib import traceback -from pipeline.utils.importer import importer_context from pipeline.contrib.external_plugins.models import source_cls_factory +from pipeline.contrib.external_plugins.utils.importer import importer_context logger = logging.getLogger('root') diff --git a/pipeline/contrib/external_plugins/models/base.py b/pipeline/contrib/external_plugins/models/base.py index 81e2a1fb9c..002c58b3d4 100644 --- a/pipeline/contrib/external_plugins/models/base.py +++ b/pipeline/contrib/external_plugins/models/base.py @@ -11,14 +11,12 @@ specific language governing permissions and limitations under the License. """ -import importlib from copy import deepcopy from abc import abstractmethod from django.db import models from django.utils.translation import ugettext_lazy as _ -from pipeline.utils.importer.utils import importer_context from pipeline.contrib.external_plugins import exceptions from pipeline.contrib.external_plugins.models.fields import JSONTextField diff --git a/pipeline/contrib/external_plugins/models/source.py b/pipeline/contrib/external_plugins/models/source.py index 3ea206de0c..0d662db0ca 100644 --- a/pipeline/contrib/external_plugins/models/source.py +++ b/pipeline/contrib/external_plugins/models/source.py @@ -15,7 +15,7 @@ from django.conf import settings from django.utils.translation import ugettext_lazy as _ -from pipeline.utils.importer.git import GitRepoModuleImporter +from pipeline.contrib.external_plugins.utils.importer.git import GitRepoModuleImporter from pipeline.contrib.external_plugins.models.base import ( GIT, diff --git a/pipeline/contrib/external_plugins/tests/mock.py b/pipeline/contrib/external_plugins/tests/mock.py index 126429889f..d0f68d870f 100644 --- a/pipeline/contrib/external_plugins/tests/mock.py +++ b/pipeline/contrib/external_plugins/tests/mock.py @@ -20,6 +20,12 @@ class Object(object): pass +class MockResponse(object): + def __init__(self, **kwargs): + self.content = kwargs.get('content') + self.ok = kwargs.get('ok', True) + + class MockPackageSourceManager(object): def __init__(self, **kwargs): self.all = MagicMock(return_value=kwargs.get('all')) diff --git a/pipeline/contrib/external_plugins/tests/mock_settings.py b/pipeline/contrib/external_plugins/tests/mock_settings.py index 8e6cf45650..8aa5aff14c 100644 --- a/pipeline/contrib/external_plugins/tests/mock_settings.py +++ b/pipeline/contrib/external_plugins/tests/mock_settings.py @@ -11,6 +11,12 @@ specific language governing permissions and limitations under the License. """ +SYS_MODULES = 'sys.modules' +SYS_META_PATH = 'sys.meta_path' +IMP_ACQUIRE_LOCK = 'imp.acquire_lock' +IMP_RELEASE_LOCK = 'imp.release_lock' +REQUESTS_GET = 'requests.get' + IMPORTLIB_IMPORT_MODULE = 'importlib.import_module' MODELS_BASE_SOURCE_CLS_FACTORY = 'pipeline.contrib.external_plugins.models.base.source_cls_factory' @@ -19,3 +25,14 @@ LOADER_SOURCE_CLS_FACTORY = 'pipeline.contrib.external_plugins.loader.source_cls_factory' LOADER__IMPORT_MODULES_IN_SOURCE = 'pipeline.contrib.external_plugins.loader._import_modules_in_source' + +UTILS_IMPORTER_BASE_EXECUTE_SRC_CODE = \ + 'pipeline.contrib.external_plugins.utils.importer.base.NonstandardModuleImporter._execute_src_code' +UTILS_IMPORTER_GIT__FETCH_REPO_FILE = \ + 'pipeline.contrib.external_plugins.utils.importer.git.GitRepoModuleImporter._fetch_repo_file' +UTILS_IMPORTER_GIT__FILE_URL = 'pipeline.contrib.external_plugins.utils.importer.git.GitRepoModuleImporter._file_url' +UTILS_IMPORTER_GIT_GET_SOURCE = 'pipeline.contrib.external_plugins.utils.importer.git.GitRepoModuleImporter.get_source' +UTILS_IMPORTER_GIT_GET_FILE = 'pipeline.contrib.external_plugins.utils.importer.git.GitRepoModuleImporter.get_file' +UTILS_IMPORTER_GIT_IS_PACKAGE = 'pipeline.contrib.external_plugins.utils.importer.git.GitRepoModuleImporter.is_package' +UTILS_IMPORTER__SETUP_IMPORTER = 'pipeline.contrib.external_plugins.utils.importer.utils._setup_importer' +UTILS_IMPORTER__REMOVE_IMPORTER = 'pipeline.contrib.external_plugins.utils.importer.utils._remove_importer' diff --git a/pipeline/tests/utils/importer/__init__.py b/pipeline/contrib/external_plugins/tests/utils/__init__.py similarity index 100% rename from pipeline/tests/utils/importer/__init__.py rename to pipeline/contrib/external_plugins/tests/utils/__init__.py diff --git a/pipeline/utils/importer/__init__.py b/pipeline/contrib/external_plugins/tests/utils/importer/__init__.py similarity index 84% rename from pipeline/utils/importer/__init__.py rename to pipeline/contrib/external_plugins/tests/utils/importer/__init__.py index c4afa9b00e..90524bb0e7 100644 --- a/pipeline/utils/importer/__init__.py +++ b/pipeline/contrib/external_plugins/tests/utils/importer/__init__.py @@ -10,6 +10,3 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ - -from pipeline.utils.importer.utils import importer_context # noqa -from pipeline.utils.importer.git import GitRepoModuleImporter # noqa diff --git a/pipeline/tests/utils/importer/test_base.py b/pipeline/contrib/external_plugins/tests/utils/importer/test_base.py similarity index 97% rename from pipeline/tests/utils/importer/test_base.py rename to pipeline/contrib/external_plugins/tests/utils/importer/test_base.py index af170a6289..94f7e20b42 100644 --- a/pipeline/tests/utils/importer/test_base.py +++ b/pipeline/contrib/external_plugins/tests/utils/importer/test_base.py @@ -16,9 +16,9 @@ from django.test import TestCase -from pipeline.tests.mock import * # noqa -from pipeline.tests.mock_settings import * # noqa -from pipeline.utils.importer.base import NonstandardModuleImporter +from pipeline.contrib.external_plugins.tests.mock import * # noqa +from pipeline.contrib.external_plugins.tests.mock_settings import * # noqa +from pipeline.contrib.external_plugins.utils.importer.base import NonstandardModuleImporter class DummyImporter(NonstandardModuleImporter): diff --git a/pipeline/tests/utils/importer/test_git.py b/pipeline/contrib/external_plugins/tests/utils/importer/test_git.py similarity index 97% rename from pipeline/tests/utils/importer/test_git.py rename to pipeline/contrib/external_plugins/tests/utils/importer/test_git.py index 95308eb380..bf7dcba7bb 100644 --- a/pipeline/tests/utils/importer/test_git.py +++ b/pipeline/contrib/external_plugins/tests/utils/importer/test_git.py @@ -13,9 +13,9 @@ from django.test import TestCase -from pipeline.tests.mock import * # noqa -from pipeline.tests.mock_settings import * # noqa -from pipeline.utils.importer.git import GitRepoModuleImporter +from pipeline.contrib.external_plugins.tests.mock import * # noqa +from pipeline.contrib.external_plugins.tests.mock_settings import * # noqa +from pipeline.contrib.external_plugins.utils.importer.git import GitRepoModuleImporter GET_FILE_RETURN = 'GET_FILE_RETURN' GET_SOURCE_RETURN = 'a=1' diff --git a/pipeline/tests/utils/importer/test_utils.py b/pipeline/contrib/external_plugins/tests/utils/importer/test_utils.py similarity index 90% rename from pipeline/tests/utils/importer/test_utils.py rename to pipeline/contrib/external_plugins/tests/utils/importer/test_utils.py index 0d4473d7e0..a921dc1f08 100644 --- a/pipeline/tests/utils/importer/test_utils.py +++ b/pipeline/contrib/external_plugins/tests/utils/importer/test_utils.py @@ -15,11 +15,11 @@ from django.test import TestCase -from pipeline.utils.importer import utils -from pipeline.utils.importer import GitRepoModuleImporter +from pipeline.contrib.external_plugins.utils.importer import utils +from pipeline.contrib.external_plugins.utils.importer import GitRepoModuleImporter -from pipeline.tests.mock import * # noqa -from pipeline.tests.mock_settings import * # noqa +from pipeline.contrib.external_plugins.tests.mock import * # noqa +from pipeline.contrib.external_plugins.tests.mock_settings import * # noqa class UtilsTestCase(TestCase): diff --git a/pipeline/contrib/external_plugins/utils/__init__.py b/pipeline/contrib/external_plugins/utils/__init__.py new file mode 100644 index 0000000000..90524bb0e7 --- /dev/null +++ b/pipeline/contrib/external_plugins/utils/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/pipeline/contrib/external_plugins/utils/importer/__init__.py b/pipeline/contrib/external_plugins/utils/importer/__init__.py new file mode 100644 index 0000000000..6bebac0b21 --- /dev/null +++ b/pipeline/contrib/external_plugins/utils/importer/__init__.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from pipeline.contrib.external_plugins.utils.importer.utils import importer_context # noqa +from pipeline.contrib.external_plugins.utils.importer.git import GitRepoModuleImporter # noqa diff --git a/pipeline/utils/importer/base.py b/pipeline/contrib/external_plugins/utils/importer/base.py similarity index 100% rename from pipeline/utils/importer/base.py rename to pipeline/contrib/external_plugins/utils/importer/base.py diff --git a/pipeline/utils/importer/git.py b/pipeline/contrib/external_plugins/utils/importer/git.py similarity index 97% rename from pipeline/utils/importer/git.py rename to pipeline/contrib/external_plugins/utils/importer/git.py index a144abc938..81305aee18 100644 --- a/pipeline/utils/importer/git.py +++ b/pipeline/contrib/external_plugins/utils/importer/git.py @@ -16,7 +16,7 @@ import urlparse import requests -from pipeline.utils.importer.base import NonstandardModuleImporter +from pipeline.contrib.external_plugins.utils.importer.base import NonstandardModuleImporter logger = logging.getLogger('root') diff --git a/pipeline/utils/importer/utils.py b/pipeline/contrib/external_plugins/utils/importer/utils.py similarity index 100% rename from pipeline/utils/importer/utils.py rename to pipeline/contrib/external_plugins/utils/importer/utils.py diff --git a/pipeline/tests/mock.py b/pipeline/tests/mock.py index 9c59d885bd..55489eebda 100644 --- a/pipeline/tests/mock.py +++ b/pipeline/tests/mock.py @@ -31,12 +31,6 @@ class Object(object): pass -class MockResponse(object): - def __init__(self, **kwargs): - self.content = kwargs.get('content') - self.ok = kwargs.get('ok', True) - - class ContextObject(object): def __init__(self, variables): self.variables = variables diff --git a/pipeline/tests/mock_settings.py b/pipeline/tests/mock_settings.py index 4aa97d534e..b18d85509f 100644 --- a/pipeline/tests/mock_settings.py +++ b/pipeline/tests/mock_settings.py @@ -11,12 +11,6 @@ specific language governing permissions and limitations under the License. """ -SYS_MODULES = 'sys.modules' -SYS_META_PATH = 'sys.meta_path' -IMP_ACQUIRE_LOCK = 'imp.acquire_lock' -IMP_RELEASE_LOCK = 'imp.release_lock' -REQUESTS_GET = 'requests.get' - PIPELINE_CORE_GATEWAY_DEFORMAT = 'pipeline.core.flow.gateway.deformat_constant_key' PIPELINE_CORE_CONSTANT_RESOLVE = 'pipeline.core.data.expression.ConstantTemplate.resolve_data' @@ -121,12 +115,3 @@ EXG_HYDRATE_DATA = 'pipeline.engine.core.handlers.exclusive_gateway.hydrate_data' CPG_HYDRATE_DATA = 'pipeline.engine.core.handlers.conditional_parallel.hydrate_data' - -UTILS_IMPORTER_BASE_EXECUTE_SRC_CODE = 'pipeline.utils.importer.base.NonstandardModuleImporter._execute_src_code' -UTILS_IMPORTER_GIT__FETCH_REPO_FILE = 'pipeline.utils.importer.git.GitRepoModuleImporter._fetch_repo_file' -UTILS_IMPORTER_GIT__FILE_URL = 'pipeline.utils.importer.git.GitRepoModuleImporter._file_url' -UTILS_IMPORTER_GIT_GET_SOURCE = 'pipeline.utils.importer.git.GitRepoModuleImporter.get_source' -UTILS_IMPORTER_GIT_GET_FILE = 'pipeline.utils.importer.git.GitRepoModuleImporter.get_file' -UTILS_IMPORTER_GIT_IS_PACKAGE = 'pipeline.utils.importer.git.GitRepoModuleImporter.is_package' -UTILS_IMPORTER__SETUP_IMPORTER = 'pipeline.utils.importer.utils._setup_importer' -UTILS_IMPORTER__REMOVE_IMPORTER = 'pipeline.utils.importer.utils._remove_importer' From 7fc65894d25d48472ba1a4cd9f934591424e0720 Mon Sep 17 00:00:00 2001 From: homholueng Date: Thu, 11 Apr 2019 23:10:06 +0800 Subject: [PATCH 16/29] minor: flake8 fix --- pipeline/conf/default_settings.py | 2 -- pipeline/contrib/external_plugins/apps.py | 4 +--- pipeline/contrib/external_plugins/models/base.py | 6 ++---- .../tests/models/base/test_external_package_source.py | 2 +- 4 files changed, 4 insertions(+), 10 deletions(-) diff --git a/pipeline/conf/default_settings.py b/pipeline/conf/default_settings.py index 976135788a..f58bfeace2 100644 --- a/pipeline/conf/default_settings.py +++ b/pipeline/conf/default_settings.py @@ -51,5 +51,3 @@ PIPELINE_PARSER_CLASS = getattr(settings, 'PIPELINE_PARSER_CLASS', 'pipeline.parser.pipeline_parser.PipelineParser') ENABLE_EXAMPLE_COMPONENTS = getattr(settings, 'ENABLE_EXAMPLE_COMPONENTS', False) - -COMPONENTS_PACKAGE_SOURCES = getattr(settings, 'COMPONENTS_PACKAGE_SOURCES', {}) diff --git a/pipeline/contrib/external_plugins/apps.py b/pipeline/contrib/external_plugins/apps.py index 5e662c29eb..d9971413aa 100644 --- a/pipeline/contrib/external_plugins/apps.py +++ b/pipeline/contrib/external_plugins/apps.py @@ -19,8 +19,6 @@ from django.conf import settings from django.db.utils import ProgrammingError -from pipeline.conf import settings - logger = logging.getLogger('root') @@ -40,7 +38,7 @@ def ready(self): ExternalPackageSource.update_package_source_from_config(getattr(settings, 'COMPONENTS_PACKAGE_SOURCES', {})) - except ProgrammingError as e: + except ProgrammingError: logger.warning('update package source failed, maybe first migration? exception: %s' % traceback.format_exc()) # first migrate diff --git a/pipeline/contrib/external_plugins/models/base.py b/pipeline/contrib/external_plugins/models/base.py index 002c58b3d4..4666a0351f 100644 --- a/pipeline/contrib/external_plugins/models/base.py +++ b/pipeline/contrib/external_plugins/models/base.py @@ -30,9 +30,7 @@ FILE_SYSTEM }) -source_cls_factory = { - -} +source_cls_factory = {} def package_source(cls): @@ -102,7 +100,7 @@ def importer(self): def modules(self): modules = [] - for _, package_info in self.packages.items(): + for package_info in self.packages.values(): modules.extend(package_info['modules']) return modules diff --git a/pipeline/contrib/external_plugins/tests/models/base/test_external_package_source.py b/pipeline/contrib/external_plugins/tests/models/base/test_external_package_source.py index 3392dd72b8..4594deb3f7 100644 --- a/pipeline/contrib/external_plugins/tests/models/base/test_external_package_source.py +++ b/pipeline/contrib/external_plugins/tests/models/base/test_external_package_source.py @@ -174,7 +174,7 @@ def test_modules(self): source = GitRepoSource.objects.get(name=SOURCE_NAME) modules = [] - for _, package_info in PACKAGES.items(): + for package_info in PACKAGES.values(): modules.extend(package_info['modules']) self.assertEqual(source.modules, modules) From 6840f049ccdb5f0c430897be9c24976e761c6122 Mon Sep 17 00:00:00 2001 From: homholueng Date: Thu, 11 Apr 2019 23:26:53 +0800 Subject: [PATCH 17/29] minor: code review fix --- pipeline/contrib/external_plugins/models/base.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pipeline/contrib/external_plugins/models/base.py b/pipeline/contrib/external_plugins/models/base.py index 4666a0351f..f65bd2f7bc 100644 --- a/pipeline/contrib/external_plugins/models/base.py +++ b/pipeline/contrib/external_plugins/models/base.py @@ -42,9 +42,7 @@ class SourceManager(models.Manager): def create_source(self, name, packages, from_config, **kwargs): create_kwargs = deepcopy(kwargs) - create_kwargs['name'] = name - create_kwargs['packages'] = packages - create_kwargs['from_config'] = from_config + create_kwargs.update({'name': name, 'packages': packages, 'from_config': from_config}) return self.create(**create_kwargs) def remove_source(self, source_id): From b8d0f8fc6f43c3fc0222691c09c3a6d7117559b3 Mon Sep 17 00:00:00 2001 From: homholueng Date: Fri, 12 Apr 2019 17:07:52 +0800 Subject: [PATCH 18/29] =?UTF-8?q?bugfix:=20=E4=BF=AE=E5=A4=8D=E5=88=A0?= =?UTF-8?q?=E9=99=A4=E6=9C=AC=E5=9C=B0=E9=85=8D=E7=BD=AE=E4=B8=AD=E7=9A=84?= =?UTF-8?q?=E5=8C=85=E6=BA=90=E9=85=8D=E7=BD=AE=E6=97=B6=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BA=93=E4=B8=AD=E7=9A=84=E9=85=8D=E7=BD=AE=E6=9C=AA=E5=88=A0?= =?UTF-8?q?=E9=99=A4=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pipeline/contrib/external_plugins/models/base.py | 2 +- .../tests/models/base/test_external_package_source.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipeline/contrib/external_plugins/models/base.py b/pipeline/contrib/external_plugins/models/base.py index f65bd2f7bc..d4f140a7db 100644 --- a/pipeline/contrib/external_plugins/models/base.py +++ b/pipeline/contrib/external_plugins/models/base.py @@ -105,7 +105,7 @@ def modules(self): @staticmethod def update_package_source_from_config(source_configs): - classified_config = {} + classified_config = {source_type: [] for source_type in source_cls_factory.keys()} for config in source_configs: classified_config.setdefault(config.pop('type'), []).append(config) diff --git a/pipeline/contrib/external_plugins/tests/models/base/test_external_package_source.py b/pipeline/contrib/external_plugins/tests/models/base/test_external_package_source.py index 4594deb3f7..947f5768d4 100644 --- a/pipeline/contrib/external_plugins/tests/models/base/test_external_package_source.py +++ b/pipeline/contrib/external_plugins/tests/models/base/test_external_package_source.py @@ -183,7 +183,7 @@ def test_modules(self): def test_update_package_source_from_config__empty_configs(self): ExternalPackageSource.update_package_source_from_config([]) for source_model_cls in source_cls_factory.values(): - source_model_cls.objects.update_source_from_config.assert_not_called() + source_model_cls.objects.update_source_from_config.assert_called_with(configs=[]) @patch(MODELS_SOURCE_MANAGER_UPDATE_SOURCE_FROM_CONFIG, MagicMock()) def test_update_package_source_from_config__normal_case(self): From 710b016d0b8308dd131dcea703097b55d72124b3 Mon Sep 17 00:00:00 2001 From: homholueng Date: Fri, 12 Apr 2019 17:13:30 +0800 Subject: [PATCH 19/29] =?UTF-8?q?improvement:=20=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E9=81=87=E5=88=B0=E4=B8=8D=E6=94=AF=E6=8C=81=E7=9A=84=E8=BF=9C?= =?UTF-8?q?=E7=A8=8B=E6=95=B0=E6=8D=AE=E7=B1=BB=E5=9E=8B=E6=97=B6=E6=8A=9B?= =?UTF-8?q?=E5=87=BA=E7=9A=84=E5=BC=82=E5=B8=B8=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pipeline/contrib/external_plugins/models/base.py | 5 ++++- .../tests/models/base/test_external_package_source.py | 9 +++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/pipeline/contrib/external_plugins/models/base.py b/pipeline/contrib/external_plugins/models/base.py index d4f140a7db..6243446a9c 100644 --- a/pipeline/contrib/external_plugins/models/base.py +++ b/pipeline/contrib/external_plugins/models/base.py @@ -111,5 +111,8 @@ def update_package_source_from_config(source_configs): classified_config.setdefault(config.pop('type'), []).append(config) for source_type, configs in classified_config.items(): - source_model_cls = source_cls_factory[source_type] + try: + source_model_cls = source_cls_factory[source_type] + except KeyError: + raise KeyError('Unsupported external source type: %s' % source_type) source_model_cls.objects.update_source_from_config(configs=configs) diff --git a/pipeline/contrib/external_plugins/tests/models/base/test_external_package_source.py b/pipeline/contrib/external_plugins/tests/models/base/test_external_package_source.py index 947f5768d4..744556da0a 100644 --- a/pipeline/contrib/external_plugins/tests/models/base/test_external_package_source.py +++ b/pipeline/contrib/external_plugins/tests/models/base/test_external_package_source.py @@ -211,3 +211,12 @@ def test_update_package_source_from_config__normal_case(self): call(configs=[{'name': '1'}, {'name': '2'}]), call(configs=[{'name': '4'}]) ]) + + def test_update_package_source_from_config__unsupported_source_type(self): + source_configs = [ + { + 'name': '1', + 'type': 'wrong_type' + } + ] + self.assertRaises(KeyError, ExternalPackageSource.update_package_source_from_config, source_configs) From ae54720d4f1ac84916e29ca81860d491c24b39b3 Mon Sep 17 00:00:00 2001 From: homholueng Date: Mon, 15 Apr 2019 10:13:37 +0800 Subject: [PATCH 20/29] =?UTF-8?q?minor:=20=E5=88=A0=E9=99=A4=E5=A4=9A?= =?UTF-8?q?=E4=BD=99=E7=A9=BA=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pipeline/contrib/external_plugins/models/source.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pipeline/contrib/external_plugins/models/source.py b/pipeline/contrib/external_plugins/models/source.py index 0d662db0ca..3276592f66 100644 --- a/pipeline/contrib/external_plugins/models/source.py +++ b/pipeline/contrib/external_plugins/models/source.py @@ -14,7 +14,6 @@ from django.db import models from django.conf import settings from django.utils.translation import ugettext_lazy as _ - from pipeline.contrib.external_plugins.utils.importer.git import GitRepoModuleImporter from pipeline.contrib.external_plugins.models.base import ( From d6474e1f967db3f3fa53e3914966eb4480b19781 Mon Sep 17 00:00:00 2001 From: homholueng Date: Mon, 15 Apr 2019 10:41:05 +0800 Subject: [PATCH 21/29] =?UTF-8?q?improvement:=20=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E4=BB=8E=E9=85=8D=E7=BD=AE=E4=B8=AD=E5=88=9B=E5=BB=BA=E5=8C=85?= =?UTF-8?q?=E6=BA=90=E5=92=8C=E4=B8=8E=20API=20=E5=88=9B=E5=BB=BA=E7=9A=84?= =?UTF-8?q?=E5=8C=85=E6=BA=90=E5=8C=85=E5=90=8D=E5=86=B2=E7=AA=81=E6=97=B6?= =?UTF-8?q?=E7=9A=84=E6=8F=90=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../contrib/external_plugins/models/base.py | 17 +++++++++++------ .../models/base/test_external_package_source.py | 11 +++++++++++ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/pipeline/contrib/external_plugins/models/base.py b/pipeline/contrib/external_plugins/models/base.py index 6243446a9c..69fd47305c 100644 --- a/pipeline/contrib/external_plugins/models/base.py +++ b/pipeline/contrib/external_plugins/models/base.py @@ -14,7 +14,7 @@ from copy import deepcopy from abc import abstractmethod -from django.db import models +from django.db import models, IntegrityError from django.utils.translation import ugettext_lazy as _ from pipeline.contrib.external_plugins import exceptions @@ -69,10 +69,15 @@ def update_source_from_config(self, configs): defaults = deepcopy(config['details']) defaults['packages'] = config['packages'] - self.update_or_create( - name=config['name'], - from_config=True, - defaults=defaults) + try: + self.update_or_create( + name=config['name'], + from_config=True, + defaults=defaults) + except IntegrityError: + raise exceptions.InvalidOperationException( + 'There is a external source named "{source_name}" but not create from config, ' + 'can not do source update operation'.format(source_name=config['name'])) class ExternalPackageSource(models.Model): @@ -107,7 +112,7 @@ def modules(self): def update_package_source_from_config(source_configs): classified_config = {source_type: [] for source_type in source_cls_factory.keys()} - for config in source_configs: + for config in deepcopy(source_configs): classified_config.setdefault(config.pop('type'), []).append(config) for source_type, configs in classified_config.items(): diff --git a/pipeline/contrib/external_plugins/tests/models/base/test_external_package_source.py b/pipeline/contrib/external_plugins/tests/models/base/test_external_package_source.py index 744556da0a..ad6abe278f 100644 --- a/pipeline/contrib/external_plugins/tests/models/base/test_external_package_source.py +++ b/pipeline/contrib/external_plugins/tests/models/base/test_external_package_source.py @@ -11,6 +11,8 @@ specific language governing permissions and limitations under the License. """ +from copy import deepcopy + from django.test import TestCase from pipeline.contrib.external_plugins import exceptions @@ -220,3 +222,12 @@ def test_update_package_source_from_config__unsupported_source_type(self): } ] self.assertRaises(KeyError, ExternalPackageSource.update_package_source_from_config, source_configs) + + def test_update_source_from_config__name_conflict(self): + source = deepcopy(SOURCE_1) + source['type'] = 'git' + ExternalPackageSource.update_package_source_from_config([source]) + GitRepoSource.objects.filter(name=source['name']).update(from_config=False) + self.assertRaises(exceptions.InvalidOperationException, + ExternalPackageSource.update_package_source_from_config, + [source]) From 55d5e99c2a7a08d9aea923b38cdbd602cb91c89b Mon Sep 17 00:00:00 2001 From: homholueng Date: Tue, 16 Apr 2019 11:43:39 +0800 Subject: [PATCH 22/29] =?UTF-8?q?feature:=20=E6=B7=BB=E5=8A=A0=20S3=20?= =?UTF-8?q?=E5=8F=8A=20FileSystem=20importer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../contrib/external_plugins/tests/mock.py | 6 + .../external_plugins/tests/mock_settings.py | 1 + .../tests/utils/importer/test_fs.py | 44 ++++++++ .../tests/utils/importer/test_git.py | 2 + .../tests/utils/importer/test_s3.py | 94 ++++++++++++++++ .../external_plugins/utils/importer/fs.py | 82 ++++++++++++++ .../external_plugins/utils/importer/s3.py | 103 ++++++++++++++++++ pipeline/requirements.txt | 1 + requirements.txt | 1 + 9 files changed, 334 insertions(+) create mode 100644 pipeline/contrib/external_plugins/tests/utils/importer/test_fs.py create mode 100644 pipeline/contrib/external_plugins/tests/utils/importer/test_s3.py create mode 100644 pipeline/contrib/external_plugins/utils/importer/fs.py create mode 100644 pipeline/contrib/external_plugins/utils/importer/s3.py diff --git a/pipeline/contrib/external_plugins/tests/mock.py b/pipeline/contrib/external_plugins/tests/mock.py index d0f68d870f..b8fabbe617 100644 --- a/pipeline/contrib/external_plugins/tests/mock.py +++ b/pipeline/contrib/external_plugins/tests/mock.py @@ -16,6 +16,12 @@ from mock import MagicMock, patch, call # noqa +def mock_s3_resource(resource, **kwargs): + ret = {'resource': resource} + ret.update(kwargs) + return ret + + class Object(object): pass diff --git a/pipeline/contrib/external_plugins/tests/mock_settings.py b/pipeline/contrib/external_plugins/tests/mock_settings.py index 8aa5aff14c..ffdbf09c17 100644 --- a/pipeline/contrib/external_plugins/tests/mock_settings.py +++ b/pipeline/contrib/external_plugins/tests/mock_settings.py @@ -16,6 +16,7 @@ IMP_ACQUIRE_LOCK = 'imp.acquire_lock' IMP_RELEASE_LOCK = 'imp.release_lock' REQUESTS_GET = 'requests.get' +BOTO3_RESOURCE = 'boto3.resource' IMPORTLIB_IMPORT_MODULE = 'importlib.import_module' diff --git a/pipeline/contrib/external_plugins/tests/utils/importer/test_fs.py b/pipeline/contrib/external_plugins/tests/utils/importer/test_fs.py new file mode 100644 index 0000000000..70b3c1e391 --- /dev/null +++ b/pipeline/contrib/external_plugins/tests/utils/importer/test_fs.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.test import TestCase + +from pipeline.contrib.external_plugins.tests.mock import * # noqa +from pipeline.contrib.external_plugins.tests.mock_settings import * # noqa +from pipeline.contrib.external_plugins.utils.importer.fs import FSModuleImporter + + +class S3ModuleImporterTestCase(TestCase): + def test__init__(self): + pass + + def test_is_package(self): + pass + + def test_get_code(self): + pass + + def test_get_source(self): + pass + + def test_get_path(self): + pass + + def test_get_file(self): + pass + + def test__file_path(self): + pass + + def test__fetch_file(self): + pass diff --git a/pipeline/contrib/external_plugins/tests/utils/importer/test_git.py b/pipeline/contrib/external_plugins/tests/utils/importer/test_git.py index bf7dcba7bb..2cc7362537 100644 --- a/pipeline/contrib/external_plugins/tests/utils/importer/test_git.py +++ b/pipeline/contrib/external_plugins/tests/utils/importer/test_git.py @@ -36,9 +36,11 @@ def setUp(self): def test__init__(self): importer = GitRepoModuleImporter(modules=[], repo_raw_url=self.repo_raw_url, branch=self.branch) self.assertEqual(importer.repo_raw_url, self.repo_raw_url) + self.assertEqual(importer.branch, self.branch) importer = GitRepoModuleImporter(modules=[], repo_raw_url=self.repo_raw_url_without_slash, branch=self.branch) self.assertEqual(importer.repo_raw_url, self.repo_raw_url) + self.assertEqual(importer.branch, self.branch) self.assertRaises(ValueError, GitRepoModuleImporter, diff --git a/pipeline/contrib/external_plugins/tests/utils/importer/test_s3.py b/pipeline/contrib/external_plugins/tests/utils/importer/test_s3.py new file mode 100644 index 0000000000..806f82baf2 --- /dev/null +++ b/pipeline/contrib/external_plugins/tests/utils/importer/test_s3.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import boto3 +from django.test import TestCase + +from pipeline.contrib.external_plugins.tests.mock import * # noqa +from pipeline.contrib.external_plugins.tests.mock_settings import * # noqa +from pipeline.contrib.external_plugins.utils.importer.s3 import S3ModuleImporter + + +class S3ModuleImporterTestCase(TestCase): + + def setUp(self): + self.service_address = 'https://test-s3-address/' + self.service_address_without_slash = 'https://test-s3-address' + self.not_secure_service_address = 'http://no-secure-address/' + self.bucket = 'bucket' + self.access_key = 'access_key' + self.secret_key = 'secret_key' + + @patch(BOTO3_RESOURCE, mock_s3_resource) + def test__init__(self): + importer = S3ModuleImporter(modules=[], + service_address=self.service_address, + bucket=self.bucket, + access_key=self.access_key, + secret_key=self.secret_key) + self.assertEqual(self.service_address, importer.service_address) + self.assertEqual(importer.s3, mock_s3_resource('s3', + aws_access_key_id=self.access_key, + aws_secret_access_key=self.secret_key, + endpoint_url=self.service_address)) + + importer = S3ModuleImporter(modules=[], + service_address=self.service_address_without_slash, + bucket=self.bucket, + access_key=self.access_key, + secret_key=self.secret_key) + self.assertEqual(self.service_address, importer.service_address) + self.assertEqual(importer.s3, mock_s3_resource('s3', + aws_access_key_id=self.access_key, + aws_secret_access_key=self.secret_key, + endpoint_url=self.service_address)) + + self.assertRaises(ValueError, + S3ModuleImporter, + modules=[], + service_address=self.not_secure_service_address, + bucket=self.bucket, + access_key=self.access_key, + secret_key=self.secret_key) + + importer = S3ModuleImporter(modules=[], + service_address=self.not_secure_service_address, + bucket=self.bucket, + access_key=self.access_key, + secret_key=self.secret_key) + self.assertEqual(self.not_secure_service_address, importer.service_address) + self.assertEqual(importer.s3, mock_s3_resource('s3', + aws_access_key_id=self.access_key, + aws_secret_access_key=self.secret_key, + endpoint_url=self.service_address)) + + def test_is_package(self): + pass + + def test_get_code(self): + pass + + def test_get_source(self): + pass + + def test_get_path(self): + pass + + def test_get_file(self): + pass + + def test__obj_key(self): + pass + + def test__fetch_obj_content(self): + pass diff --git a/pipeline/contrib/external_plugins/utils/importer/fs.py b/pipeline/contrib/external_plugins/utils/importer/fs.py new file mode 100644 index 0000000000..17a0e44eb8 --- /dev/null +++ b/pipeline/contrib/external_plugins/utils/importer/fs.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import os +import logging +import traceback + +from pipeline.contrib.external_plugins.utils.importer.base import NonstandardModuleImporter + +logger = logging.getLogger('root') + + +class FSModuleImporter(NonstandardModuleImporter): + def __init__(self, + modules, + path, + use_cache=True): + super(FSModuleImporter, self).__init__(modules=modules) + + self.path = path if path.endswith('/') else '%s/' % path + self.use_cache = use_cache + self.file_cache = {} + + def is_package(self, fullname): + return os.path.exists(self._file_path(fullname, is_pkg=True)) + + def get_code(self, fullname): + return compile(self.get_source(fullname), self.get_file(fullname), 'exec') + + def get_source(self, fullname): + source_code = self._fetch_file(self._file_path(fullname, is_pkg=self.is_package(fullname))) + + if source_code is None: + raise ImportError('Can not find {module} in {path}'.format( + module=fullname, + path=self.path + )) + + return source_code + + def get_path(self, fullname): + return [self._file_path(fullname, is_pkg=True).rpartition('/')[0]] + + def get_file(self, fullname): + return self._file_path(fullname, is_pkg=self.is_package(fullname)) + + def _file_path(self, fullname, is_pkg=False): + base_path = '{path}{file_path}'.format(path=self.path, file_path=fullname.replace('.', '/')) + file_path = '%s/__init__.py' % base_path if is_pkg else '%s.py' % base_path + return file_path + + def _fetch_file(self, file_path): + logger.info('Try to fetch file {file_path}'.format(file_path=file_path)) + + if self.use_cache and file_path in self.file_cache: + logger.info('Use content in cache for file: {file_path}'.format(file_path=file_path)) + return self.file_cache[file_path] + + try: + with open(file_path) as f: + file_content = f.read() + except IOError: + logger.error('Error occurred when read {file_path} content: {trace}'.format( + file_path=file_path, + trace=traceback.format_exc() + )) + return None + + if self.use_cache: + self.file_cache[file_path] = file_content + + return file_content diff --git a/pipeline/contrib/external_plugins/utils/importer/s3.py b/pipeline/contrib/external_plugins/utils/importer/s3.py new file mode 100644 index 0000000000..536892368e --- /dev/null +++ b/pipeline/contrib/external_plugins/utils/importer/s3.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import logging + +import boto3 +from botocore.exceptions import ClientError + +from pipeline.contrib.external_plugins.utils.importer.base import NonstandardModuleImporter + +logger = logging.getLogger('root') + + +class S3ModuleImporter(NonstandardModuleImporter): + def __init__(self, + modules, + service_address, + bucket, + access_key, + secret_key, + use_cache=True, + secure_only=True): + super(S3ModuleImporter, self).__init__(modules=modules) + + if secure_only and not service_address.startswith('https'): + raise ValueError('Only accept https when secure_only it True.') + elif not secure_only: + logger.warning('Using not secure protocol is extremely dangerous!!') + + self.service_address = service_address if service_address.endswith('/') else '%s/' % service_address + self.bucket = bucket + self.use_cache = use_cache + self.s3 = boto3.resource('s3', + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + endpoint_url=self.service_address) + self.obj_cache = {} + + def is_package(self, fullname): + return self._fetch_obj_content(self._obj_key(fullname, is_pkg=True)) is not None + + def get_code(self, fullname): + return compile(self.get_source(fullname), self.get_file(fullname), 'exec') + + def get_source(self, fullname): + source_code = self._fetch_obj_content(self._obj_key(fullname, is_pkg=self.is_package(fullname))) + + if source_code is None: + raise ImportError('Can not find {module} in {service_address}{bucket}'.format( + module=fullname, + service_address=self.service_address, + bucket=self.bucket + )) + + return source_code + + def get_path(self, fullname): + return [self.get_file(fullname).rpartition('/')[0]] + + def get_file(self, fullname): + return '{service_address}{bucket}/{key}'.format( + service_address=self.service_address, + bucket=self.bucket, + key=self._obj_key(fullname, is_pkg=self.is_package(fullname)) + ) + + def _obj_key(self, fullname, is_pkg): + base_key = fullname.replace('.', '/') + key = '%s/__init__.py' % base_key if is_pkg else '%s.py' % base_key + return key + + def _fetch_obj_content(self, key): + logger.info('Try to fetch object: {key}'.format(key=key)) + + if self.use_cache and key in self.obj_cache: + logger.info('Use content in cache for s3 object: {key}'.format(key=key)) + return self.obj_cache[key] + + obj = self.s3.Object(bucket_name=self.bucket, key=key) + + try: + resp = obj.get() + obj_content = resp['Body'].read() + except ClientError as e: + if e.response['Error']['Code'] == 'NoSuchKey': + obj_content = None + else: + raise + + if self.use_cache: + self.obj_cache[key] = obj_content + + return obj_content diff --git a/pipeline/requirements.txt b/pipeline/requirements.txt index 14445da9d4..4e4114d1ec 100644 --- a/pipeline/requirements.txt +++ b/pipeline/requirements.txt @@ -32,3 +32,4 @@ redis-py-cluster==1.3.5 django-timezone-field==3.0 mock==2.0.0 factory_boy==2.11.1 +boto3==1.9.130 diff --git a/requirements.txt b/requirements.txt index 14445da9d4..4e4114d1ec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,3 +32,4 @@ redis-py-cluster==1.3.5 django-timezone-field==3.0 mock==2.0.0 factory_boy==2.11.1 +boto3==1.9.130 From 3899f51e7abbc62d562fce0baba55e3f5e4ce5a2 Mon Sep 17 00:00:00 2001 From: homholueng Date: Tue, 16 Apr 2019 17:50:11 +0800 Subject: [PATCH 23/29] =?UTF-8?q?minor:=20=E6=B7=BB=E5=8A=A0=20S3Importer?= =?UTF-8?q?=20=E5=8F=8A=20FSImporter=20=E7=9A=84=E5=8D=95=E5=85=83?= =?UTF-8?q?=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../external_plugins/tests/mock_settings.py | 17 +++ .../tests/utils/importer/test_fs.py | 110 ++++++++++++-- .../tests/utils/importer/test_git.py | 2 + .../tests/utils/importer/test_s3.py | 142 ++++++++++++++++-- .../external_plugins/utils/importer/fs.py | 17 ++- .../external_plugins/utils/importer/s3.py | 11 +- 6 files changed, 269 insertions(+), 30 deletions(-) diff --git a/pipeline/contrib/external_plugins/tests/mock_settings.py b/pipeline/contrib/external_plugins/tests/mock_settings.py index ffdbf09c17..7b5ae2fab2 100644 --- a/pipeline/contrib/external_plugins/tests/mock_settings.py +++ b/pipeline/contrib/external_plugins/tests/mock_settings.py @@ -17,6 +17,7 @@ IMP_RELEASE_LOCK = 'imp.release_lock' REQUESTS_GET = 'requests.get' BOTO3_RESOURCE = 'boto3.resource' +OS_PATH_EXISTS = 'os.path.exists' IMPORTLIB_IMPORT_MODULE = 'importlib.import_module' @@ -37,3 +38,19 @@ UTILS_IMPORTER_GIT_IS_PACKAGE = 'pipeline.contrib.external_plugins.utils.importer.git.GitRepoModuleImporter.is_package' UTILS_IMPORTER__SETUP_IMPORTER = 'pipeline.contrib.external_plugins.utils.importer.utils._setup_importer' UTILS_IMPORTER__REMOVE_IMPORTER = 'pipeline.contrib.external_plugins.utils.importer.utils._remove_importer' + +UTILS_IMPORTER_S3__FETCH_OBJ_CONTENT = \ + 'pipeline.contrib.external_plugins.utils.importer.s3.S3ModuleImporter._fetch_obj_content' +UTILS_IMPORTER_S3_GET_SOURCE = 'pipeline.contrib.external_plugins.utils.importer.s3.S3ModuleImporter.get_source' +UTILS_IMPORTER_S3_GET_FILE = 'pipeline.contrib.external_plugins.utils.importer.s3.S3ModuleImporter.get_file' +UTILS_IMPORTER_S3_IS_PACKAGE = 'pipeline.contrib.external_plugins.utils.importer.s3.S3ModuleImporter.is_package' +UTILS_IMPORTER_S3__GET_S3_OBJ_CONTENT = \ + 'pipeline.contrib.external_plugins.utils.importer.s3.S3ModuleImporter._get_s3_obj_content' + +UTILS_IMPORTER_FS_GET_SOURCE = 'pipeline.contrib.external_plugins.utils.importer.fs.FSModuleImporter.get_source' +UTILS_IMPORTER_FS_GET_FILE = 'pipeline.contrib.external_plugins.utils.importer.fs.FSModuleImporter.get_file' +UTILS_IMPORTER_FS_IS_PACKAGE = 'pipeline.contrib.external_plugins.utils.importer.fs.FSModuleImporter.is_package' +UTILS_IMPORTER_FS__FETCH_FILE_CONTENT = \ + 'pipeline.contrib.external_plugins.utils.importer.fs.FSModuleImporter._fetch_file_content' +UTILS_IMPORTER_FS__GET_FILE_CONTENT = \ + 'pipeline.contrib.external_plugins.utils.importer.fs.FSModuleImporter._get_file_content' diff --git a/pipeline/contrib/external_plugins/tests/utils/importer/test_fs.py b/pipeline/contrib/external_plugins/tests/utils/importer/test_fs.py index 70b3c1e391..8872c54bd5 100644 --- a/pipeline/contrib/external_plugins/tests/utils/importer/test_fs.py +++ b/pipeline/contrib/external_plugins/tests/utils/importer/test_fs.py @@ -17,28 +17,118 @@ from pipeline.contrib.external_plugins.tests.mock_settings import * # noqa from pipeline.contrib.external_plugins.utils.importer.fs import FSModuleImporter +GET_FILE_RETURN = 'GET_FILE_RETURN' +GET_SOURCE_RETURN = 'a=1' +IS_PACKAGE_RETURN = True +_FETCH_FILE_RETURN = '_FETCH_FILE_RETURN' + + +class FSModuleImporterTestCase(TestCase): + def setUp(self): + self.path = '/usr/imp/custom_components/' + self.path_without_salsh = '/usr/imp/custom_components' + self.fullname = 'module1.module2.module3' + self.module_url = '/usr/imp/custom_components/module1/module2/module3.py' + self.package_url = '/usr/imp/custom_components/module1/module2/module3/__init__.py' -class S3ModuleImporterTestCase(TestCase): def test__init__(self): - pass + importer = FSModuleImporter(modules=[], path=self.path) + self.assertEqual(self.path, importer.path) + + importer = FSModuleImporter(modules=[], path=self.path_without_salsh) + self.assertEqual(self.path, importer.path) def test_is_package(self): - pass + importer = FSModuleImporter(modules=[], path=self.path) + with patch(OS_PATH_EXISTS, MagicMock(return_value=True)): + self.assertTrue(importer.is_package(self.fullname)) + + with patch(OS_PATH_EXISTS, MagicMock(return_value=False)): + self.assertFalse(importer.is_package(self.fullname)) + + @patch(UTILS_IMPORTER_FS_GET_FILE, MagicMock(return_value=GET_FILE_RETURN)) + @patch(UTILS_IMPORTER_FS_GET_SOURCE, MagicMock(return_value=GET_SOURCE_RETURN)) def test_get_code(self): - pass + expect_code = compile(GET_SOURCE_RETURN, GET_FILE_RETURN, 'exec') + importer = FSModuleImporter(modules=[], path=self.path) + + self.assertEqual(expect_code, importer.get_code(self.fullname)) + @patch(UTILS_IMPORTER_FS_IS_PACKAGE, MagicMock(return_value=IS_PACKAGE_RETURN)) + @patch(UTILS_IMPORTER_FS__FETCH_FILE_CONTENT, MagicMock(return_value=_FETCH_FILE_RETURN)) def test_get_source(self): - pass + importer = FSModuleImporter(modules=[], path=self.path) + + self.assertEqual(_FETCH_FILE_RETURN, importer.get_source(self.fullname)) + importer._fetch_file_content.assert_called_once_with( + importer._file_path(self.fullname, is_pkg=IS_PACKAGE_RETURN)) + + @patch(UTILS_IMPORTER_FS_IS_PACKAGE, MagicMock(return_value=IS_PACKAGE_RETURN)) + @patch(UTILS_IMPORTER_FS__FETCH_FILE_CONTENT, MagicMock(return_value=None)) + def test_get_source__fetch_none(self): + importer = FSModuleImporter(modules=[], path=self.path) + + self.assertRaises(ImportError, importer.get_source, self.fullname) + importer._fetch_file_content.assert_called_once_with( + importer._file_path(self.fullname, is_pkg=IS_PACKAGE_RETURN)) def test_get_path(self): - pass + importer = FSModuleImporter(modules=[], path=self.path) + + self.assertEqual(importer.get_path(self.fullname), ['/usr/imp/custom_components/module1/module2/module3']) def test_get_file(self): - pass + importer = FSModuleImporter(modules=[], path=self.path) + + with patch(UTILS_IMPORTER_FS_IS_PACKAGE, MagicMock(return_value=True)): + self.assertEqual(importer.get_file(self.fullname), self.package_url) + + with patch(UTILS_IMPORTER_FS_IS_PACKAGE, MagicMock(return_value=False)): + self.assertEqual(importer.get_file(self.fullname), self.module_url) def test__file_path(self): - pass + importer = FSModuleImporter(modules=[], path=self.path) + + self.assertEqual(importer._file_path(self.fullname, is_pkg=True), self.package_url) + self.assertEqual(importer._file_path(self.fullname, is_pkg=False), self.module_url) + + def test__fetch_file__nocache(self): + importer = FSModuleImporter(modules=[], path=self.path, use_cache=False) + + first_file_content = 'first_file_content' + second_file_content = 'second_file_content' + + with patch(UTILS_IMPORTER_FS__GET_FILE_CONTENT, MagicMock(return_value=first_file_content)): + self.assertEqual(importer._fetch_file_content(self.module_url), first_file_content) + self.assertEqual(importer.file_cache, {}) + + with patch(UTILS_IMPORTER_FS__GET_FILE_CONTENT, MagicMock(return_value=second_file_content)): + self.assertEqual(importer._fetch_file_content(self.module_url), second_file_content) + self.assertEqual(importer.file_cache, {}) + + with patch(UTILS_IMPORTER_FS__GET_FILE_CONTENT, MagicMock(return_value=None)): + self.assertIsNone(importer._fetch_file_content(self.module_url)) + self.assertEqual(importer.file_cache, {}) + + def test__fetch_file__use_cache(self): + importer = FSModuleImporter(modules=[], path=self.path) + + first_file_content = 'first_file_content' + second_file_content = 'second_file_content' + + with patch(UTILS_IMPORTER_FS__GET_FILE_CONTENT, MagicMock(return_value=first_file_content)): + self.assertEqual(importer._fetch_file_content(self.module_url), first_file_content) + self.assertEqual(importer.file_cache[self.module_url], first_file_content) + + with patch(UTILS_IMPORTER_FS__GET_FILE_CONTENT, MagicMock(return_value=second_file_content)): + self.assertEqual(importer._fetch_file_content(self.module_url), first_file_content) + self.assertEqual(importer.file_cache[self.module_url], first_file_content) + + with patch(UTILS_IMPORTER_FS__GET_FILE_CONTENT, MagicMock(return_value=None)): + self.assertIsNone(importer._fetch_file_content(self.package_url)) + self.assertEqual(importer.file_cache[self.package_url], None) - def test__fetch_file(self): - pass + with patch(UTILS_IMPORTER_FS__GET_FILE_CONTENT, MagicMock(return_value=second_file_content)): + self.assertIsNone(importer._fetch_file_content(self.package_url)) + self.assertEqual(importer.file_cache[self.package_url], None) diff --git a/pipeline/contrib/external_plugins/tests/utils/importer/test_git.py b/pipeline/contrib/external_plugins/tests/utils/importer/test_git.py index 2cc7362537..2698b9bc22 100644 --- a/pipeline/contrib/external_plugins/tests/utils/importer/test_git.py +++ b/pipeline/contrib/external_plugins/tests/utils/importer/test_git.py @@ -76,6 +76,7 @@ def test__fetch_repo_file__no_cache(self): with patch(REQUESTS_GET, MagicMock(return_value=MockResponse(ok=False))): self.assertIsNone(importer._fetch_repo_file(self.module_url)) + self.assertEqual(importer.file_cache, {}) def test__fetch_repo_file__use_cache(self): importer = GitRepoModuleImporter(modules=[], repo_raw_url=self.repo_raw_url, branch=self.branch) @@ -137,6 +138,7 @@ def test_get_source__fetch_none(self): def test_get_path(self): importer = GitRepoModuleImporter(modules=[], repo_raw_url=self.repo_raw_url, branch=self.branch) + self.assertEqual(importer.get_path(self.fullname), ['https://test-git-repo-raw/master/module1/module2/module3']) def test_get_file(self): diff --git a/pipeline/contrib/external_plugins/tests/utils/importer/test_s3.py b/pipeline/contrib/external_plugins/tests/utils/importer/test_s3.py index 806f82baf2..3c3c63ad4d 100644 --- a/pipeline/contrib/external_plugins/tests/utils/importer/test_s3.py +++ b/pipeline/contrib/external_plugins/tests/utils/importer/test_s3.py @@ -11,13 +11,17 @@ specific language governing permissions and limitations under the License. """ -import boto3 from django.test import TestCase from pipeline.contrib.external_plugins.tests.mock import * # noqa from pipeline.contrib.external_plugins.tests.mock_settings import * # noqa from pipeline.contrib.external_plugins.utils.importer.s3 import S3ModuleImporter +GET_FILE_RETURN = 'GET_FILE_RETURN' +GET_SOURCE_RETURN = 'a=1' +IS_PACKAGE_RETURN = True +_FETCH_OBJ_CONTENT_RETURN = '_FETCH_OBJ_CONTENT_RETURN' + class S3ModuleImporterTestCase(TestCase): @@ -28,6 +32,11 @@ def setUp(self): self.bucket = 'bucket' self.access_key = 'access_key' self.secret_key = 'secret_key' + self.fullname = 'module1.module2.module3' + self.module_url = 'https://test-s3-address/bucket/module1/module2/module3.py' + self.package_url = 'https://test-s3-address/bucket/module1/module2/module3/__init__.py' + self.module_key = 'module1/module2/module3.py' + self.package_key = 'module1/module2/module3/__init__.py' @patch(BOTO3_RESOURCE, mock_s3_resource) def test__init__(self): @@ -65,30 +74,141 @@ def test__init__(self): service_address=self.not_secure_service_address, bucket=self.bucket, access_key=self.access_key, - secret_key=self.secret_key) + secret_key=self.secret_key, + secure_only=False) self.assertEqual(self.not_secure_service_address, importer.service_address) self.assertEqual(importer.s3, mock_s3_resource('s3', aws_access_key_id=self.access_key, aws_secret_access_key=self.secret_key, - endpoint_url=self.service_address)) + endpoint_url=self.not_secure_service_address)) def test_is_package(self): - pass + importer = S3ModuleImporter(modules=[], + service_address=self.service_address, + bucket=self.bucket, + access_key=self.access_key, + secret_key=self.secret_key) + + with patch(UTILS_IMPORTER_S3__FETCH_OBJ_CONTENT, MagicMock(return_value='')): + self.assertTrue(importer.is_package('a.b.c')) + with patch(UTILS_IMPORTER_S3__FETCH_OBJ_CONTENT, MagicMock(return_value=None)): + self.assertFalse(importer.is_package('a.b.c')) + + @patch(UTILS_IMPORTER_S3_GET_FILE, MagicMock(return_value=GET_FILE_RETURN)) + @patch(UTILS_IMPORTER_S3_GET_SOURCE, MagicMock(return_value=GET_SOURCE_RETURN)) def test_get_code(self): - pass + expect_code = compile(GET_SOURCE_RETURN, GET_FILE_RETURN, 'exec') + importer = S3ModuleImporter(modules=[], + service_address=self.service_address, + bucket=self.bucket, + access_key=self.access_key, + secret_key=self.secret_key) + self.assertEqual(expect_code, importer.get_code(self.fullname)) + + @patch(UTILS_IMPORTER_S3_IS_PACKAGE, MagicMock(return_value=IS_PACKAGE_RETURN)) + @patch(UTILS_IMPORTER_S3__FETCH_OBJ_CONTENT, MagicMock(return_value=_FETCH_OBJ_CONTENT_RETURN)) def test_get_source(self): - pass + importer = S3ModuleImporter(modules=[], + service_address=self.service_address, + bucket=self.bucket, + access_key=self.access_key, + secret_key=self.secret_key) + + self.assertEqual(_FETCH_OBJ_CONTENT_RETURN, importer.get_source(self.fullname)) + importer._fetch_obj_content.assert_called_once_with(importer._obj_key(self.fullname, is_pkg=IS_PACKAGE_RETURN)) + @patch(UTILS_IMPORTER_S3_IS_PACKAGE, MagicMock(return_value=IS_PACKAGE_RETURN)) + @patch(UTILS_IMPORTER_S3__FETCH_OBJ_CONTENT, MagicMock(return_value=None)) + def test_get_source__fetch_none(self): + importer = S3ModuleImporter(modules=[], + service_address=self.service_address, + bucket=self.bucket, + access_key=self.access_key, + secret_key=self.secret_key) + + self.assertRaises(ImportError, importer.get_source, self.fullname) + importer._fetch_obj_content.assert_called_once_with(importer._obj_key(self.fullname, is_pkg=IS_PACKAGE_RETURN)) + + @patch(UTILS_IMPORTER_S3_IS_PACKAGE, MagicMock(return_value=IS_PACKAGE_RETURN)) def test_get_path(self): - pass + importer = S3ModuleImporter(modules=[], + service_address=self.service_address, + bucket=self.bucket, + access_key=self.access_key, + secret_key=self.secret_key) + + self.assertEqual(importer.get_path(self.fullname), ['https://test-s3-address/bucket/module1/module2/module3']) def test_get_file(self): - pass + importer = S3ModuleImporter(modules=[], + service_address=self.service_address, + bucket=self.bucket, + access_key=self.access_key, + secret_key=self.secret_key) + + with patch(UTILS_IMPORTER_S3_IS_PACKAGE, MagicMock(return_value=False)): + self.assertEqual(importer.get_file(self.fullname), self.module_url) + + with patch(UTILS_IMPORTER_S3_IS_PACKAGE, MagicMock(return_value=True)): + self.assertEqual(importer.get_file(self.fullname), self.package_url) def test__obj_key(self): - pass + importer = S3ModuleImporter(modules=[], + service_address=self.service_address, + bucket=self.bucket, + access_key=self.access_key, + secret_key=self.secret_key) + + self.assertEqual('module1/module2/module3/__init__.py', importer._obj_key(self.fullname, is_pkg=True)) + self.assertEqual('module1/module2/module3.py', importer._obj_key(self.fullname, is_pkg=False)) + + def test__fetch_obj_content__no_cache(self): + importer = S3ModuleImporter(modules=[], + service_address=self.service_address, + bucket=self.bucket, + access_key=self.access_key, + secret_key=self.secret_key, + use_cache=False) + + first_obj_content = 'first_obj_content' + second_obj_content = 'second_obj_content' + + with patch(UTILS_IMPORTER_S3__GET_S3_OBJ_CONTENT, MagicMock(return_value=first_obj_content)): + self.assertEqual(importer._fetch_obj_content(self.module_key), first_obj_content) + self.assertEqual(importer.obj_cache, {}) + + with patch(UTILS_IMPORTER_S3__GET_S3_OBJ_CONTENT, MagicMock(return_value=second_obj_content)): + self.assertEqual(importer._fetch_obj_content(self.module_key), second_obj_content) + self.assertEqual(importer.obj_cache, {}) + + with patch(UTILS_IMPORTER_S3__GET_S3_OBJ_CONTENT, MagicMock(return_value=None)): + self.assertIsNone(importer._fetch_obj_content(self.module_key)) + self.assertEqual(importer.obj_cache, {}) + + def test__fetch_obj_content__use_cache(self): + importer = S3ModuleImporter(modules=[], + service_address=self.service_address, + bucket=self.bucket, + access_key=self.access_key, + secret_key=self.secret_key) + + first_obj_content = 'first_obj_content' + second_obj_content = 'second_obj_content' + + with patch(UTILS_IMPORTER_S3__GET_S3_OBJ_CONTENT, MagicMock(return_value=first_obj_content)): + self.assertEqual(importer._fetch_obj_content(self.module_key), first_obj_content) + self.assertEqual(importer.obj_cache[self.module_key], first_obj_content) + + with patch(UTILS_IMPORTER_S3__GET_S3_OBJ_CONTENT, MagicMock(return_value=second_obj_content)): + self.assertEqual(importer._fetch_obj_content(self.module_key), first_obj_content) + self.assertEqual(importer.obj_cache[self.module_key], first_obj_content) + + with patch(UTILS_IMPORTER_S3__GET_S3_OBJ_CONTENT, MagicMock(return_value=None)): + self.assertIsNone(importer._fetch_obj_content(self.package_key)) + self.assertIsNone(importer.obj_cache[self.package_key]) - def test__fetch_obj_content(self): - pass + with patch(UTILS_IMPORTER_S3__GET_S3_OBJ_CONTENT, MagicMock(return_value=first_obj_content)): + self.assertIsNone(importer._fetch_obj_content(self.package_key)) + self.assertIsNone(importer.obj_cache[self.package_key]) diff --git a/pipeline/contrib/external_plugins/utils/importer/fs.py b/pipeline/contrib/external_plugins/utils/importer/fs.py index 17a0e44eb8..21b5d6296a 100644 --- a/pipeline/contrib/external_plugins/utils/importer/fs.py +++ b/pipeline/contrib/external_plugins/utils/importer/fs.py @@ -38,7 +38,7 @@ def get_code(self, fullname): return compile(self.get_source(fullname), self.get_file(fullname), 'exec') def get_source(self, fullname): - source_code = self._fetch_file(self._file_path(fullname, is_pkg=self.is_package(fullname))) + source_code = self._fetch_file_content(self._file_path(fullname, is_pkg=self.is_package(fullname))) if source_code is None: raise ImportError('Can not find {module} in {path}'.format( @@ -59,13 +59,21 @@ def _file_path(self, fullname, is_pkg=False): file_path = '%s/__init__.py' % base_path if is_pkg else '%s.py' % base_path return file_path - def _fetch_file(self, file_path): + def _fetch_file_content(self, file_path): logger.info('Try to fetch file {file_path}'.format(file_path=file_path)) if self.use_cache and file_path in self.file_cache: logger.info('Use content in cache for file: {file_path}'.format(file_path=file_path)) return self.file_cache[file_path] + file_content = self._get_file_content(file_path) + + if self.use_cache: + self.file_cache[file_path] = file_content + + return file_content + + def _get_file_content(self, file_path): try: with open(file_path) as f: file_content = f.read() @@ -74,9 +82,6 @@ def _fetch_file(self, file_path): file_path=file_path, trace=traceback.format_exc() )) - return None - - if self.use_cache: - self.file_cache[file_path] = file_content + file_content = None return file_content diff --git a/pipeline/contrib/external_plugins/utils/importer/s3.py b/pipeline/contrib/external_plugins/utils/importer/s3.py index 536892368e..1cfc2a4fda 100644 --- a/pipeline/contrib/external_plugins/utils/importer/s3.py +++ b/pipeline/contrib/external_plugins/utils/importer/s3.py @@ -86,6 +86,14 @@ def _fetch_obj_content(self, key): logger.info('Use content in cache for s3 object: {key}'.format(key=key)) return self.obj_cache[key] + obj_content = self._get_s3_obj_content(key) + + if self.use_cache: + self.obj_cache[key] = obj_content + + return obj_content + + def _get_s3_obj_content(self, key): obj = self.s3.Object(bucket_name=self.bucket, key=key) try: @@ -97,7 +105,4 @@ def _fetch_obj_content(self, key): else: raise - if self.use_cache: - self.obj_cache[key] = obj_content - return obj_content From c0b0efb735fe1029ee1179fa6d9b82e76e704ecb Mon Sep 17 00:00:00 2001 From: homholueng Date: Tue, 16 Apr 2019 19:05:01 +0800 Subject: [PATCH 24/29] =?UTF-8?q?feature:=20s3=20=E4=B8=8E=20fs=20?= =?UTF-8?q?=E7=B1=BB=E5=9E=8B=E7=9A=84=20ExternalSource=20=E8=BF=94?= =?UTF-8?q?=E5=9B=9E=E5=AF=B9=E5=BA=94=E7=9A=84=20importer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../contrib/external_plugins/models/source.py | 15 ++++++++++++--- .../external_plugins/utils/importer/__init__.py | 2 ++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/pipeline/contrib/external_plugins/models/source.py b/pipeline/contrib/external_plugins/models/source.py index 3276592f66..66f7fe16d8 100644 --- a/pipeline/contrib/external_plugins/models/source.py +++ b/pipeline/contrib/external_plugins/models/source.py @@ -14,7 +14,11 @@ from django.db import models from django.conf import settings from django.utils.translation import ugettext_lazy as _ -from pipeline.contrib.external_plugins.utils.importer.git import GitRepoModuleImporter +from pipeline.contrib.external_plugins.utils.importer import ( + GitRepoModuleImporter, + S3ModuleImporter, + FSModuleImporter +) from pipeline.contrib.external_plugins.models.base import ( GIT, @@ -52,7 +56,11 @@ def type(): return S3 def importer(self): - pass + return S3ModuleImporter(modules=self.packages.keys(), + service_address=self.service_address, + bucket=self.bucket, + access_key=self.access_key, + secret_key=self.secret_key) @package_source @@ -64,4 +72,5 @@ def type(): return FILE_SYSTEM def importer(self): - pass + return FSModuleImporter(modules=self.packages.keys(), + path=self.path) diff --git a/pipeline/contrib/external_plugins/utils/importer/__init__.py b/pipeline/contrib/external_plugins/utils/importer/__init__.py index 6bebac0b21..f51cd27482 100644 --- a/pipeline/contrib/external_plugins/utils/importer/__init__.py +++ b/pipeline/contrib/external_plugins/utils/importer/__init__.py @@ -13,3 +13,5 @@ from pipeline.contrib.external_plugins.utils.importer.utils import importer_context # noqa from pipeline.contrib.external_plugins.utils.importer.git import GitRepoModuleImporter # noqa +from pipeline.contrib.external_plugins.utils.importer.s3 import S3ModuleImporter # noqa +from pipeline.contrib.external_plugins.utils.importer.fs import FSModuleImporter # noqa From 23c1cdd9d67c177c70af80a37165360030971a45 Mon Sep 17 00:00:00 2001 From: homholueng Date: Wed, 17 Apr 2019 10:55:57 +0800 Subject: [PATCH 25/29] =?UTF-8?q?feature:=20=E5=85=81=E8=AE=B8=E7=94=A8?= =?UTF-8?q?=E6=88=B7=E9=80=9A=E8=BF=87=E9=85=8D=E7=BD=AE=E6=8E=A7=E5=88=B6?= =?UTF-8?q?=E6=AF=8F=E4=B8=AA=E6=BA=90=E7=9A=84=E5=AE=89=E5=85=A8=E9=99=90?= =?UTF-8?q?=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pipeline/contrib/external_plugins/models/source.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pipeline/contrib/external_plugins/models/source.py b/pipeline/contrib/external_plugins/models/source.py index 66f7fe16d8..10a251f292 100644 --- a/pipeline/contrib/external_plugins/models/source.py +++ b/pipeline/contrib/external_plugins/models/source.py @@ -41,7 +41,9 @@ def importer(self): return GitRepoModuleImporter(repo_raw_url=self.repo_raw_address, branch=self.branch, modules=self.packages.keys(), - proxy=getattr(settings, 'EXTERNAL_SOURCE_PROXY')) + proxy=getattr(settings, 'EXTERNAL_SOURCE_PROXY'), + secure_only=getattr(settings, + 'EXTERNAL_SOURCE_SECURE_RESTRICT', {}).get(self.name, True)) @package_source @@ -60,7 +62,9 @@ def importer(self): service_address=self.service_address, bucket=self.bucket, access_key=self.access_key, - secret_key=self.secret_key) + secret_key=self.secret_key, + secure_only=getattr(settings, + 'EXTERNAL_SOURCE_SECURE_RESTRICT', {}).get(self.name, True)) @package_source From 7f4465f2be14351d2312cde410b31e8c71cb1bd8 Mon Sep 17 00:00:00 2001 From: homholueng Date: Wed, 17 Apr 2019 11:01:28 +0800 Subject: [PATCH 26/29] =?UTF-8?q?bugfix:=20=E4=BF=AE=E5=A4=8D=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=E7=BB=84=E4=BB=B6=E5=AE=9E=E4=BE=8B=E6=97=B6=E6=B2=A1?= =?UTF-8?q?=E6=9C=89=E5=A4=84=E7=90=86=E7=BB=84=E4=BB=B6=E5=8F=AF=E8=83=BD?= =?UTF-8?q?=E4=B8=8D=E5=AD=98=E5=9C=A8=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pipeline/component_framework/library.py | 8 +++++--- pipeline/tests/component_framework/test_library.py | 3 +++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pipeline/component_framework/library.py b/pipeline/component_framework/library.py index 4d2572a893..d44cd6ad8e 100644 --- a/pipeline/component_framework/library.py +++ b/pipeline/component_framework/library.py @@ -25,8 +25,7 @@ def __new__(cls, *args, **kwargs): raise ValueError('please pass a component_code in args or kwargs: ' 'ComponentLibrary(\'code\') or ComponentLibrary(component_code=\'code\')') if component_code not in cls.components: - raise ComponentNotExistException('component %s does not exist.' % - component_code) + raise ComponentNotExistException('component %s does not exist.' % component_code) return cls.components[component_code] @classmethod @@ -35,4 +34,7 @@ def get_component_class(cls, component_code): @classmethod def get_component(cls, component_code, data_dict): - return cls.get_component_class(component_code)(data_dict) + component_cls = cls.get_component_class(component_code) + if component_cls is None: + raise ComponentNotExistException('component %s does not exist.' % component_code) + return component_cls(data_dict) diff --git a/pipeline/tests/component_framework/test_library.py b/pipeline/tests/component_framework/test_library.py index 09f359a2c7..118a3e1a90 100644 --- a/pipeline/tests/component_framework/test_library.py +++ b/pipeline/tests/component_framework/test_library.py @@ -67,6 +67,9 @@ def outputs_format(self): self.assertEqual(ComponentLibrary.get_component('code', {}).__class__, TestComponent) + def test_get_component__raise(self): + self.assertRaises(ComponentNotExistException, ComponentLibrary.get_component, 'c_not_exist', {}) + def test_args_new(self): component = ComponentLibrary(self.component.code) self.assertEqual(component, self.component) From 9cc8e7efa7e835e9cc70fc576ed261b449f9370e Mon Sep 17 00:00:00 2001 From: homholueng Date: Wed, 17 Apr 2019 12:03:39 +0800 Subject: [PATCH 27/29] =?UTF-8?q?feature:=20=E6=B7=BB=E5=8A=A0=20form=5Fis?= =?UTF-8?q?=5Fembbeded=20=E6=96=B9=E6=B3=95=E6=9D=A5=E5=88=A4=E6=96=AD?= =?UTF-8?q?=E7=BB=84=E4=BB=B6=E6=98=AF=E5=90=A6=E5=8C=85=E5=90=AB=E5=B5=8C?= =?UTF-8?q?=E5=85=A5=E5=BC=8F=E7=9A=84=E8=A1=A8=E5=8D=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pipeline/component_framework/component.py | 4 ++++ pipeline/tests/component_framework/test_component.py | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/pipeline/component_framework/component.py b/pipeline/component_framework/component.py index 1cc98320fd..22d63c6bf8 100644 --- a/pipeline/component_framework/component.py +++ b/pipeline/component_framework/component.py @@ -29,6 +29,10 @@ def outputs_format(cls): outputs = map(lambda oi: oi._asdict(), outputs) return outputs + @classmethod + def form_is_embedded(cls): + return getattr(cls, 'embedded_form', False) + def clean_execute_data(self, context): return self.data_dict diff --git a/pipeline/tests/component_framework/test_component.py b/pipeline/tests/component_framework/test_component.py index 5f7329a1ac..a3ba920675 100644 --- a/pipeline/tests/component_framework/test_component.py +++ b/pipeline/tests/component_framework/test_component.py @@ -43,8 +43,16 @@ class CCUpdateHostModuleComponent(Component): code = 'cc_update_module' form = 'form path' + class CCUpdateHostModuleComponentEmbeddedForm(Component): + name = u'修改主机所属模块' + bound_service = CCUpdateHostModuleService + code = 'cc_update_module_embedded_form' + embedded_form = True + form = 'form path' + self.service = CCUpdateHostModuleService self.component = CCUpdateHostModuleComponent + self.component_embedded_form = CCUpdateHostModuleComponentEmbeddedForm def tearDown(self): ComponentModel.objects.all().delete() @@ -92,3 +100,7 @@ def test_data_for_execution_lack_of_inputs(self): } component = self.component(data) self.assertRaises(ComponentDataLackException, execution_data=component.data_for_execution, args=[None, None]) + + def test_form_is_embedded(self): + self.assertFalse(self.component.form_is_embedded()) + self.assertTrue(self.component_embedded_form.form_is_embedded()) From 225abf021f15d92ffcbdedc6913bba41b40fc19f Mon Sep 17 00:00:00 2001 From: homholueng Date: Wed, 17 Apr 2019 12:10:05 +0800 Subject: [PATCH 28/29] =?UTF-8?q?feature:=20component=20=E7=9B=B8=E5=85=B3?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E6=B7=BB=E5=8A=A0=20form=5Fis=5Fembedded=20?= =?UTF-8?q?=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- gcloud/webservice3/resources.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gcloud/webservice3/resources.py b/gcloud/webservice3/resources.py index e0f2909c9e..52c4fe6403 100644 --- a/gcloud/webservice3/resources.py +++ b/gcloud/webservice3/resources.py @@ -354,6 +354,7 @@ def alter_list_data_to_serialize(self, request, data): bundle.data['output'] = component.outputs_format() bundle.data['form'] = component.form bundle.data['desc'] = component.desc + bundle.data['form_is_embedded'] = component.form_is_embedded() # 国际化 name = bundle.data['name'].split('-') bundle.data['group_name'] = _(name[0]) @@ -367,6 +368,7 @@ def alter_detail_data_to_serialize(self, request, data): bundle.data['output'] = component.outputs_format() bundle.data['form'] = component.form bundle.data['desc'] = component.desc + bundle.data['form_is_embedded'] = component.form_is_embedded() # 国际化 name = bundle.data['name'].split('-') bundle.data['group_name'] = _(name[0]) From 8ef0134eba8ce5c809981b8682f77f60c87ccded Mon Sep 17 00:00:00 2001 From: homholueng Date: Wed, 17 Apr 2019 14:45:34 +0800 Subject: [PATCH 29/29] minor: sync pipeline code --- pipeline/component_framework/test.py | 204 ++++++++++++++++++ pipeline/engine/api.py | 10 +- pipeline/engine/core/runtime.py | 4 +- pipeline/engine/core/schedule.py | 10 +- pipeline/engine/utils.py | 4 +- pipeline/tests/engine/core/test_runtime.py | 4 +- pipeline/tests/engine/core/test_schedule.py | 59 ++++- pipeline/tests/engine/test_api.py | 20 +- .../tests/engine/utils/test_utils_func.py | 55 +++++ pipeline/tests/mock.py | 3 +- pipeline/validators/base.py | 11 +- 11 files changed, 363 insertions(+), 21 deletions(-) create mode 100644 pipeline/component_framework/test.py create mode 100644 pipeline/tests/engine/utils/test_utils_func.py diff --git a/pipeline/component_framework/test.py b/pipeline/component_framework/test.py new file mode 100644 index 0000000000..e5c95fb824 --- /dev/null +++ b/pipeline/component_framework/test.py @@ -0,0 +1,204 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import logging + +from abc import abstractproperty + +from pipeline.core.data.base import DataObject + +logger = logging.getLogger(__name__) + + +class ComponentTestMixin(object): + + @abstractproperty + def component_cls(self): + raise NotImplementedError() + + @abstractproperty + def cases(self): + raise NotImplementedError() + + @property + def patchers(self): + return [] + + @property + def _component_cls_name(self): + return self.component_cls.__name__ + + def _format_failure_message(self, no, name, msg): + return '{component_cls} case {no}:{name} fail: {msg}'.format( + component_cls=self._component_cls_name, + no=no + 1, + name=name, + msg=msg + ) + + def _do_case_assert(self, service, method, assertion, no, name, args=None, kwargs=None): + + do_continue = False + args = args or [service] + kwargs = kwargs or {} + + data = kwargs.get('data') or args[0] + + if assertion.exc: + # raise assertion + + try: + getattr(service, method)(*args, **kwargs) + except Exception as e: + assert e.__class__ == assertion.exc, self._format_failure_message( + no=no, + name=name, + msg='{method} raise assertion failed,\nexcept: {e}\nactual: {a}'.format( + method=method, + e=assertion.exc, + a=e.__class__ + )) + do_continue = True + else: + self.assertTrue(False, msg=self._format_failure_message( + no=no, + name=name, + msg='{method} raise assertion failed, {method} not raise any exception'.format( + method=method + ) + )) + else: + + result = getattr(service, method)(*args, **kwargs) + + if result is None or result is True: + self.assertTrue(assertion.success, msg=self._format_failure_message( + no=no, + name=name, + msg='{method} success assertion failed, {method} execute success'.format( + method=method + ) + )) + + self.assertDictEqual(data.outputs, assertion.outputs, msg=self._format_failure_message( + no=no, + name=name, + msg='{method} outputs assertion failed,\nexcept: {e}\nactual: {a}'.format( + method=method, + e=data.outputs, + a=assertion.outputs + ) + )) + + else: + self.assertFalse(assertion.success, msg=self._format_failure_message( + no=no, + name=name, + msg='{method} success assertion failed, {method} execute failed'.format( + method=method + ) + )) + + do_continue = True + + return do_continue + + def test_component(self): + patchers = self.patchers + for patcher in patchers: + patcher.start() + + component = self.component_cls({}) + + bound_service = component.service() + + for no, case in enumerate(self.cases): + data = DataObject(inputs=case.inputs) + parent_data = DataObject(inputs=case.parent_data) + + # execute result check + do_continue = self._do_case_assert(service=bound_service, + method='execute', + args=(data, parent_data), + assertion=case.execute_assertion, + no=no, + name=case.name) + + if do_continue: + continue + + if bound_service.need_schedule(): + + if bound_service.interval is None: + # callback case + self._do_case_assert(service=bound_service, + method='schedule', + args=(data, parent_data, case.schedule_assertion.callback_data), + assertion=case.schedule_assertion, + no=no, + name=case.name) + + else: + # schedule case + assertions = case.schedule_assertion + assertions = assertions if isinstance(assertions, list) else [assertions] + + for assertion in assertions: + do_continue = self._do_case_assert(service=bound_service, + method='schedule', + args=(data, parent_data), + assertion=assertion, + no=no, + name=case.name) + + if do_continue: + break + + logger.info('{component} paas {num} cases.'.format( + component=self._component_cls_name, + num=len(self.cases) + )) + + for patcher in patchers: + patcher.stop() + + +class ComponentTestCase(object): + def __init__(self, + inputs, + parent_data, + execute_assertion, + schedule_assertion, + name=''): + self.inputs = inputs + self.parent_data = parent_data + self.execute_assertion = execute_assertion + self.schedule_assertion = schedule_assertion + self.name = name + + +class Assertion(object): + def __init__(self, success, outputs, exc=None): + self.success = success + self.outputs = outputs + self.exc = exc + + +class ExecuteAssertion(Assertion): + pass + + +class ScheduleAssertion(Assertion): + def __init__(self, callback_data, *args, **kwargs): + self.callback_data = callback_data + super(ScheduleAssertion, self).__init__(*args, **kwargs) diff --git a/pipeline/engine/api.py b/pipeline/engine/api.py index 945e601ed4..854110a779 100644 --- a/pipeline/engine/api.py +++ b/pipeline/engine/api.py @@ -145,8 +145,14 @@ def revoke_pipeline(pipeline_id): return action_result process = PipelineModel.objects.get(id=pipeline_id).process - process.revoke_subprocess() - process.destroy_all() + + if not process: + return ActionResult(result=False, message='relate process is none, this pipeline may be revoked.') + + with transaction.atomic(): + PipelineProcess.objects.select_for_update().get(id=process.id) + process.revoke_subprocess() + process.destroy_all() return action_result diff --git a/pipeline/engine/core/runtime.py b/pipeline/engine/core/runtime.py index c7ba2e5785..2d91e373bb 100644 --- a/pipeline/engine/core/runtime.py +++ b/pipeline/engine/core/runtime.py @@ -74,8 +74,8 @@ def run_loop(process): return # try to transit current node to running state - action = Status.objects.transit(id=current_node.id, to_state=states.RUNNING, start=True, - name=str(current_node.__class__)) + name = (current_node.name or str(current_node.__class__))[:64] + action = Status.objects.transit(id=current_node.id, to_state=states.RUNNING, start=True, name=name) # check rerun limit if not isinstance(current_node, SubProcess) and RERUN_MAX_LIMIT != 0 and \ diff --git a/pipeline/engine/core/schedule.py b/pipeline/engine/core/schedule.py index f2d859d597..62a9b312d9 100644 --- a/pipeline/engine/core/schedule.py +++ b/pipeline/engine/core/schedule.py @@ -102,8 +102,14 @@ def schedule(process_id, schedule_id): logger.info('node %s %s timeout monitor revoke' % (service_act.id, version)) Data.objects.write_node_data(service_act, ex_data=ex_data) - process = PipelineProcess.objects.get(id=sched_service.process_id) - process.adjust_status() + + with transaction.atomic(): + process = PipelineProcess.objects.select_for_update().get(id=sched_service.process_id) + if not process.is_alive: + logger.info('pipeline %s has been revoked, status adjust failed.' % process.root_pipeline_id) + return + + process.adjust_status() # send activity error signal diff --git a/pipeline/engine/utils.py b/pipeline/engine/utils.py index d9696d8c8a..63153e6773 100644 --- a/pipeline/engine/utils.py +++ b/pipeline/engine/utils.py @@ -47,9 +47,9 @@ def calculate_elapsed_time(started_time, archived_time): """ if archived_time and started_time: # when status_tree['archived_time'] == status_tree['started_time'], set elapsed_time to 1s - elapsed_time = (archived_time - started_time).seconds or 1 + elapsed_time = (archived_time - started_time).total_seconds() or 1 elif started_time: - elapsed_time = (timezone.now() - started_time).seconds + elapsed_time = (timezone.now() - started_time).total_seconds() else: elapsed_time = 0 return elapsed_time diff --git a/pipeline/tests/engine/core/test_runtime.py b/pipeline/tests/engine/core/test_runtime.py index 2f657b2d77..005f5a25f4 100644 --- a/pipeline/tests/engine/core/test_runtime.py +++ b/pipeline/tests/engine/core/test_runtime.py @@ -253,7 +253,7 @@ def test_run_loop(self): with patch('pipeline.engine.core.runtime.FLOW_NODE_HANDLERS', mock_handlers): # 6.1. test should return - current_node = IdentifyObject() + current_node = IdentifyObject(name='name') process = MockPipelineProcess(top_pipeline=PipelineObject(node=current_node), destination_id=uniqid(), current_node_id=current_node.id) @@ -275,7 +275,7 @@ def test_run_loop(self): Status.objects.transit.assert_called_with(id=current_node.id, to_state=states.RUNNING, start=True, - name=str(current_node.__class__)) + name=current_node.name) process.refresh_current_node.assert_called_once_with(current_node.id) diff --git a/pipeline/tests/engine/core/test_schedule.py b/pipeline/tests/engine/core/test_schedule.py index ba33fd4387..ec54facc60 100644 --- a/pipeline/tests/engine/core/test_schedule.py +++ b/pipeline/tests/engine/core/test_schedule.py @@ -161,7 +161,8 @@ def test_schedule__schedule_return_fail_and_transit_success(self): for timeout in (True, False): process = MockPipelineProcess() mock_ss = MockScheduleService(schedule_return=False, service_timeout=timeout) - with mock.patch(PIPELINE_PROCESS_GET, mock.MagicMock(return_value=process)): + with mock.patch(PIPELINE_PROCESS_SELECT_FOR_UPDATE, + mock.MagicMock(return_value=MockQuerySet(get_return=process))): with mock.patch(PIPELINE_SCHEDULE_SERVICE_GET, mock.MagicMock(return_value=mock_ss)): process_id = uniqid() @@ -250,7 +251,8 @@ def test_schedule__schedule_raise_exception_and_transit_success(self): mock_ss = MockScheduleService(schedule_exception=e, service_timeout=timeout) process = MockPipelineProcess() - with mock.patch(PIPELINE_PROCESS_GET, mock.MagicMock(return_value=process)): + with mock.patch(PIPELINE_PROCESS_SELECT_FOR_UPDATE, + mock.MagicMock(return_value=MockQuerySet(get_return=process))): with mock.patch(PIPELINE_SCHEDULE_SERVICE_GET, mock.MagicMock(return_value=mock_ss)): process_id = uniqid() @@ -290,6 +292,59 @@ def test_schedule__schedule_raise_exception_and_transit_success(self): signals.service_schedule_fail.send.reset_mock() valve.send.reset_mock() + @mock.patch(PIPELINE_SCHEDULE_SERVICE_FILTER, mock.MagicMock(return_value=MockQuerySet(exists_return=True))) + @mock.patch(PIPELINE_STATUS_TRANSIT, mock.MagicMock(return_value=MockActionResult(result=True))) + @mock.patch(PIPELINE_DATA_WRITE_NODE_DATA, mock.MagicMock()) + @mock.patch(PIPELINE_STATUS_FILTER, mock.MagicMock(return_value=MockQuerySet(exists_return=True))) + @mock.patch(SCHEDULE_DELETE_PARENT_DATA, mock.MagicMock()) + @mock.patch(SCHEDULE_GET_SCHEDULE_PARENT_DATA, mock.MagicMock(return_value=PARENT_DATA)) + @mock.patch(SCHEDULE_SET_SCHEDULE_DATA, mock.MagicMock()) + @mock.patch(ENGINE_SIGNAL_TIMEOUT_END_SEND, mock.MagicMock()) + @mock.patch(ENGINE_SIGNAL_ACT_SCHEDULE_FAIL_SEND, mock.MagicMock()) + @mock.patch(SIGNAL_VALVE_SEND, mock.MagicMock()) + def test_schedule__schedule_raise_exception_and_process_is_not_alive(self): + for timeout in (True, False): + e = Exception() + mock_ss = MockScheduleService(schedule_exception=e, service_timeout=timeout) + process = MockPipelineProcess(is_alive=False) + + with mock.patch(PIPELINE_PROCESS_SELECT_FOR_UPDATE, + mock.MagicMock(return_value=MockQuerySet(get_return=process))): + with mock.patch(PIPELINE_SCHEDULE_SERVICE_GET, mock.MagicMock(return_value=mock_ss)): + process_id = uniqid() + + schedule.schedule(process_id, mock_ss.id) + + mock_ss.service_act.schedule.assert_called_once_with(PARENT_DATA, mock_ss.callback_data) + + self.assertEqual(mock_ss.schedule_times, 1) + + mock_ss.destroy.assert_not_called() + + if timeout: + signals.service_activity_timeout_monitor_end.send.assert_called_once_with( + sender=mock_ss.service_act.__class__, + node_id=mock_ss.service_act.id, + version=mock_ss.version + ) + else: + signals.service_activity_timeout_monitor_end.send.assert_not_called() + + Data.objects.write_node_data.assert_called() + + process.adjust_status.assert_not_called() + + mock_ss.service_act.schedule_fail.assert_not_called() + + signals.service_schedule_fail.send.assert_not_called() + + valve.send.assert_not_called() + + signals.service_activity_timeout_monitor_end.send.reset_mock() + Data.objects.write_node_data.reset_mock() + signals.service_schedule_fail.send.reset_mock() + valve.send.reset_mock() + @mock.patch(PIPELINE_SCHEDULE_SERVICE_FILTER, mock.MagicMock(return_value=MockQuerySet(exists_return=True))) @mock.patch(PIPELINE_STATUS_TRANSIT, mock.MagicMock(return_value=MockActionResult(result=True))) @mock.patch(PIPELINE_DATA_WRITE_NODE_DATA, mock.MagicMock()) diff --git a/pipeline/tests/engine/test_api.py b/pipeline/tests/engine/test_api.py index 785c5ca9a4..461f463b6c 100644 --- a/pipeline/tests/engine/test_api.py +++ b/pipeline/tests/engine/test_api.py @@ -163,18 +163,30 @@ def test_revoke_pipeline__transit_fail(self): self.assertFalse(act_result.result) + @patch(PIPELINE_FUNCTION_SWITCH_IS_FROZEN, MagicMock(return_value=False)) + @patch(PIPELINE_STATUS_TRANSIT, MagicMock(return_value=MockActionResult(result=True))) + def test_revoke_pipeline__process_is_none(self): + pipeline_model = MockPipelineModel(process=None) + + with patch(PIPELINE_PIPELINE_MODEL_GET, MagicMock(return_value=pipeline_model)): + act_result = api.revoke_pipeline(self.pipeline_id) + + self.assertFalse(act_result.result) + @patch(PIPELINE_FUNCTION_SWITCH_IS_FROZEN, MagicMock(return_value=False)) @patch(PIPELINE_STATUS_TRANSIT, MagicMock(return_value=MockActionResult(result=True))) def test_revoke_pipeline__transit_success(self): pipeline_model = MockPipelineModel() with patch(PIPELINE_PIPELINE_MODEL_GET, MagicMock(return_value=pipeline_model)): - act_result = api.revoke_pipeline(self.pipeline_id) + with mock.patch(PIPELINE_PROCESS_SELECT_FOR_UPDATE, + mock.MagicMock(return_value=MockQuerySet(get_return=pipeline_model.process))): + act_result = api.revoke_pipeline(self.pipeline_id) - self.assertTrue(act_result.result) + self.assertTrue(act_result.result) - pipeline_model.process.revoke_subprocess.assert_called_once() - pipeline_model.process.destroy_all.assert_called_once() + pipeline_model.process.revoke_subprocess.assert_called_once() + pipeline_model.process.destroy_all.assert_called_once() @patch(PIPELINE_FUNCTION_SWITCH_IS_FROZEN, MagicMock(return_value=False)) @patch(PIPELINE_STATUS_TRANSIT, MagicMock(return_value=MockActionResult(result=True))) diff --git a/pipeline/tests/engine/utils/test_utils_func.py b/pipeline/tests/engine/utils/test_utils_func.py new file mode 100644 index 0000000000..cfe1c282fa --- /dev/null +++ b/pipeline/tests/engine/utils/test_utils_func.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import datetime + +from django.test import TestCase +from django.utils import timezone + +from pipeline.engine.utils import calculate_elapsed_time + + +class EngineUtilsFuncTestCase(TestCase): + + def test_calculate_elapsed_time(self): + self.assertEqual(calculate_elapsed_time(None, None), 0) + + self.assertEqual(calculate_elapsed_time(started_time=None, archived_time=timezone.now()), 0) + + self.assertNotEqual(calculate_elapsed_time(started_time=timezone.now() - datetime.timedelta(seconds=1), + archived_time=None), + 0) + + # seconds + start = timezone.now() + archive = start + datetime.timedelta(seconds=59) + + self.assertEqual(calculate_elapsed_time(started_time=start, archived_time=archive), 59) + + # minutes + start = timezone.now() + archive = start + datetime.timedelta(minutes=3) + + self.assertEqual(calculate_elapsed_time(started_time=start, archived_time=archive), 3 * 60) + + # hours + start = timezone.now() + archive = start + datetime.timedelta(hours=3) + + self.assertEqual(calculate_elapsed_time(started_time=start, archived_time=archive), 3 * 60 * 60) + + # days + start = timezone.now() + archive = start + datetime.timedelta(days=3) + + self.assertEqual(calculate_elapsed_time(started_time=start, archived_time=archive), 3 * 24 * 60 * 60) diff --git a/pipeline/tests/mock.py b/pipeline/tests/mock.py index 9c59d885bd..e3f4ecd5f2 100644 --- a/pipeline/tests/mock.py +++ b/pipeline/tests/mock.py @@ -51,8 +51,9 @@ def get_outputs(self): class IdentifyObject(object): - def __init__(self, id=None): + def __init__(self, id=None, name=None): self.id = id or uniqid() + self.name = name or '' class StartEventObject(IdentifyObject): diff --git a/pipeline/validators/base.py b/pipeline/validators/base.py index 5c13d85b61..9f4a492792 100644 --- a/pipeline/validators/base.py +++ b/pipeline/validators/base.py @@ -22,13 +22,16 @@ def validate_pipeline_tree(pipeline_tree, cycle_tolerate=False): # 1. connection validation - validate_graph_connection(pipeline_tree) + try: + validate_graph_connection(pipeline_tree) + except exceptions.ConnectionValidateError as e: + raise exceptions.ParserException(e.detail) # do not tolerate circle in flow if not cycle_tolerate: - result = find_graph_circle(pipeline_tree) - if not result['result']: - raise exceptions.CycleErrorException(result['message']) + no_cycle = find_graph_circle(pipeline_tree) + if not no_cycle['result']: + raise exceptions.ParserException(no_cycle['message']) # 2. gateway validation validate_gateways(pipeline_tree)