Feature remote import (#114)

* feature: 添加非标准存储代码导入器及 git 仓库代码导入器


* feature: 添加远程包源模型

* feature: 添加加载远程模块相关逻辑代码

* minor: 添加加载远程插件 APP

* feature: pipeline.utils.importer 添加导入器上下文

* minors: 完善 external_plugins app 的单元测试

* feature: GitRepoImporter 添加强制 HTTPS 开关

* minor: 添加远程包源配置获取语句

* minor: 单元测试完善，Importer Logger 配置修改，GitImporter 支持配置代理

config/default.py

@@ -62,6 +62,7 @@
     'pipeline.log',
     'pipeline.contrib.statistics',
     'pipeline.contrib.periodic_task',
+    'pipeline.contrib.external_plugins',
     'django_signal_valve',
     'pipeline_plugins',
     'pipeline_plugins.components',
@@ -103,7 +104,7 @@
 )
 
 # 所有环境的日志级别可以在这里配置
-# LOG_LEVEL = 'INFO'
+LOG_LEVEL = 'INFO'
 
 # 静态资源文件(js,css等）在APP上线更新后, 由于浏览器有缓存,
 # 可能会造成没更新的情况. 所以在引用静态资源的地方，都把这个加上
@@ -132,6 +133,12 @@
 # load logging settings
 LOGGING = get_logging_config_dict(locals())
 
+LOGGING['loggers']['pipeline'] = {
+    'handlers': ['root'],
+    'level': LOG_LEVEL,
+    'propagate': True,
+}
+
 # 初始化管理员列表，列表中的人员将拥有预发布环境和正式环境的管理员权限
 # 注意：请在首次提测和上线前修改，之后的修改将不会生效
 INIT_SUPERUSER = []

pipeline/contrib/external_plugins/__init__.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+"""
+Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community
+Edition) available.
+Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+
+default_app_config = 'pipeline.contrib.external_plugins.apps.ExternalPluginsConfig'

pipeline/contrib/external_plugins/admin.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+"""
+Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community
+Edition) available.
+Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+
+from django.contrib import admin
+
+from pipeline.contrib.external_plugins.models import (
+    GitRepoSource,
+    S3Source,
+    FileSystemSource
+)
+
+
+# Register your models here.
+
+@admin.register(GitRepoSource)
+class GitRepoSourceAdmin(admin.ModelAdmin):
+    list_display = ['name', 'from_config', 'repo_raw_address', 'branch']
+    search_fields = ['name', 'branch', 'repo_raw_address']
+
+
+@admin.register(S3Source)
+class S3SourceAdmin(admin.ModelAdmin):
+    list_display = ['name', 'from_config', 'service_address', 'bucket']
+    search_fields = ['name', 'bucket', 'service_address']
+
+
+@admin.register(FileSystemSource)
+class FileSystemSourceAdmin(admin.ModelAdmin):
+    list_display = ['name', 'from_config', 'path']
+    search_fields = ['name', 'path']

pipeline/contrib/external_plugins/apps.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+"""
+Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community
+Edition) available.
+Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+
+import sys
+import logging
+import traceback
+
+from django.apps import AppConfig
+from django.conf import settings
+from django.db.utils import ProgrammingError
+
+logger = logging.getLogger('root')
+
+
+class ExternalPluginsConfig(AppConfig):
+    name = 'pipeline.contrib.external_plugins'
+
+    def ready(self):
+        from pipeline.contrib.external_plugins import loader  # noqa
+        from pipeline.contrib.external_plugins.models import ExternalPackageSource  # noqa
+
+        triggers = getattr(settings, 'EXTERNAL_COMPONENTS_LOAD_TRIGGER', {'runserver', 'celery', 'worker'})
+        command = sys.argv[1]
+
+        if command in triggers:
+            try:
+                logger.info('Start to update package source from config file...')
+                ExternalPackageSource.update_package_source_from_config(getattr(settings,
+                                                                                'COMPONENTS_PACKAGE_SOURCES',
+                                                                                {}))
+            except ProgrammingError:
+                logger.warning('update package source failed, maybe first migration? exception: %s' %
+                               traceback.format_exc())
+                # first migrate
+                return
+
+            logger.info('Start to load external modules...')
+
+            loader.load_external_modules()

pipeline/contrib/external_plugins/exceptions.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+"""
+Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community
+Edition) available.
+Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+
+
+class InvalidOperationException(Exception):
+    pass

pipeline/contrib/external_plugins/loader.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+"""
+Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community
+Edition) available.
+Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+
+import logging
+import importlib
+import traceback
+
+from pipeline.contrib.external_plugins.models import source_cls_factory
+from pipeline.contrib.external_plugins.utils.importer import importer_context
+
+logger = logging.getLogger('root')
+
+
+def load_external_modules():
+    for source_type, source_model_cls in source_cls_factory.items():
+        # get all external source
+        sources = source_model_cls.objects.all()
+
+        # get importer for source
+        for source in sources:
+            _import_modules_in_source(source)
+
+
+def _import_modules_in_source(source):
+    importer = source.importer()
+
+    with importer_context(importer):
+        for module in source.modules:
+            try:
+                importlib.import_module(module)
+            except Exception as e:
+                logger.error('An error occurred when loading {%s}: %s' % (module, traceback.format_exc()))
+                raise e

pipeline/contrib/external_plugins/migrations/0001_initial.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+"""
+Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community
+Edition) available.
+Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import pipeline.contrib.external_plugins.models.fields
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='FileSystemSource',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=128, unique=True, verbose_name='\u5305\u6e90\u540d')),
+                ('from_config', models.BooleanField(default=False, verbose_name='\u662f\u5426\u662f\u4ece\u914d\u7f6e\u6587\u4ef6\u4e2d\u8bfb\u53d6\u7684')),
+                ('packages', pipeline.contrib.external_plugins.models.fields.JSONTextField(verbose_name='\u6a21\u5757\u914d\u7f6e')),
+                ('path', models.TextField(verbose_name='\u6587\u4ef6\u7cfb\u7edf\u8def\u5f84')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='GitRepoSource',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=128, unique=True, verbose_name='\u5305\u6e90\u540d')),
+                ('from_config', models.BooleanField(default=False, verbose_name='\u662f\u5426\u662f\u4ece\u914d\u7f6e\u6587\u4ef6\u4e2d\u8bfb\u53d6\u7684')),
+                ('packages', pipeline.contrib.external_plugins.models.fields.JSONTextField(verbose_name='\u6a21\u5757\u914d\u7f6e')),
+                ('repo_raw_address', models.TextField(verbose_name='\u6587\u4ef6\u6258\u7ba1\u4ed3\u5e93\u94fe\u63a5')),
+                ('branch', models.CharField(max_length=128, verbose_name='\u5206\u652f\u540d')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='S3Source',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=128, unique=True, verbose_name='\u5305\u6e90\u540d')),
+                ('from_config', models.BooleanField(default=False, verbose_name='\u662f\u5426\u662f\u4ece\u914d\u7f6e\u6587\u4ef6\u4e2d\u8bfb\u53d6\u7684')),
+                ('packages', pipeline.contrib.external_plugins.models.fields.JSONTextField(verbose_name='\u6a21\u5757\u914d\u7f6e')),
+                ('service_address', models.TextField(verbose_name='\u5bf9\u8c61\u5b58\u50a8\u670d\u52a1\u5730\u5740')),
+                ('bucket', models.TextField(verbose_name='bucket \u540d')),
+                ('access_key', models.TextField(verbose_name='access key')),
+                ('secret_key', models.TextField(verbose_name='secret key')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+    ]

pipeline/contrib/external_plugins/migrations/__init__.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+"""
+Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community
+Edition) available.
+Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""

pipeline/contrib/external_plugins/models/__init__.py

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+"""
+Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community
+Edition) available.
+Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+
+from pipeline.contrib.external_plugins.models.base import source_cls_factory  # noqa
+from pipeline.contrib.external_plugins.models.source import *  # noqa