feature: 添加远程包源同步到本地缓存任务 TencentBlueKing#275

gcloud/external_plugins/apps.py

@@ -19,3 +19,6 @@
 class ExternalPluginsConfig(AppConfig):
     name = 'gcloud.external_plugins'
     verbose_name = 'GcloudExternalPlugins'
+
+    def ready(self):
+        from gcloud.external_plugins.signals.handlers import sync_task_post_save_handler  # noqa

gcloud/external_plugins/models/__init__.py

@@ -19,3 +19,9 @@
     source_cls_factory
 )
 from gcloud.external_plugins.models.cache import CachePackageSource  # noqa
+from gcloud.external_plugins.models.sync import (  # noqa
+    SyncTask,
+    RUNNING,
+    SUCCEEDED,
+    FAILED
+)

gcloud/external_plugins/models/sync.py

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+"""
+Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community
+Edition) available.
+Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+
+from pipeline.contrib.external_plugins.models.fields import JSONTextField
+
+from gcloud.utils import convert_readable_username
+
+RUNNING = 'RUNNING'
+SUCCEEDED = 'SUCCEEDED'
+FAILED = 'FAILED'
+
+SYNC_TASK_STATUS = [
+    (RUNNING, _(u"执行中")),
+    (SUCCEEDED, _(u"成功")),
+    (FAILED, _(u"失败"))
+]
+
+SYNC_TASK_CREATED = [
+    ('manual', _(u"手动触发")),
+    ('auto', _(u"部署自动触发"))
+]
+
+
+class SyncTask(models.Model):
+    creator = models.CharField(_(u"执行者"), max_length=32, blank=True)
+    create_method = models.CharField(_(u"创建方式"), defualt='manual', choices=SYNC_TASK_CREATED)
+    start_time = models.DateTimeField(_(u"启动时间"), auto_now_add=True)
+    finish_time = models.DateTimeField(_(u"结束时间"), null=True, blank=True)
+    status = models.CharField(_(u"同步状态"), defualt=RUNNING, choices=SYNC_TASK_STATUS)
+    details = JSONTextField(_(u"同步详情信息"), blank=True)
+
+    class Meta:
+        verbose_name = _(u"远程包源同步任务 SyncTask")
+        verbose_name_plural = _(u"远程包源同步任务 SyncTask")
+        ordering = ['-id']
+
+    @property
+    def creator_name(self):
+        return convert_readable_username(self.creator)
+
+    @property
+    def status_display(self):
+        return self.get_status_display()

gcloud/external_plugins/signals/__init__.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+"""
+Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community
+Edition) available.
+Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""

gcloud/external_plugins/signals/handlers.py

@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+"""
+Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community
+Edition) available.
+Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+
+import logging
+
+from django.db.models.signals import post_save
+from django.dispatch import receiver
+from celery import task
+
+from gcloud.external_plugins.models import (
+    source_cls_factory,
+    CachePackageSource,
+    SyncTask,
+    SUCCEEDED,
+    FAILED
+)
+
+
+logger = logging.getLogger('celery')
+
+
+@receiver(post_save, sender=SyncTask)
+def sync_task_post_save_handler(sender, instance, created, **kwargs):
+    if created:
+        sync_task.delay(task_id=instance.id)
+
+
+@task
+def sync_task(task_id):
+    for origin_model in source_cls_factory.values():
+        origins = origin_model.objects.all()
+        for origin in origins:
+            try:
+                origin.read()
+            except Exception as e:
+                SyncTask.objects.filter(id=task_id).update(status=FAILED)
+                logger.error('Origin package[type={origin_type, id={origin_id}}] read error: {error}'.format(
+                    origin_type=origin.type,
+                    origin_id=origin.id,
+                    error=e.message
+                ))
+                return False
+
+    caches = CachePackageSource.objects.all()
+    for cache in caches:
+        try:
+            cache.write()
+        except Exception as e:
+            SyncTask.objects.filter(id=task_id).update(status=FAILED)
+            logger.error('Cache package[type={origin_type, id={origin_id}}] write error: {error}'.format(
+                origin_type=cache.type,
+                origin_id=cache.id,
+                error=e.message
+            ))
+            return False
+
+    SyncTask.objects.filter(id=task_id).update(status=SUCCEEDED)
+    return True