Merge remote-tracking branch 'base/master'

bcs-app/backend/apps/configuration/models/template.py

@@ -226,44 +226,12 @@ def copy_tempalte(self, project_id, new_template_name, username):
             if not item_ids:
                 continue
             item_id_list = item_ids.split(',')
-            item_ins_list = MODULE_DICT.get(
-                item).objects.filter(id__in=item_id_list)
+            item_ins_list = MODULE_DICT.get(item).objects.filter(id__in=item_id_list)
             new_item_id_list = []
             for _ins in item_ins_list:
                 # 复制资源
                 _ins.pk = None
                 _ins.save()
-                # 重命名资源名称
-                _ins_name = _ins.get_name
-                _ins_real_name = _ins_name.split(COPY_TEMPLATE)[0]
-                # 查看已经名称的个数
-                search_name = '%s%s' % (_ins_real_name, COPY_TEMPLATE)
-                search_list = MODULE_DICT.get(item).objects.filter(
-                    name__contains=search_name).values_list('name', flat=True)
-
-                # 获取重命名的后缀
-                _s_num_list = []
-                for _s_name in set(search_list):
-                    _s_num = _s_name.split(search_name)[-1]
-                    try:
-                        _s_num = int(_s_num)
-                        _s_num_list.append(_s_num)
-                    except Exception:
-                        pass
-                search_name_count = max(_s_num_list) + 1 if _s_num_list else 1
-                new_name = '%s%s%s' % (
-                    _ins_real_name, COPY_TEMPLATE, search_name_count)
-
-                # 需要修改 config 中的name信息
-                _new_config = _ins.get_config()
-                _new_config_name = _new_config.get('metadata', {}).get('name')
-                if _new_config_name:
-                    _new_config['metadata']['name'] = new_name
-
-                # 更新 db 中的name信息
-                _ins.config = json.dumps(_new_config)
-                _ins.name = new_name
-                _ins.save()
                 new_item_id_list.append(str(_ins.id))
 
             new_entity[item] = ','.join(new_item_id_list)

bcs-app/backend/apps/instance/generator.py

@@ -62,6 +62,7 @@
 from backend.apps.instance.resources.utils import handle_number_var
 from backend.apps.instance import constants as instance_constants
 from backend.utils.basic import getitems
+from backend.resources.constants import K8sServiceTypes
 
 logger = logging.getLogger(__name__)
 HANDLED_NUM_VAR_PATTERN = re.compile(r"%s}" % NUM_VAR_PATTERN)
@@ -1206,7 +1207,7 @@ def handel_k8s_service_db_config(db_config, deploy_tag_list, version_id,
 
     if not db_config.get('spec', {}).get('clusterIP', ''):
         remove_key(db_config['spec'], 'clusterIP')
-    elif s_type == 'NodePort' and not is_upadte:
+    elif s_type == K8sServiceTypes.NodePort.value and not is_upadte:
         remove_key(db_config['spec'], 'clusterIP')
     # Service 关联的端口应用的端口信
     # 获取所有端口中的id
@@ -1234,8 +1235,8 @@ def handel_k8s_service_db_config(db_config, deploy_tag_list, version_id,
                 _p['targetPort'] = int(_p['targetPort'])
             except Exception:
                 pass
-            # 只有 Service 类型为 NodePort 时才传 nodePort 字段
-            if s_type != 'NodePort':
+            # 只有 Service 类型为 NodePort或LoadBalancer类型 时才传 nodePort 字段
+            if s_type == K8sServiceTypes.ClusterIP.value:
                 remove_key(_p, "nodePort")
             else:
                 if not _p['nodePort']:

bcs-app/backend/apps/metric/urls_new.py

@@ -66,11 +66,7 @@
     ),
 ]
 
-metrics_urlpatterns = [
-    url(r"^servicemonitors/$", views.servicemonitor.ServiceMonitor.as_view({"get": "list", "post": "create"}),),
-]
 
 urlpatterns = [
     url(r"^clusters/(?P<cluster_id>[\w-]+)/metrics/", include(cluster_urlpatterns)),
-    url(r"^metrics/", include(metrics_urlpatterns)),
 ]

bcs-app/backend/apps/metric/views/servicemonitor.py

@@ -76,6 +76,13 @@ class ServiceMonitor(viewsets.ViewSet):
     SERVICE_NAME_LABEL = "io.tencent.bcs.service_name"
     TIME_DURATION_PATTERN = re.compile(r"^((?P<hours>\d+)h)?((?P<minutes>\d+)m)?((?P<seconds>\d+)s)?$")
 
+    def _get_client(self, request, project_id, cluster_id):
+        if request.project.kind == ProjectKind.MESOS.value:
+            client = mesos.MesosClient(request.user.token.access_token, project_id, cluster_id, env=None)
+        else:
+            client = k8s.K8SClient(request.user.token.access_token, project_id, cluster_id, env=None)
+        return client
+
     def get_duration(self, duration, default=None):
         """解析普罗米修斯时间范围
         """
@@ -176,25 +183,18 @@ def _get_namespace_map(self, project_id):
         namespace_map = {i["name"]: i["id"] for i in namespace_list}
         return namespace_map
 
-    def list(self, request, project_id, cluster_id=None):
-        access_token = request.user.token.access_token
+    def list(self, request, project_id, cluster_id):
         cluster_map = self._get_cluster_map(project_id)
         namespace_map = self._get_cluster_map(project_id)
         data = []
 
-        if cluster_id:
-            if cluster_id not in cluster_map:
-                raise error_codes.APIError(_("cluster_id not valid"))
-            client = k8s.K8SClient(access_token, project_id, cluster_id, env=None)
-            items = self._handle_items(cluster_id, cluster_map, namespace_map, client.list_service_monitor())
-            data.extend(items)
-        else:
-            for cluster in cluster_map.values():
-                cluster_id = cluster["cluster_id"]
-                cluster_env = cluster.get("environment")
-                client = k8s.K8SClient(access_token, project_id, cluster_id, env=cluster_env)
-                items = self._handle_items(cluster_id, cluster_map, namespace_map, client.list_service_monitor())
-                data.extend(items)
+        if cluster_id not in cluster_map:
+            raise error_codes.APIError(_("cluster_id not valid"))
+
+        client = self._get_client(request, project_id, cluster_id)
+
+        items = self._handle_items(cluster_id, cluster_map, namespace_map, client.list_service_monitor())
+        data.extend(items)
 
         perm = bcs_perm.Namespace(request, project_id, bcs_perm.NO_RES)
         data = perm.hook_perms(data, ns_id_flag="namespace_id")
@@ -240,7 +240,8 @@ def create(self, request, project_id, cluster_id=None):
             },
         }
 
-        client = k8s.K8SClient(request.user.token.access_token, project_id, cluster_id, env=None)
+        client = self._get_client(request, project_id, cluster_id)
+
         result = client.create_service_monitor(data["namespace"], spec)
         if result.get("status") == "Failure":
             message = _("创建Metrics:{}失败, [命名空间:{}], {}").format(
@@ -256,8 +257,7 @@ def create(self, request, project_id, cluster_id=None):
     def get(self, request, project_id, cluster_id, namespace, name):
         """获取单个serviceMonitor
         """
-        access_token = request.user.token.access_token
-        client = k8s.K8SClient(access_token, project_id, cluster_id, env=None)
+        client = self._get_client(request, project_id, cluster_id)
         result = client.get_service_monitor(namespace, name)
         if result.get("status") == "Failure":
             raise error_codes.APIError(result.get("message", ""))
@@ -275,8 +275,7 @@ def delete(self, request, project_id, cluster_id, namespace, name):
         """删除servicemonitor
         """
         self._validate_namespace_use_perm(request, project_id, [namespace])
-
-        client = k8s.K8SClient(request.user.token.access_token, project_id, cluster_id, env=None)
+        client = self._get_client(request, project_id, cluster_id)
         result = client.delete_service_monitor(namespace, name)
         if result.get("status") == "Failure":
             message = _("删除Metrics:{}失败, [命名空间:{}], {}").format(name, namespace, result.get("message", ""))
@@ -309,8 +308,7 @@ def update(self, request, project_id, cluster_id, namespace, name):
         data = slz.validated_data
 
         self._validate_namespace_use_perm(request, project_id, [namespace])
-
-        client = k8s.K8SClient(request.user.token.access_token, project_id, cluster_id, env=None)
+        client = self._get_client(request, project_id, cluster_id)
         result = client.get_service_monitor(namespace, name)
         if result.get("status") == "Failure":
             message = _("更新Metrics:{}失败, [命名空间:{}], {}").format(name, namespace, result.get("message", ""))
@@ -513,15 +511,19 @@ def _get_version(self, image):
     def get(self, request, project_id, cluster_id):
         """获取targets列表
         """
+        data = {"need_update": False, "update_tooltip": ""}
+
+        # Mesos不存在Prometheus CRD, 直接返回
+        if request.project.kind == ProjectKind.MESOS.value:
+            return Response(data)
+
         access_token = request.user.token.access_token
         client = k8s.K8SClient(access_token, project_id, cluster_id, env=None)
         resp = client.get_prometheus("thanos", "po-prometheus-operator-prometheus")
         spec = resp.get("spec") or {}
         if not spec:
             raise error_codes.APIError(_("Prometheus未安装，请联系管理员解决"))
 
-        data = {"need_update": False, "update_tooltip": ""}
-
         for container in spec.get("containers") or []:
             if container["name"] not in settings.PROMETHEUS_VERSIONS:
                 continue

bcs-app/backend/apps/network/utils_bk.py

@@ -18,6 +18,10 @@ def get_svc_extended_routes(project_id):
     return {}
 
 
+def delete_svc_extended_routes(request, project_id, cluster_id, namespace, svc_name):
+    pass
+
+
 def get_svc_access_info(manifest, cluster_id, extended_routes):
     """
     {

bcs-app/backend/apps/network/views/service.py

@@ -28,7 +28,7 @@
 from backend.apps.configuration.models import Template, Application, VersionedEntity, Service, ShowVersion, K8sService
 from backend.components.bcs import k8s, mesos
 from backend.apps import constants
-from backend.apps.network.utils_bk import get_svc_access_info, get_svc_extended_routes
+from backend.apps.network.utils_bk import get_svc_access_info, get_svc_extended_routes, delete_svc_extended_routes
 from backend.apps.instance.constants import (LABLE_TEMPLATE_ID, LABLE_INSTANCE_ID, SEVICE_SYS_CONFIG,
                                              ANNOTATIONS_CREATOR, ANNOTATIONS_UPDATOR, ANNOTATIONS_CREATE_TIME,
                                              ANNOTATIONS_UPDATE_TIME, ANNOTATIONS_WEB_CACHE, K8S_SEVICE_SYS_CONFIG,
@@ -358,6 +358,8 @@ def delete_single_service(self, request, project_id, project_kind, cluster_id, n
             resp = client.delete_service(namespace, name)
             s_cate = 'K8sService'
 
+            delete_svc_extended_routes(request, project_id, cluster_id, namespace, name)
+
         if resp.get("code") == ErrorCode.NoError:
             # 删除成功则更新状态
             now_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')

bcs-app/backend/apps/whitelist_bk.py

@@ -36,3 +36,9 @@ def enable_helm_v3(cluster_id: str) -> bool:
     """是否允许集群使用helm3功能
     """
     return False
+
+
+def enable_incremental_sync_chart_repo(project_id: str) -> bool:
+    """是否开启增量同步仓库数据
+    """
+    return False

bcs-app/backend/bcs_k8s/helm/tasks.py

@@ -17,10 +17,14 @@
 
 from celery import shared_task
 from natsort import natsorted
+from django.utils import timezone
 
 from .models.repo import Repository
 from .models.chart import Chart, ChartVersion
 from .utils.repo import (prepareRepoCharts, InProcessSign)
+from backend.apps.whitelist_bk import enable_incremental_sync_chart_repo
+from backend.bcs_k8s.helm.utils.repo_bk import get_incremental_charts_and_hash_value
+from backend.utils.basic import normalize_time
 
 logger = logging.getLogger(__name__)
 
@@ -43,6 +47,16 @@ def force_sync_all_repo():
             logger.exception("force sync_helm_repo %s failed %s" % (repo.id, e))
 
 
+def enable_increment(force, project_id):
+    """判断是否增量功能
+    NOTE: 当force为false，并且在白名单中才允许增量操作
+    """
+    if not force and enable_incremental_sync_chart_repo(project_id):
+        return True
+
+    return False
+
+
 @shared_task
 def sync_helm_repo(repo_id, force=False):
     # if in processing, then do nothing
@@ -58,7 +72,17 @@ def sync_helm_repo(repo_id, force=False):
     plain_auths = repo.plain_auths
 
     try:
-        charts_info, charts_info_hash = prepareRepoCharts(repo_url, repo_name, plain_auths)
+        # NOTE: 针对白名单中的项目先开启增量同步
+        if enable_increment(force, repo.project_id):
+            if not plain_auths:
+                username, password = None, None
+            else:
+                credentials = plain_auths[0]["credentials"]
+                username, password = credentials["username"], credentials["password"]
+            start_time = normalize_time(repo.refreshed_at)
+            charts_info, charts_info_hash = get_incremental_charts_and_hash_value(repo_url, username, password, start_time)
+        else:
+            charts_info, charts_info_hash = prepareRepoCharts(repo_url, repo_name, plain_auths)
     except Exception as e:
         logger.exception("prepareRepoCharts fail: repo_url=%s, repo_name=%s, error: %s", repo_url, repo_name, e)
         return
@@ -79,12 +103,50 @@ def sync_helm_repo(repo_id, force=False):
         return
 
     try:
-        _do_helm_repo_charts_update(repo, sign, charts_info, charts_info_hash, force)
+        # 增量获取的数据，直接添加到本地记录
+        if enable_increment(force, repo.project_id):
+            _add_charts(repo, sign, charts_info, charts_info_hash, force)
+        else:
+            _do_helm_repo_charts_update(repo, sign, charts_info, charts_info_hash, force)
     except Exception as e:
         logger.exception("_do_helm_repo_charts_update fail, error: %s", e)
         sign.delete()
 
 
+def _add_charts(repo, sign, charts, index_hash, force=False):
+    """添加charts
+    """
+    for chart_name, versions in charts.items():
+        chart, _created = Chart.objects.get_or_create(name=chart_name, repository=repo)
+        # 开始添加版本
+        full_chart_versions = {}
+        for version in versions:
+            # update sign every time
+            sign.update()
+            chart_version = ChartVersion()
+            try:
+                chart_version.update_from_import_version(chart, version, force)
+            except Exception as e:
+                logger.exception(
+                    "save_chart_version fail: chart=%s, version=%s, error: %s", chart, version, e)
+                continue
+            # 更新chart icon
+            icon_url = version.get("icon")
+            if not chart.icon and icon_url:
+                chart.update_icon(icon_url)
+            # 记录版本，便于更新chart默认版本
+            full_chart_versions[chart_version.id] = chart_version
+        # 更新chart的变更时间
+        chart.changed_at = timezone.now()
+        chart.save(update_fields=["changed_at"])
+        # 更新chart默认版本为最新推送的chart版本
+        _update_default_chart_version(chart, full_chart_versions)
+    # 更新hash
+    repo.refreshed(index_hash)
+    # delete sign
+    sign.delete()
+
+
 def _update_default_chart_version(chart, full_chart_versions):
     """更新chart对应的默认版本信息
     """

bcs-app/backend/bcs_k8s/helm/utils/repo_bk.py

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+#
+# Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community Edition) available.
+# Copyright (C) 2017-2019 THL A29 Limited, a Tencent company. All rights reserved.
+# Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://opensource.org/licenses/MIT
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+def get_incremental_charts_and_hash_value(url_prefix, username, password, start_time):
+    return None, None

bcs-app/backend/bcs_k8s/helm/views.py

@@ -192,8 +192,8 @@ class RepositorySyncView(FilterByProjectMixin, viewsets.ViewSet):
     def create(self, request, project_id, repo_id, *args, **kwargs):
         """Sync Chart Repository
         """
-
-        sync_helm_repo(repo_id, True)
+        # 默认不需要设置为强制同步
+        sync_helm_repo(repo_id, request.data.get("force_sync") or False)
 
         data = {
             "code": 0,