Merge pull request #7431 from normal-wls/dependencies_udpate_merge_ma…

…ster

Dependencies udpate merge master

app.yml

@@ -6,7 +6,7 @@ is_use_celery: True
 author: 蓝鲸智云
 introduction: 标准运维是通过一套成熟稳定的任务调度引擎，把在多系统间的工作整合到一个流程，助力运维实现跨系统调度自动化的SaaS应用。
 introduction_en: SOPS is a SaaS application that utilizes a set of mature and stable task scheduling engines to help realize cross-system scheduling automation, and integrates the work among multiple systems into a single process.
-version: 3.31.25
+version: 3.32.2
 category: 运维工具
 language_support: 中文
 desktop:

app_desc.yaml

@@ -1,5 +1,5 @@
 spec_version: 2
-app_version: "3.31.25"
+app_version: "3.32.2"
 app:
     region: default
     bk_app_code: bk_sops

config/default.py

@@ -213,7 +213,7 @@
 # mako模板中：<script src="/a.js?v=${ STATIC_VERSION }"></script>
 # 如果静态资源修改了以后，上线前改这个版本号即可
 
-STATIC_VERSION = "3.31.25"
+STATIC_VERSION = "3.32.2"
 DEPLOY_DATETIME = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
 
 STATICFILES_DIRS = [os.path.join(BASE_DIR, "static")]
@@ -846,3 +846,15 @@ def check_engine_admin_permission(request, *args, **kwargs):
 
 # 通知中心
 BK_NOTICE = {"BK_API_URL_TMPL": env.BK_APIGW_URL_TMPL or ""}
+
+# 周期任务自动关闭扫描频率
+PERIODIC_TASK_REMINDER_TIME = env.PERIODIC_TASK_REMINDER_TIME
+
+# 发送周期任务消息通知开关(默认关)
+PERIODIC_TASK_REMINDER_SWITCH = env.PERIODIC_TASK_REMINDER_SWITCH
+
+# 周期任务自动关闭扫描周期
+PERIODIC_TASK_REMINDER_SCAN_CRON = env.PERIODIC_TASK_REMINDER_SCAN_CRON
+
+# 周期任务消息通知类型
+PERIODIC_TASK_REMINDER_NOTIFY_TYPE = env.PERIODIC_TASK_REMINDER_NOTIFY_TYPE

env.py

@@ -127,3 +127,15 @@
     REMOTE_PLUGIN_FIX_INTERVAL_CODES_STR.split(",") if REMOTE_PLUGIN_FIX_INTERVAL_CODES_STR else []
 )
 REMOTE_PLUGIN_FIX_INTERVAL = int(os.getenv("BKAPP_REMOTE_PLUGIN_FIX_INTERVAL", 60))
+
+# 周期任务自动关闭扫描频率(单位：月)
+PERIODIC_TASK_REMINDER_TIME = int(os.getenv("BKAPP_PERIODIC_TASK_REMINDER_TIME", 1))
+
+# 发送周期任务消息通知开关
+PERIODIC_TASK_REMINDER_SWITCH = int(os.getenv("BKAPP_PERIODIC_TASK_REMINDER_SWITCH", 0))
+
+# 周期任务自动关闭扫描周期(每月1号10点20)
+PERIODIC_TASK_REMINDER_SCAN_CRON = tuple(os.getenv("BKAPP_PERIODIC_TASK_REMINDER_SCAN_CRON", "20 10 1 * *").split())
+
+# 周期任务消息通知类型
+PERIODIC_TASK_REMINDER_NOTIFY_TYPE = json.loads(os.getenv("PERIODIC_TASK_REMINDER_NOTIFY_TYPE", '["email"]'))

frontend/desktop/src/components/common/RenderForm/FormItem.vue

@@ -405,6 +405,11 @@
                     return '******'
                 }
 
+                // 元变量表单数据为Object格式时，取默认值
+                if (this.constants[this.scheme.tag_code]?.is_meta && checkDataType(val) === 'Object') {
+                    return val.default
+                }
+
                 if (defaultValueType.includes(valueType)) {
                     formValue = tools.deepClone(val)
                 } else {

gcloud/contrib/admin/urls.py

@@ -57,4 +57,5 @@
     url(r"^batch_insert_project_based_component/$", batch_insert_project_based_component),
     url(r"^batch_delete_project_based_component/$", batch_delete_project_based_component),
     url(r"^batch_revoke_task/$", batch_revoke_task),
+    url(r"^command/get_enabled_periodic_task/$", views.get_enabled_periodic_task),
 ]

gcloud/contrib/admin/views/command.py

@@ -27,7 +27,7 @@
 from gcloud.contrib.admin.utils import force_tasks
 from gcloud.core.decorators import check_is_superuser
 from gcloud.core.models import ProjectBasedComponent
-from gcloud.core.tasks import migrate_pipeline_parent_data_task
+from gcloud.core.tasks import migrate_pipeline_parent_data_task, scan_periodic_task
 from gcloud.openapi.schema import AnnotationAutoSchema
 from gcloud.taskflow3.models import TaskFlowInstance, TaskFlowRelation
 
@@ -197,3 +197,10 @@ def batch_revoke_task(request):
         return JsonResponse({"result": False, "data": [], "message": "存在终止异常的任务 = {}".format(revoke_failed_tasks)})
 
     return JsonResponse({"result": True, "data": [], "message": f"revoked -> {tasks.count()}"})
+
+
+@check_is_superuser()
+def get_enabled_periodic_task(request):
+    is_send_notify = True if request.GET.get("is_send_notify", "false") == "true" else False
+    data = scan_periodic_task(is_send_notify)
+    return JsonResponse({"result": True, "code": err_code.SUCCESS.code, "data": data})

gcloud/core/apis/drf/viewsets/periodic_task.py

@@ -182,7 +182,7 @@ def list(self, request, *args, **kwargs):
 
         if project_id:
             order_by = request.query_params.get("order_by") or "-id"
-            orderings = ("-is_collected", order_by)
+            orderings = ("-is_collected", "-task__celery_task__enabled", order_by)
 
             collection_task_ids, collection_task_map = Collection.objects.get_user_project_collection_instance_info(
                 project_id=project_id, username=request.user.username, category="periodic_task"
@@ -268,5 +268,7 @@ def partial_update(self, request, *args, **kwargs):
                 instance.task.save(update_fields=["name"])
             instance.editor = request.user.username
             instance.save(update_fields=["editor", "edit_time"])
+            instance.task.creator = request.user.username
+            instance.task.save()
 
         return Response(PeriodicTaskReadOnlySerializer(instance=instance).data)

gcloud/core/tasks.py

@@ -11,24 +11,31 @@
 specific language governing permissions and limitations under the License.
 """
 
+import datetime
 import logging
 import time
 import traceback
 from contextlib import contextmanager
 from time import monotonic
 
+import dateutil.relativedelta
 from blueapps.contrib.celery_tools.periodic import periodic_task
 from celery import current_app
+from celery.schedules import crontab
 from django.contrib.sessions.models import Session
 from django.core.cache import cache
+from django.template.loader import render_to_string
 from django.utils import timezone
+from django_celery_beat.models import PeriodicTask as DjangoCeleryBeatPeriodicTask
 from pipeline.contrib.periodic_task.djcelery.tzcrontab import TzAwareCrontab
 from pipeline.engine.core.data.api import _backend, _candidate_backend
 from pipeline.engine.core.data.redis_backend import RedisDataBackend
 
 from gcloud import exceptions
 from gcloud.conf import settings
 from gcloud.core.project import sync_projects_from_cmdb
+from gcloud.periodictask.models import PeriodicTask
+from gcloud.shortcuts.message.send_msg import send_message
 
 logger = logging.getLogger("celery")
 
@@ -120,3 +127,99 @@ def migrate_pipeline_parent_data_task():
             logger.exception("[migrate_pipeline_parent_data] {} key migrate err.".format(i))
 
     logger.info("[migrate_pipeline_parent_data] migrate done!")
+
+
+@periodic_task(run_every=TzAwareCrontab(minute=1, hour="*/4"))
+def cmdb_business_sync_shutdown_periodic_task():
+    task_id = cmdb_business_sync_shutdown_periodic_task.request.id
+    with redis_lock(LOCK_ID, task_id) as acquired:
+        if acquired:
+            try:
+                task_ids = [
+                    item["task__celery_task__id"]
+                    for item in PeriodicTask.objects.filter(project__is_disable=True).values("task__celery_task__id")
+                ]
+                logger.info("[shutdown_periodic_task] disabled the deleted periodic task: {}".format(task_ids))
+                DjangoCeleryBeatPeriodicTask.objects.filter(id__in=task_ids).update(enabled=False)
+            except Exception as e:
+                logger.exception(
+                    "[shutdown_periodic_task] closing periodic task from cmdb, message: {msg}".format(msg=str(e))
+                )
+        else:
+            logger.info("[shutdown_periodic_task] Can not get sync_business lock, sync operation abandon")
+
+
+@current_app.task
+def send_periodic_task_notify(executor, notify_type, receivers, title, content):
+    try:
+        send_message(executor, notify_type, receivers, title, content)
+    except Exception as e:
+        logger.exception(f"send periodic task notify error: {e}")
+
+
+@periodic_task(run_every=(crontab(*settings.PERIODIC_TASK_REMINDER_SCAN_CRON)))
+def scan_periodic_task(is_send_notify: bool = True):
+    if not settings.PERIODIC_TASK_REMINDER_SWITCH:
+        return
+    title = "【标准运维 APP 提醒】请确认您正在运行的周期任务状态"
+    # 以执行人维度发邮件通知
+    periodic_tasks = (
+        PeriodicTask.objects.filter(task__celery_task__enabled=True)
+        .values(
+            "id",
+            "task__creator",
+            "project__id",
+            "project__name",
+            "edit_time",
+            "task__last_run_at",
+            "task__name",
+            "task__total_run_count",
+        )
+        .order_by("-edit_time")
+    )
+    data = {}
+    user_task_id = {}
+    for p_task in periodic_tasks:
+        last_month_time = datetime.datetime.now() + dateutil.relativedelta.relativedelta(
+            months=-int(settings.PERIODIC_TASK_REMINDER_TIME)
+        )
+        if last_month_time.timestamp() < p_task["edit_time"].timestamp():
+            continue
+        creator = p_task["task__creator"]
+        project_name = p_task["project__name"]
+        task_dict = {
+            "edit_time": p_task["edit_time"],
+            "last_run_at": p_task["task__last_run_at"],
+            "name": p_task["task__name"],
+            "total_run_count": p_task["task__total_run_count"],
+            "task_link": settings.BK_SOPS_HOST.rstrip("/")
+            + f"/taskflow/home/periodic/{p_task['project__id']}/?limit=15&page=1&task_id={p_task['id']}",
+        }
+        if creator in data:
+            if project_name in data[creator]:
+                data[creator][project_name].append(task_dict)
+            else:
+                data[creator][project_name] = [task_dict]
+            user_task_id[creator].add(p_task["id"])
+        else:
+            data[creator] = {project_name: [task_dict]}
+            user_task_id[creator] = {p_task["id"]}
+    # 发送通知
+    if is_send_notify:
+        for notifier, tasks in data.items():
+            logger.info(f"{notifier} has {user_task_id[notifier]} tasks")
+            try:
+                mail_content = render_to_string(
+                    "core/period_task_notice_mail.html",
+                    {
+                        "notifier": notifier,
+                        "period_task_times": settings.PERIODIC_TASK_REMINDER_TIME,
+                        "task_projects": tasks,
+                    },
+                )
+                send_periodic_task_notify.delay(
+                    "admin", settings.PERIODIC_TASK_REMINDER_NOTIFY_TYPE, notifier, title, mail_content
+                )
+            except Exception as e:
+                logger.exception(f"send periodic task notify error: {e}")
+    return data