feat: modify vul save logic

dongtai_common/migrations/0015_vul_status.py

@@ -0,0 +1,18 @@
+from django.db import migrations
+
+
+def update_vul_status(apps, schema_editor):
+    # We can't import the Person model directly as it may be a newer
+    # version than this migration expects. We use the historical version.
+    IastVulnerabilityStatus = apps.get_model("dongtai_common", "IastVulnerabilityStatus")
+    IastVulnerabilityStatus(id=7, name="已忽略", name_zh="已忽略", name_en="Ignored").save()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("dongtai_common", "0014_auto_20230828_1132"),
+    ]
+
+    operations = [
+        migrations.RunPython(update_vul_status),
+    ]

dongtai_common/migrations/0016_auto_20230829_1145.py

@@ -0,0 +1,59 @@
+# Generated by Django 3.2.20 on 2023-08-29 11:45
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("dongtai_common", "0015_vul_status"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="iastvulnerabilitymodel",
+            name="bottom_stack",
+            field=models.CharField(blank=True, default="", max_length=255),
+        ),
+        migrations.AlterField(
+            model_name="iastvulnerabilitymodel",
+            name="full_stack",
+            field=models.TextField(blank=True, default=""),
+        ),
+        migrations.AlterField(
+            model_name="iastvulnerabilitymodel",
+            name="language",
+            field=models.CharField(blank=True, default="", max_length=10),
+        ),
+        migrations.AlterField(
+            model_name="iastvulnerabilitymodel",
+            name="param_name",
+            field=models.CharField(blank=True, default="", max_length=255),
+        ),
+        migrations.AlterField(
+            model_name="iastvulnerabilitymodel",
+            name="pattern_uri",
+            field=models.CharField(blank=True, default="", max_length=255),
+        ),
+        migrations.AlterField(
+            model_name="iastvulnerabilitymodel",
+            name="taint_position",
+            field=models.CharField(blank=True, default="", max_length=255),
+        ),
+        migrations.AlterField(
+            model_name="iastvulnerabilitymodel",
+            name="taint_value",
+            field=models.CharField(blank=True, default="", max_length=255),
+        ),
+        migrations.AlterField(
+            model_name="iastvulnerabilitymodel",
+            name="top_stack",
+            field=models.CharField(blank=True, default="", max_length=255),
+        ),
+        migrations.AddIndex(
+            model_name="iastvulnerabilitymodel",
+            index=models.Index(
+                fields=["http_method", "param_name", "pattern_uri", "project_id", "status_id", "strategy_id"],
+                name="iast_vulner_http_me_f84d4f_idx",
+            ),
+        ),
+    ]

dongtai_common/models/vulnerablity.py

@@ -34,34 +34,32 @@ class IastVulnerabilityModel(models.Model):
     level = models.ForeignKey(IastVulLevel, models.DO_NOTHING, blank=True)
     url = models.CharField(max_length=2000, blank=True)
     uri = models.CharField(max_length=255, blank=True)
-    pattern_uri = models.CharField(max_length=255, blank=True, null=True)
+    pattern_uri = models.CharField(max_length=255, blank=True, default="")
     # 模糊搜索 全文索引 查询
     vul_title = models.CharField(max_length=255, blank=True, default="")
     http_method = models.CharField(max_length=10, blank=True)
     http_scheme = models.CharField(max_length=255, blank=True)
     http_protocol = models.CharField(max_length=255, blank=True)
     req_header = models.TextField(blank=True)
     req_params = models.CharField(max_length=2000, blank=True, default="")
-    req_data = models.TextField(
-        blank=True,
-    )
+    req_data = models.TextField(blank=True)
     res_header = models.TextField(blank=True)
     res_body = models.TextField(blank=True)
-    full_stack = models.TextField(blank=True, null=True)
-    top_stack = models.CharField(max_length=255, blank=True, null=True)
-    bottom_stack = models.CharField(max_length=255, blank=True, null=True)
-    taint_value = models.CharField(max_length=255, blank=True, null=True)
-    taint_position = models.CharField(max_length=255, blank=True, null=True)
+    full_stack = models.TextField(blank=True, default="")
+    top_stack = models.CharField(max_length=255, blank=True, default="")
+    bottom_stack = models.CharField(max_length=255, blank=True, default="")
+    taint_value = models.CharField(max_length=255, blank=True, default="")
+    taint_position = models.CharField(max_length=255, blank=True, default="")
     agent = models.ForeignKey(IastAgent, models.DO_NOTHING, blank=True)
-    language = models.CharField(max_length=10, blank=True, null=True)
+    language = models.CharField(max_length=10, blank=True, default="")
     context_path = models.CharField(max_length=255, blank=True)
     counts = models.IntegerField(blank=True)
     first_time = models.IntegerField(blank=True)
     latest_time = models.IntegerField(blank=True)
     latest_time_desc = models.IntegerField(blank=True, default=0)
     level_id_desc = models.SmallIntegerField(blank=True, default=0)
     client_ip = models.CharField(max_length=255, blank=True)
-    param_name = models.CharField(max_length=255, blank=True, null=True)
+    param_name = models.CharField(max_length=255, blank=True, default="")
     is_del = models.SmallIntegerField(blank=True, default=0)
     method_pool_id = models.IntegerField(default=-1, blank=True)
     strategy = models.ForeignKey(
@@ -90,6 +88,9 @@ class IastVulnerabilityModel(models.Model):
     class Meta:
         managed = get_managed()
         db_table = "iast_vulnerability"
+        indexes = [
+            models.Index(fields=("http_method", "param_name", "pattern_uri", "project_id", "status_id", "strategy_id")),
+        ]
 
     def save(self, *args, **kwargs):
         key_works = [

dongtai_common/utils/const.py

@@ -84,8 +84,8 @@
 VUL_PENDING = 1
 VUL_VERIFYING = 2
 VUL_CONFIRMED = 3
-VUL_IGNORE = 4
 VUL_SOLVED = 5
+VUL_IGNORE = 6
 
 
 # API 操作 tag

dongtai_engine/plugins/strategy_headers.py

@@ -121,7 +121,7 @@ def check_response_header(method_pool):
         )
 
 
-def save_vul(vul_type, method_pool, position=None, data=None):
+def save_vul(vul_type, method_pool, position="", data=""):
     if is_strategy_enable(vul_type, method_pool) is False:
         return
     vul_strategy = IastStrategyModel.objects.filter(
@@ -206,9 +206,9 @@ def save_vul(vul_type, method_pool, position=None, data=None):
             req_data=method_pool.req_data,
             res_header=method_pool.res_header,
             res_body=method_pool.res_body,
-            full_stack=None,
-            top_stack=None,
-            bottom_stack=None,
+            full_stack="",
+            top_stack="",
+            bottom_stack="",
             taint_value=data,
             taint_position=position,
             agent=method_pool.agent,
@@ -218,7 +218,7 @@ def save_vul(vul_type, method_pool, position=None, data=None):
             first_time=method_pool.create_time,
             latest_time=timestamp,
             client_ip=method_pool.clent_ip,
-            param_name=None,
+            param_name="",
             method_pool_id=method_pool.id,
             project_version_id=method_pool.agent.project_version_id,
             project_id=method_pool.agent.bind_project_id,

dongtai_engine/signals/handlers/vul_handler.py

@@ -266,6 +266,17 @@ def save_vul(vul_meta, vul_level, strategy_id, vul_stack, top_stack, bottom_stac
     is_api_cached = uuid_key != cache.get_or_set(cache_key, uuid_key)
     if is_api_cached:
         return None
+
+    if IastVulnerabilityModel.objects.filter(
+        strategy_id=strategy_id,
+        pattern_uri=pattern_uri,
+        http_method=vul_meta.http_method,
+        project_id=vul_meta.agent.bind_project_id,
+        param_name=param_name,
+        status_id=const.VUL_IGNORE,
+    ).exists():
+        return None
+
     # 获取 相同项目版本下的数据
     vul = (
         IastVulnerabilityModel.objects.filter(
@@ -300,6 +311,7 @@ def save_vul(vul_meta, vul_level, strategy_id, vul_stack, top_stack, bottom_stac
         vul.method_pool_id = vul_meta.id
         vul.language = vul_meta.agent.language
         vul.full_stack = json.dumps(vul_stack, ensure_ascii=False)
+        vul.is_del = 0
         vul.save(
             update_fields=[
                 "url",
@@ -320,6 +332,7 @@ def save_vul(vul_meta, vul_level, strategy_id, vul_stack, top_stack, bottom_stac
                 "latest_time",
                 "latest_time_desc",
                 "language",
+                "is_del",
             ]
         )
     else: