Merge pull request #179 from LoRexxar/develop

- 2021-08-13
    - KunLun-M 2.6.1
    - 更新了Web模式页面以适配组件漏洞扫描结果展示
    - 修复了数据可能会导致重复入库的bug
    - 更新了web模式的api数据
    - 为基础扫描添加去重功能，现在同一文件泄露的同一问题会被去重。
    - 为JS的语义分析扫描添加硬限制以应对混淆代码。

Kunlun_M/middleware.py

@@ -10,7 +10,7 @@
 '''
 
 
-from web.index.models import ScanTask, ScanResultTask, Rules, Tampers, Project
+from web.index.models import ScanTask, ScanResultTask, Rules, Tampers, Project, VendorVulns
 
 
 class SDataMiddleware:
@@ -21,12 +21,13 @@ def __call__(self, request):
         response = self.get_response(request)
 
         if request.user.is_authenticated:
-            request.session["rules_count"] = Rules.objects.all().count()
-            request.session["project_count"] = Rules.objects.all().count()
-            request.session["tasks_count"] = ScanTask.objects.all().count()
+            request.session["rules_count"] = Rules.objects.count()
+            request.session["project_count"] = Project.objects.count()
+            request.session["tasks_count"] = ScanTask.objects.count()
             request.session["tasks_finished_count"] = ScanTask.objects.filter(is_finished=True).count()
             request.session["tampers_count"] = Tampers.objects.all().count()
+            request.session["vendor_vuls_count"] = VendorVulns.objects.count()
 
-            request.session["vul_count"] = ScanResultTask.objects.all().count()
+            request.session["vul_count"] = ScanResultTask.objects.filter(is_active=1).count()
 
         return response

core/__init__.py

@@ -21,7 +21,8 @@
 
 from django.core.management import call_command
 from utils.log import log, logger, log_add, log_rm
-from utils.utils import get_mainstr_from_filename, get_scan_id
+from utils.utils import get_mainstr_from_filename
+from utils.status import get_scan_id
 from utils.web import upload_log
 from utils.file import load_kunlunmignore
 

core/__version__.py

@@ -7,7 +7,7 @@
 __issue_page__ = 'https://github.com/LoRexxar/Kunlun-M/issues/new'
 __python_version__ = sys.version.split()[0]
 __platform__ = platform.platform()
-__version__ = '2.6.0'
+__version__ = '2.6.1'
 __author__ = 'LoRexxar'
 __author_email__ = 'LoRexxar@gmail.com'
 __license__ = 'MIT License'

core/cli.py

@@ -23,7 +23,7 @@
 
 from core.pretreatment import ast_object
 from utils.export import write_to_file
-from utils.log import logger
+from utils.log import logger, logger_console
 from utils.file import Directory, load_kunlunmignore
 from utils.utils import show_context
 from utils.utils import ParseArgs
@@ -140,7 +140,8 @@ def display_result(scan_id, is_ask=False):
             logger.info("[Chain] Vul {}".format(sr.id))
             for rf in rfs:
                 logger.info("[Chain] {}, {}, {}:{}".format(rf.node_type, rf.node_content, rf.node_path, rf.node_lineno))
-                show_context(rf.node_path, rf.node_lineno)
+                if not show_context(rf.node_path, rf.node_lineno):
+                    logger_console.info(rf.node_source)
 
             logger.info(
                 "[SCAN] ending\r\n -------------------------------------------------------------------------")

core/console.py

@@ -27,7 +27,8 @@
 
 from utils.log import logger, logger_console, log, log_add
 from utils import readlineng as readline
-from utils.utils import get_mainstr_from_filename, get_scan_id, file_output_format, show_context
+from utils.utils import get_mainstr_from_filename, file_output_format, show_context
+from utils.status import get_scan_id
 
 from Kunlun_M.settings import HISTORY_FILE_PATH, MAX_HISTORY_LENGTH
 from Kunlun_M.settings import RULES_PATH, PROJECT_DIRECTORY, LOGS_PATH
@@ -1222,17 +1223,21 @@ def command_show(self, *args, **kwargs):
 
                                 # show Vuls Chain
                                 ResultFlow = get_resultflow_class(int(self.result_task_id))
-                                rfs = ResultFlow.objects.filter(vul_id=sr.id)
-
-                                if rfs:
-                                    logger.info("[Chain] Vul {}".format(sr.id))
-                                    for rf in rfs:
-                                        logger.info("[Chain] {}, {}, {}:{}".format(rf.node_type, rf.node_content,
-                                                                                   rf.node_path, rf.node_lineno))
-                                        show_context(rf.node_path, rf.node_lineno)
-                                    logger.info("[SCAN] ending\r\n -------------------------------------------------------------------------")
-                                    logger.warn("[Console] Use 'del vuls <id>' could delete Wrong vul.")
-                                    return
+
+                                if ResultFlow:
+                                    rfs = ResultFlow.objects.filter(vul_id=sr.id)
+
+                                    if rfs:
+                                        logger.info("[Chain] Vul {}".format(sr.id))
+                                        for rf in rfs:
+                                            logger.info("[Chain] {}, {}, {}:{}".format(rf.node_type, rf.node_content,
+                                                                                       rf.node_path, rf.node_lineno))
+                                            if not show_context(rf.node_path, rf.node_lineno):
+                                                logger_console.info(rf.node_source)
+
+                                        logger.info("[SCAN] ending\r\n -------------------------------------------------------------------------")
+                                        logger.warn("[Console] Use 'del vuls <id>' could delete Wrong vul.")
+                                        return
 
                             else:
                                 logger.error("[Console] ScanTask {} not found id {}. please check you result id.".format(self.result_task_id, key))

core/core_engine/php/parser.py

@@ -19,7 +19,7 @@
 # from asgiref.sync import sync_to_async, async_to_sync
 
 from utils.log import logger
-from utils.utils import SCAN_ID
+from utils.status import SCAN_ID
 
 from core.pretreatment import ast_object
 from core.internal_defines.php.functions import function_dict as php_function_dict

core/engine.py

@@ -38,7 +38,7 @@
 from utils.utils import show_context
 from utils.file import FileParseAll, get_line
 from utils.log import logger
-from utils.utils import get_scan_id
+from utils.status import get_scan_id
 
 from web.index.models import ScanResultTask, NewEvilFunc
 from web.index.models import get_resultflow_class, check_update_or_new_scanresult

core/pretreatment.py

@@ -344,6 +344,9 @@ async def pre_ast(self):
                     new_filepath = filepath + ".pretty"
 
                     try:
+                        # 添加新限制，如果js文件内容大于一定程度，则不解析
+                        if len(code_content) > 3000 or code_content.count('\n') > 500 or code_content.count('\n') < 10:
+                            continue
 
                         if not os.path.isfile(new_filepath):
                             fi2 = codecs.open(new_filepath, "w", encoding='utf-8', errors='ignore')

core/vendors.py

@@ -23,54 +23,12 @@
 
 from utils.log import logger
 from utils.file import check_filepath
+from utils.utils import compare_vendor, abstract_version
 
 from Kunlun_M.const import VENDOR_FILE_DICT, VENDOR_CVIID, vendor_source_match
 
 from web.index.models import ProjectVendors, update_and_new_project_vendor, update_and_new_vendor_vuln
-from web.index.models import Project, VendorVulns, check_update_or_new_scanresult
-
-
-def abstract_version(vendor_version):
-    version_reg = '([0-9]+(\.[0-9]+)*)'
-    result_version = ''
-
-    if re.search(version_reg, vendor_version, re.I):
-
-        p = re.compile(version_reg)
-        matchs = p.finditer(vendor_version)
-
-        for match in matchs:
-            result_version = match.group(1)
-    else:
-        result_version = False
-
-    return result_version
-
-
-def compare_vendor(vendor_version, compare_version):
-
-    # vendor_version = abstract_version(vendor_version)
-    compare_version = abstract_version(compare_version)
-
-    vendor_version_list = vendor_version.split('.')
-    compare_version_list = compare_version.split('.')
-
-    is_smaller_vendor = False
-    smallest_range = len(vendor_version_list) if len(compare_version_list) > len(vendor_version_list) else len(compare_version_list)
-
-    for i in range(smallest_range):
-        if int(vendor_version_list[i]) < int(compare_version_list[i]):
-            is_smaller_vendor = True
-            return is_smaller_vendor
-
-        if int(vendor_version_list[i]) > int(compare_version_list[i]):
-            is_smaller_vendor = False
-            return is_smaller_vendor
-
-    if len(compare_version_list) >= len(vendor_version_list):
-        is_smaller_vendor = True
-
-    return is_smaller_vendor
+from web.index.models import Project, VendorVulns, check_update_or_new_scanresult, get_resultflow_class
 
 
 def get_project_vendor_by_name(vendor_name):
@@ -140,9 +98,9 @@ def get_project_by_version(vendor_name, vendor_version):
     pvs = get_project_vendor_by_name(vendor_name.strip())
 
     for pv in pvs:
-        pv_version = abstract_version(pv.version)
+        # pv_version = abstract_version(pv.version)
 
-        if not is_need_version_check or compare_vendor(pv_version, vendor_version):
+        if not is_need_version_check or compare_vendor(pv.version, vendor_version):
             pid = pv.project_id
             project = Project.objects.filter(id=pid).first()
 
@@ -167,14 +125,14 @@ def check_and_save_result(task_id, language, vendor_name, vendor_version):
     :return:
     """
     vvs = get_vendor_vul_by_name(vendor_name.strip())
-    vendor_version = abstract_version(vendor_version)
+    # vendor_version = abstract_version(vendor_version)
     result_list = []
 
     for vv in vvs:
         if not vendor_version or compare_vendor(vendor_version, vv.vendor_version):
 
             if task_id:
-                check_update_or_new_scanresult(
+                sr = check_update_or_new_scanresult(
                     scan_task_id=task_id,
                     cvi_id=VENDOR_CVIID,
                     language=language,
@@ -184,6 +142,16 @@ def check_and_save_result(task_id, language, vendor_name, vendor_version):
                     is_unconfirm=False,
                     is_active=True
                 )
+                #  save into get_resultflow_class
+                ResultFlow = get_resultflow_class(int(task_id))
+
+                if sr:
+                    node_source = vv.description
+                    rf = ResultFlow(vul_id=sr.id, node_type='sca_scan',
+                                    node_content=vv.title, node_path=vv.reference,
+                                    node_source=node_source, node_lineno=0)
+                    rf.save()
+
             else:
                 result_list.append(vv)
 

docs/changelog.md

@@ -262,4 +262,12 @@
     - 将默认日志重置为Debug模式
     - 新增组件安全扫描功能@LuckyC4t #158 #144
     - 新增search命令用于快速搜索包含某个组件的项目列表，并支持*语法
-    - 优化了底层数据结构
+    - 优化了底层数据结构
+- 2021-08-13
+    - KunLun-M 2.6.1
+    - 更新了Web模式页面以适配组件漏洞扫描结果展示
+    - 修复了数据可能会导致重复入库的bug
+    - 更新了web模式的api数据
+    - 为基础扫描添加去重功能，现在同一文件泄露的同一问题会被去重。
+    - 为JS的语义分析扫描添加硬限制以应对混淆代码。
+

rules/base/CVI_5001.py

@@ -30,7 +30,7 @@ def __init__(self):
 
         # 部分配置
         self.match_mode = "only-regex"
-        self.match = ['((password)\\b[\'"]?\\s*[:=(,]+\\s*[\'"]?(\\w{3,})[\'"]?\\b)']
+        self.match = ['((password)\\b[\'"]?\\s*[:=(,]+\\s*[\'"]?([^\'"\\s]{3,})[\'"]?\\b)']
 
         # for solidity
         self.match_name = None

templates/dashboard/base.html

@@ -167,6 +167,19 @@
             <li id="task_list"><a href="{% url 'dashboard:tasks_list' %}"><i class="fa fa-circle-o"></i>Tasks List</a></li>
           </ul>
         </li>
+        <li class="treeview" id="vendors">
+          <a href="#"><i class="fa ion-ios-analytics-outline"></i> <span>Vendors</span>
+            <span class="pull-right-container">
+                <i class="fa fa-angle-left pull-right"></i>
+              </span>
+          </a>
+          <ul class="treeview-menu">
+            <li id="vendors_list"><a href="{% url 'dashboard:vendors_list' %}"><i class="fa fa-circle-o"></i>Vendors List</a></li>
+          </ul>
+          <ul class="treeview-menu">
+            <li id="vendor_vulns_list"><a href="{% url 'dashboard:vendor_vulns_list' %}"><i class="fa fa-circle-o"></i>Vendors Vuln List</a></li>
+          </ul>
+        </li>
         <li class="treeview" id="rules">
           <a href="#"><i class="fa ion-document-text"></i> <span>Rules</span>
             <span class="pull-right-container">

templates/dashboard/index.html

@@ -1,7 +1,7 @@
 {% extends "dashboard/base.html" %}
 {% block body %}
 <div class="row">
-        <div class="col-lg-3 col-xs-6">
+        <div class="col-lg-2 col-xs-6">
           <!-- small box -->
           <div class="small-box bg-aqua">
             <div class="inner">
@@ -16,7 +16,7 @@ <h3>{% if request.session.tasks_count %}{{ request.session.tasks_count }}{% else
           </div>
         </div>
         <!-- ./col -->
-        <div class="col-lg-3 col-xs-6">
+        <div class="col-lg-2 col-xs-6">
           <!-- small box -->
           <div class="small-box bg-green">
             <div class="inner">
@@ -31,7 +31,7 @@ <h3>{% if  request.session.project_count %}{{  request.session.project_count }}{
           </div>
         </div>
         <!-- ./col -->
-        <div class="col-lg-3 col-xs-6">
+        <div class="col-lg-2 col-xs-6">
           <!-- small box -->
           <div class="small-box bg-red">
             <div class="inner">
@@ -47,7 +47,7 @@ <h3>{% if request.session.vul_count %}{{ request.session.vul_count }}{% else %}0
         </div>
 
 
-        <div class="col-lg-3 col-xs-6">
+        <div class="col-lg-2 col-xs-6">
           <!-- small box -->
           <div class="small-box bg-yellow">
             <div class="inner">
@@ -61,6 +61,35 @@ <h3>{% if request.session.rules_count %}{{ request.session.rules_count }}{% else
             <a href="{% url 'dashboard:rules_list' %}" class="small-box-footer">More info <i class="fa fa-arrow-circle-right"></i></a>
           </div>
         </div>
+        <div class="col-lg-2 col-xs-6">
+          <!-- small box -->
+          <div class="small-box bg-orange">
+            <div class="inner">
+              <h3>{% if request.session.vendor_vuls_count %}{{ request.session.vendor_vuls_count }}{% else %}0{% endif %}</h3>
+
+              <p>Vendor Vuls</p>
+            </div>
+            <div class="icon">
+              <i class="ion ion-eye-disabled"></i>
+            </div>
+            <a href="{% url 'dashboard:vendor_vulns_list' %}" class="small-box-footer">More info <i class="fa fa-arrow-circle-right"></i></a>
+          </div>
+        </div>
+        <div class="col-lg-2 col-xs-6">
+          <!-- small box -->
+          <div class="small-box bg-purple">
+            <div class="inner">
+              <h3>{% if request.session.tampers_count %}{{ request.session.tampers_count }}{% else %}0{% endif %}</h3>
+
+              <p>Tampers</p>
+            </div>
+            <div class="icon">
+              <i class="ion ion-android-desktop"></i>
+            </div>
+            <a href="{% url 'dashboard:tampers_list' %}" class="small-box-footer">More info <i class="fa fa-arrow-circle-right"></i></a>
+          </div>
+        </div>
+
       </div>
       <div class="box box-info">
             <div class="box-header with-border">

templates/dashboard/projects/project_detail.html

@@ -98,8 +98,9 @@ <h3 class="box-title">Results</h3>
                   <th>ID</th>
                   <th>CVI ID</th>
                   <th>Language</th>
-                  <th>VulFile Path</th>
+                  <th>VulFile Path/Title</th>
                   <th>Source</th>
+                  <th>Level</th>
                   <th>Type</th>
                   <th>Is Confirm</th>
                   <th>Operate</th>
@@ -110,7 +111,12 @@ <h3 class="box-title">Results</h3>
                     <td>{{ taskresult.cvi_id }}</td>
                     <td>{{ taskresult.language }}</td>
                     <td>{{ taskresult.vulfile_path }}</td>
+                    {% if taskresult.result_type == 'vendor_source_match' %}
+                        <td><a href="{{ taskresult.source_code }}">{{ taskresult.source_code }}</a></td>
+                    {% else %}
                     <td>{{ taskresult.source_code }}</td>
+                    {% endif %}
+                    <td>{{ taskresult.level }}</td>
                     <td>{{ taskresult.result_type }}</td>
 
                       <td>

templates/dashboard/projects/projects_list.html

@@ -35,7 +35,7 @@ <h3 class="box-title">Projects List</h3>
             <!-- /.box-body -->
 
             <div class="box-footer">
-            <ul class="pagination pagination-sm m-0 float-right">
+                <ul class="pagination pagination-sm m-0 float-right">
                   <li class="page-item"><a class="page-link" href="{% url 'dashboard:projects_list' %}">«</a></li>
                 {% for i in page_range %}
                   <li class="page-item {% if page == i %}active{% endif %}"><a class="page-link" href="{% url 'dashboard:projects_list' %}?p={{ i }}">{{ i }}</a></li>