Merge pull request #208 from LoRexxar/develop

KunLun-M 2.6.4.1
LoRexxar · Dec 30, 2021 · 06a68cf · 06a68cf
2 parents d740ff6 + 3cdde8e
commit 06a68cf
Show file tree

Hide file tree

Showing 12 changed files with 249 additions and 75 deletions.
diff --git a/core/__init__.py b/core/__init__.py
@@ -32,7 +32,7 @@
 
 from .__version__ import __title__, __introduction__, __url__, __version__
 from .__version__ import __author__, __author_email__, __license__
-from .__version__ import __copyright__, __epilog__, __scan_epilog__
+from .__version__ import __copyright__, __epilog__, __scan_epilog__, __database_epilog__
 
 from core.rule import RuleCheck, TamperCheck
 from core.console import KunlunInterpreter
@@ -65,7 +65,7 @@ def main():
         parser_group_init.add_argument('migrationname', default='migrationname',  nargs='?', help='Check migration name')
 
         # load config into database
-        parser_group_core = subparsers.add_parser('config', help='config for rule&tamper', description=__introduction__.format(detail='config for rule&tamper'), formatter_class=argparse.RawDescriptionHelpFormatter, usage=argparse.SUPPRESS, add_help=True)
+        parser_group_core = subparsers.add_parser('config', help='config for rule&tamper', description=__introduction__.format(detail='config for rule&tamper'), epilog=__database_epilog__, formatter_class=argparse.RawDescriptionHelpFormatter, usage=argparse.SUPPRESS, add_help=True)
         parser_group_core.add_argument('load', choices=['load', 'recover', 'loadtamper', 'retamper'], default=False, help='operate for rule&tamper')
 
         parser_group_scan = subparsers.add_parser('scan', help='scan target path', description=__introduction__.format(detail='scan target path'), epilog=__scan_epilog__, formatter_class=argparse.RawDescriptionHelpFormatter, add_help=True)

diff --git a/core/__version__.py b/core/__version__.py
@@ -7,7 +7,7 @@
 __issue_page__ = 'https://github.com/LoRexxar/Kunlun-M/issues/new'
 __python_version__ = sys.version.split()[0]
 __platform__ = platform.platform()
-__version__ = '2.6.4'
+__version__ = '2.6.4.1'
 __author__ = 'LoRexxar'
 __author_email__ = 'LoRexxar@gmail.com'
 __license__ = 'MIT License'
@@ -45,4 +45,8 @@
   python {m} scan -t {td} --lan php -b vendor --debug
   python {m} scan -t {td} --lan php -tp roundcube -d -uc
 
-""".format(m='kunlun.py', td='tests/vulnerabilities')
+""".format(m='kunlun.py', td='tests/vulnerabilities')
+__database_epilog__ = """Usage:
+  python {m} init initialize
+  python {m} init checksql index 0009_projectvendors_source
+""".format(m='kunlun.py')
diff --git a/core/cli.py b/core/cli.py
@@ -143,6 +143,9 @@ def display_result(scan_id, is_ask=False):
                 logger.info("[Chain] {}, {}, {}:{}".format(rf.node_type, rf.node_content, rf.node_path, rf.node_lineno))
 
                 try:
+                    if author == 'SCA':
+                        continue
+
                     if not show_context(rf.node_path, rf.node_lineno):
                         logger_console.info(rf.node_source)
                 except:

diff --git a/core/vendors.py b/core/vendors.py
@@ -193,6 +193,9 @@ def __init__(self, task_id, project_id, target, files):
         self.ext_list = []
         self.exist_file_list = []
 
+        # java temp vendor list
+        self.java_temp_vendor_list = {}
+
         for lan in VENDOR_FILE_DICT:
             self.vendor_file_list.extend(VENDOR_FILE_DICT[lan])
 
@@ -279,15 +282,17 @@ def check_vendor(self):
                         vendor_version = vendor[-1].strip()
                         if len(vendor) < 2:
                             vendor_version = None
+                        ext = "php"
 
                         update_and_new_project_vendor(self.project_id, name=vendor_name, version=vendor_version,
-                                                      language=language, ext=savefilepath)
+                                                      language=language, source=savefilepath, ext=ext)
 
                         get_and_save_vendor_vuls(self.task_id, vendor_name, vendor_version, language)
 
                 elif filename == 'composer.json':
                     vendors = json.loads(filecontent)
                     vendors_list = []
+                    ext = "php"
 
                     if not len(vendors):
                         continue
@@ -300,7 +305,7 @@ def check_vendor(self):
                         vendor_version = vendors_list[vendor].strip()
 
                         update_and_new_project_vendor(self.project_id, name=vendor_name, version=vendor_version,
-                                                      language=language, ext=savefilepath)
+                                                      language=language, source=savefilepath, ext=ext)
 
                         get_and_save_vendor_vuls(self.task_id, vendor_name, vendor_version, language)
 
@@ -327,15 +332,17 @@ def check_vendor(self):
 
                             vendor_name = vendor[0].strip()
                             vendor_version = vendor[-1].strip()
+                            ext = go_version
 
                             update_and_new_project_vendor(self.project_id, name=vendor_name, version=vendor_version,
-                                                          language=language, ext=savefilepath)
+                                                          language=language, source=savefilepath, ext=ext)
 
                             get_and_save_vendor_vuls(self.task_id, vendor_name, vendor_version, language)
 
                 elif filename == 'pom.xml':
                     reg = r'xmlns="([\w\.\\/:]+)"'
                     pom_ns = None
+                    ext = ""
 
                     if re.search(reg, filecontent, re.I):
 
@@ -345,21 +352,47 @@ def check_vendor(self):
                         for match in matchs:
                             pom_ns = match.group(1)
 
+                    tree = self.parse_xml(filepath)
+                    root = tree.getroot()
+
+                    # 匹配default
+                    if pom_ns:
+                        default_xpath_reg = ".//{%s}parent" % pom_ns
+                    else:
+                        default_xpath_reg = ".//parent"
+
+                    parents = root.findall(default_xpath_reg)
+                    default_version = "lastest"
+                    for parent in parents:
+                        default_version = parent.getchildren()[2].text
+
+                    # 匹配通用配置
+                    if pom_ns:
+                        java_base_xpath_reg = ".//{%s}properties" % pom_ns
+                    else:
+                        java_base_xpath_reg = ".//properties"
+
+                    base_tags = root.findall(java_base_xpath_reg)
+
+                    if base_tags:
+                        btags = base_tags[0].getchildren()
+                        for btag in btags:
+                            self.java_temp_vendor_list[btag.tag.replace("{%s}" % pom_ns, "")] = btag.text
+
+                    # 匹配dependency
                     if pom_ns:
                         xpath_reg = ".//{%s}dependency" % pom_ns
                     else:
                         xpath_reg = ".//dependency"
 
-                    tree = self.parse_xml(filepath)
-                    root = tree.getroot()
                     childs = root.findall(xpath_reg)
                     for child in childs:
                         group_id = child.getchildren()[0].text
                         artifact_id = child.getchildren()[1].text
-                        if len(child.getchildren()) > 2:
+                        if len(child.getchildren()) > 2 and "version" in child.getchildren()[2].tag:
                             version = child.getchildren()[2].text
                         else:
-                            version = 'latest'
+                            version = default_version
 
                         var_reg = "\${([\w\.\_-]+)}"
                         if re.search(var_reg, version, re.I):
@@ -368,6 +401,16 @@ def check_vendor(self):
 
                             for match in matchs:
                                 varname = match.group(1)
+
+                                # 处理内置变量
+                                if varname == "project.version":
+                                    version = default_version
+                                    continue
+
+                                if varname in self.java_temp_vendor_list:
+                                    version = self.java_temp_vendor_list[varname]
+                                    continue
+
                                 if pom_ns:
                                     var_xpath_reg = ".//{%s}%s" % (pom_ns, varname)
                                 else:
@@ -377,13 +420,20 @@ def check_vendor(self):
 
                                 for child in varchilds:
                                     version = child.text
+                                    ext = varname
+
+                                # 如果没有匹配到，那么需要去数据库查询
+                                if not varchilds:
+                                    pv = ProjectVendors.objects.filter(project_id=self.project_id, ext=varname).first()
+                                    if pv:
+                                        version = pv.version
 
                         vendor_name = "{}:{}".format(group_id, artifact_id)
                         vendor_version = version
-                        ext = "maven"
+                        # ext = "maven"
 
                         update_and_new_project_vendor(self.project_id, name=vendor_name, version=vendor_version,
-                                                      language=language, ext=savefilepath)
+                                                      language=language, source=savefilepath, ext=ext)
 
                         get_and_save_vendor_vuls(self.task_id, vendor_name, vendor_version, language, ext)
 
@@ -417,7 +467,7 @@ def check_vendor(self):
 
                             if vendor_name and vendor_version:
                                 update_and_new_project_vendor(self.project_id, name=vendor_name, version=vendor_version,
-                                                              language=language, ext=savefilepath)
+                                                              language=language, source=savefilepath, ext=ext)
 
                                 get_and_save_vendor_vuls(self.task_id, vendor_name, vendor_version, language, ext)
                             continue

diff --git a/docs/changelog.md b/docs/changelog.md
@@ -287,4 +287,11 @@
     - 修复了墨非api的部分使用问题
     - 删除了tasklog中无意义的数据显示，优化使用体验
     - 在组件数据中加入数据来源路径便于检查
-    - 修复了部分bug#197 #199 #200
+    - 修复了部分bug#197 #199 #200
+- 2021-12-30
+    - KunLun-M 2.6.4.1
+    - 修复了一部分组件数据的解析bug，maven的数据更精确了
+    - 为组件数据添加了source字段，标准了组件的来源位置
+    - 更新了相应的前端显示
+    - 为项目页面做了数据优化，现在不那么烧资源了，并添加了项目搜索功能
+
diff --git a/templates/dashboard/projects/project_detail.html b/templates/dashboard/projects/project_detail.html
@@ -163,13 +163,15 @@ <h3 class="box-title">dependencies Vendors</h3>
               <th>Version</th>
               <th>Language</th>
               <th>Remark</th>
+              <th>Ext</th>
             </tr>
             {% for project_vendor in project_vendors %}
             <tr>
                 <td>{{ project_vendor.id }}</td>
                 <td>{{ project_vendor.name }}</td>
                 <td>{{ project_vendor.version }}</td>
                 <td>{{ project_vendor.language }}</td>
+                <td>{{ project_vendor.source }}</td>
                 <td>{{ project_vendor.ext }}</td>
 
               </tr>

diff --git a/templates/dashboard/projects/projects_list.html b/templates/dashboard/projects/projects_list.html
@@ -3,50 +3,85 @@
 
 {% block body %}
 <div class="row">
-        <div class="col-xs-12">
-          <div class="box">
-            <div class="box-header">
-              <h3 class="box-title">Projects List</h3>
-            </div>
-            <!-- /.box-header -->
-            <div class="box-body table-responsive no-padding">
-              <table class="table table-hover">
-                <tbody><tr>
-                  <th>ID</th>
-                  <th>Origin</th>
-                  <th>Last Scan Time</th>
-                  <th>Task Count</th>
-                  <th>Vul Count</th>
-                  <th>Vendor Count</th>
-                </tr>
-                {% for project in projects %}
-                <tr>
-                    <td><a href="{% url 'dashboard:project_detail' project.id %}">{{ project.id }} {{ project.project_name }}</a></td>
-                      <td>{{ project.project_origin }}</td>
-                      <td>{{ project.last_scan_time }}</td>
-                      <td>{{ project.tasks_count }}</td>
-                      <td>{{ project.results_count }}</td>
-                      <td>{{ project.vendors_count }}</td>
+<div class="col-xs-12">
 
-                  </tr>
-                {% endfor %}
-              </tbody></table>
-            </div>
-            <!-- /.box-body -->
+<form id="fileform" role="form" method="get" enctype="multipart/form-data" action="{% url 'dashboard:projects_list' %}">
 
-            <div class="box-footer">
-                <ul class="pagination pagination-sm m-0 float-right">
-                  <li class="page-item"><a class="page-link" href="{% url 'dashboard:projects_list' %}">«</a></li>
-                {% for i in page_range %}
-                  <li class="page-item {% if page == i %}active{% endif %}"><a class="page-link" href="{% url 'dashboard:projects_list' %}?p={{ i }}">{{ i }}</a></li>
-                {% endfor %}
-                  <li class="page-item"><a class="page-link" href="{% url 'dashboard:projects_list' %}?p={{ max_page }}">»</a></li>
-                </ul>
-            </div>
-          </div>
-          <!-- /.box -->
-        </div>
+
+<div class="box box-primary">
+    <div class="box-header with-border">
+  <h3 class="box-title">Search Project</h3>
+</div>
+<div class="box-body">
+  <!-- Date -->
+  <!-- Date and time range -->
+  <div class="form-group">
+    <label>Project Name:</label>
+
+    <div class="input-group">
+      <div class="input-group-addon">
+        <i class="fa fa-laptop "></i>
       </div>
+      <input type="text" class="form-control pull-right" name="project_name" value="{{ search_project_name }}" >
+    </div>
+              ps: use * to indicate wildcard, just like *test*.
+    <!-- /.input group -->
+  </div>
+  </div>
+  <!-- /.box -->
+    <div class="box-footer">
+        <div class="btn-group">
+          <button id="submit" type="submit" class="btn btn-primary">Search</button>
+        </div>
+  </div>
+</div>
+</form>
+</div>
+
+<div class="col-xs-12">
+  <div class="box">
+    <div class="box-header">
+      <h3 class="box-title">Projects List</h3>
+    </div>
+    <!-- /.box-header -->
+    <div class="box-body table-responsive no-padding">
+      <table class="table table-hover">
+        <tbody><tr>
+          <th>ID</th>
+          <th>Origin</th>
+          <th>Last Scan Time</th>
+          <th>Task Count</th>
+          <th>Vul Count</th>
+          <th>Vendor Count</th>
+        </tr>
+        {% for project in projects %}
+        <tr>
+            <td><a href="{% url 'dashboard:project_detail' project.id %}">{{ project.id }} {{ project.project_name }}</a></td>
+              <td>{{ project.project_origin }}</td>
+              <td>{{ project.last_scan_time }}</td>
+              <td>{{ project.tasks_count }}</td>
+              <td>{{ project.results_count }}</td>
+              <td>{{ project.vendors_count }}</td>
+
+          </tr>
+        {% endfor %}
+      </tbody></table>
+    </div>
+    <!-- /.box-body -->
+
+    <div class="box-footer">
+        <ul class="pagination pagination-sm m-0 float-right">
+          <li class="page-item"><a class="page-link" href="{% url 'dashboard:projects_list' %}">«</a></li>
+        {% for i in page_range %}
+          <li class="page-item {% if page == i %}active{% endif %}"><a class="page-link" href="{% url 'dashboard:projects_list' %}?p={{ i }}">{{ i }}</a></li>
+        {% endfor %}
+          <li class="page-item"><a class="page-link" href="{% url 'dashboard:projects_list' %}?p={{ max_page }}">»</a></li>
+        </ul>
+    </div>
+  </div>
+  <!-- /.box -->
+</div>
+</div>
 {% endblock %}
 
 {% block script %}

diff --git a/templates/dashboard/vendors/vendor_detail.html b/templates/dashboard/vendors/vendor_detail.html
@@ -18,6 +18,7 @@ <h3 class="box-title">Search Result for {{ vendor_name }}</h3>
                   <th>Name</th>
                   <th>Version</th>
                   <th>Language</th>
+                  <th>Source</th>
                   <th>Ext</th>
                 </tr>
                 {% for vendor in vendors %}
@@ -28,6 +29,7 @@ <h3 class="box-title">Search Result for {{ vendor_name }}</h3>
                     <td><a href="{% url 'dashboard:vendor_details'%}?vendorname={{ vendor.name }}">{{ vendor.name }}</a></td>
                       <td>{{ vendor.version }}</td>
                       <td>{{ vendor.language }}</td>
+                      <td>{{ vendor.source }}</td>
                       <td>{{ vendor.ext }}</td>
                   </tr>
                 {% endfor %}

diff --git a/templates/dashboard/vendors/vendors_list.html b/templates/dashboard/vendors/vendors_list.html
@@ -52,6 +52,7 @@ <h3 class="box-title">Vendors List</h3>
                   <th>Name</th>
                   <th>Version</th>
                   <th>Language</th>
+                  <th>Source</th>
                   <th>Ext</th>
                 </tr>
                 {% for vendor in vendors %}
@@ -62,6 +63,7 @@ <h3 class="box-title">Vendors List</h3>
                     <td><a href="{% url 'dashboard:vendor_details'%}?vendorname={{ vendor.name }}">{{ vendor.name }}</a></td>
                       <td>{{ vendor.version }}</td>
                       <td>{{ vendor.language }}</td>
+                      <td>{{ vendor.source }}</td>
                       <td>{{ vendor.ext }}</td>
                   </tr>
                 {% endfor %}