initial code

.gitignore

@@ -0,0 +1 @@
+/logs/*

InsecurityRefactoring/.gitignore

@@ -0,0 +1,2 @@
+target
+output.txt

InsecurityRefactoring/.~lock.results_2020_09_07.csv#

@@ -0,0 +1 @@
+,blubbomat,blubbomat-ThinkPad,08.09.2020 09:56,file:///home/blubbomat/.config/libreoffice/4;

InsecurityRefactoring/analyse_crawler_data.py

@@ -0,0 +1,80 @@
+import os
+
+path = os.path.expanduser("~/crawl_data/2020-09-07")
+import re
+
+S = ';'
+
+tit = "Name" +S+    "LOC"+S+    "Time"+S+       "PIPs" +S+      "Paths" +S+     "Paths vuln" +S+    "Paths timeout" +S+     "XSS" +S+           "SQLi"+S+           "EVAL"  +S+ "SPECIFICS"
+
+reports = []
+
+
+for fil in os.listdir(path):
+    try:
+        types = {'xss':'0', 'eval':'0', 'sqli':'0'}
+        specifics = []
+        sinks = {}
+        if not fil.endswith(".result"):
+           continue 
+
+        filePath = os.path.join(path, fil)
+        f = open(filePath, "r")
+
+        for line in f:
+            line = line.strip()
+            if line.startswith("Vulnerable: "):
+                vulns = re.search('(?<=Vulnerable: )\w+', line).group(0)
+            if line.startswith("FOUND PIPs: "):
+                pips = re.search('(?<=FOUND PIPs: )\w+', line).group(0)
+
+            if line.startswith("Path Amount: "):
+                pathAmount = re.search('(?<=Path Amount: )\w+', line).group(0)
+            if line.startswith("Path Vulns: "):
+                pathVulns = re.search('(?<=Path Vulns: )\w+', line).group(0)
+            if line.startswith("Path Timeouts: "):
+                pathTimeouts = re.search('(?<=Path Timeouts: )\w+', line).group(0)
+
+            if line.startswith("PIP TYPE "):
+                pipType = re.search('(?<=TYPE).+', line).group(0).strip()
+                typ, num = pipType.split(":")
+                types[typ]=num
+
+            if line.startswith("PIP SPECIFIC "):
+                specific = re.search('(?<=SPECIFIC).+', line).group(0).strip()
+                sink, num = specific.rsplit(":", 1)
+                sinks[sink]=num
+
+            if line.startswith("PHP "):
+                loc = re.search('(?<=PHP).+', line).group(0).strip()
+                loc = loc.split(" ")[-1]
+
+            if line.startswith("Scan took:"):
+                scanTime = re.search('(?<=Scan took: ).+', line).group(0)
+
+            if line.startswith("PIP SPECIFIC"):
+                specific = re.search('(?<=PIP SPECIFIC).+', line).group(0)
+                specifics.append(specific)
+
+            if line.startswith("Sink("):
+                name, num = line.rsplit("):", 1)
+                sinks[name] = num
+
+        csv = fil +S+       loc+S+      scanTime+S+     pips +S+        pathAmount +S+  pathVulns +S+       pathTimeouts +S+        types['xss'] +S+    types['sqli']+S+    types['eval'] +S
+
+        for specific in specifics:
+            csv = csv + specific + ","
+        csv = csv +S
+
+        # for sink, num in sinks.items():
+        #     tit = tit +S+ sink
+        #     csv = csv +S+ num
+
+        reports.append(csv)
+    except e:
+        print(e)
+        continue
+
+print(tit)
+for report in reports:
+    print(report)

InsecurityRefactoring/crawl_github.py

@@ -0,0 +1,61 @@
+from github import Github
+from runpath import call_on_path
+import shutil
+import os.path
+import time
+from time import gmtime, strftime
+
+character = "hprose/hprose-php"
+character2 = "phpbb_phpbb"
+
+g = Github("277ffeb5a718ed5aa481cf7bd1a82665d0da4947")
+
+work_dir='/home/blubbomat/pip_scans/'
+repoDir="{}source".format(work_dir)
+
+# repos = g.search_repositories(query='language:php', sort='stars')
+repos = g.search_repositories(query=character)
+
+# Then play with your Github objects:
+i=0
+maxRepos=10000000
+for repo in repos:
+    uniqueName = repo.full_name
+    cloneUrl = "https://github.com/{}.git".format(uniqueName)
+    cloneCmd = "git clone {url} {target}".format(url=cloneUrl, target=repoDir)
+    outputFile = "{}/{}.result".format(work_dir, uniqueName.replace('/', "_"))
+    scan = "java -jar target/InsecurityRefactoring*.jar -o -p {} >> {}".format(repoDir, outputFile)
+    loc = "cloc --quiet {} >> {}".format(repoDir, outputFile)
+
+    if repo.full_name.lower().startswith(character.lower()) or repo.full_name.lower().startswith(character2.lower()):
+        print(repo.full_name + " " + str(repo.stargazers_count))
+    else:
+        print("SKIP " + uniqueName)
+        continue
+
+    if os.path.isfile(outputFile):
+        # Already scanned
+        print("Skipping {}".format(uniqueName))
+        continue
+
+    # Remove dir
+    o, e = call_on_path(work_dir, "rm -rf {}".format(repoDir))
+    # Clone repo
+    o, e = call_on_path(work_dir, cloneCmd)
+    # Count lines of code
+    o, e = call_on_path(work_dir, loc)
+    # Scan
+    startTime = time.time()
+    print("Starting time: {}".format( strftime("%Y-%m-%d %H:%M:%S", gmtime())   ))
+    o, e = call_on_path('.', scan)
+    endTime = time.time()
+
+    elapsedTime = endTime - startTime
+    niceTime = strftime("%H:%M:%S", time.gmtime(elapsedTime))
+    print("It took: {}".format(niceTime))
+    o, e = call_on_path('target', "echo 'Scan took: {}' >> {}".format(niceTime, outputFile))
+
+
+    i=i+1
+    if i >= maxRepos:
+        break

InsecurityRefactoring/nbactions.xml

@@ -0,0 +1,67 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<actions>
+        <action>
+            <actionName>run</actionName>
+            <packagings>
+                <packaging>jar</packaging>
+            </packagings>
+            <goals>
+                <goal>process-classes</goal>
+                <goal>org.codehaus.mojo:exec-maven-plugin:1.5.0:exec</goal>
+            </goals>
+            <properties>
+                <exec.args>-classpath %classpath ntnuhtwg.insecurityrefactoring.Main -p /home/blubbomat/Development/FindPOI/InsecurityRefactoring/src/test/non-packaged-test-files/pips/recursive -o -g</exec.args>
+                <exec.executable>java</exec.executable>
+            </properties>
+        </action>
+        <action>
+            <actionName>debug</actionName>
+            <packagings>
+                <packaging>jar</packaging>
+            </packagings>
+            <goals>
+                <goal>process-classes</goal>
+                <goal>org.codehaus.mojo:exec-maven-plugin:1.5.0:exec</goal>
+            </goals>
+            <properties>
+                <exec.args>-agentlib:jdwp=transport=dt_socket,server=n,address=${jpda.address} -classpath %classpath ntnuhtwg.insecurityrefactoring.Main -p /home/blubbomat/Development/FindPOI/InsecurityRefactoring/src/test/non-packaged-test-files/pips/recursive -o -g</exec.args>
+                <exec.executable>java</exec.executable>
+                <jpda.listen>true</jpda.listen>
+            </properties>
+        </action>
+        <action>
+            <actionName>profile</actionName>
+            <packagings>
+                <packaging>jar</packaging>
+            </packagings>
+            <goals>
+                <goal>process-classes</goal>
+                <goal>org.codehaus.mojo:exec-maven-plugin:1.5.0:exec</goal>
+            </goals>
+            <properties>
+                <exec.args>-classpath %classpath ntnuhtwg.insecurityrefactoring.Main -p /home/blubbomat/Development/FindPOI/InsecurityRefactoring/src/test/non-packaged-test-files/pips/recursive -o -g</exec.args>
+                <exec.executable>java</exec.executable>
+            </properties>
+        </action>
+        <action>
+            <actionName>build</actionName>
+            <packagings>
+                <packaging>*</packaging>
+            </packagings>
+            <goals>
+                <goal>install</goal>
+                <goal>-DskipITs</goal>
+            </goals>
+        </action>
+        <action>
+            <actionName>rebuild</actionName>
+            <packagings>
+                <packaging>*</packaging>
+            </packagings>
+            <goals>
+                <goal>clean</goal>
+                <goal>install</goal>
+                <goal>-DskipITs</goal>
+            </goals>
+        </action>
+    </actions>