v0.1

Author: GeekMasher

codeqldepgraph/__init__.py

@@ -0,0 +1,4 @@
+__name__ = "codeqldepgraph"
+__version__ = "0.1.0"
+
+__url__ = "https://github.com/GeekMasher/codeql-dependency-graph-action"

codeqldepgraph/__main__.py

@@ -1,19 +1,37 @@
+import os
+import json
 import argparse
 
-import os
 from codeqldepgraph.codeql import CodeQL, find_codeql, find_codeql_databases
+from codeqldepgraph.dependencies import parseDependencies, exportDependencies
+from codeqldepgraph.octokit import Octokit
 
 parser = argparse.ArgumentParser(
     description="Generate a dependency graph for a CodeQL database."
 )
 
+parser.add_argument("-sha", default=os.environ.get("GITHUB_SHA"), help="Commit SHA")
+parser.add_argument("-ref", default=os.environ.get("GITHUB_REF"), help="Commit ref")
+
 parser_github = parser.add_argument_group("GitHub")
-parser_github.add_argument("--github-token", help="GitHub API token")
-parser_github.add_argument("--github-repo", help="GitHub repository")
-parser_github.add_argument("--github-instance", help="GitHub instance")
+parser_github.add_argument(
+    "-r",
+    "--github-repo",
+    default=os.environ.get("GITHUB_REPOSITORY"),
+    help="GitHub repository",
+)
+parser_github.add_argument(
+    "--github-instance",
+    default=os.environ.get("GITHUB_SERVER_URL", "https://github.com"),
+    help="GitHub instance",
+)
+parser_github.add_argument(
+    "--github-token", default=os.environ.get("GITHUB_TOKEN"), help="GitHub API token"
+)
 
 parser_codeql = parser.add_argument_group("CodeQL")
 parser_codeql.add_argument("--codeql-path", help="CodeQL executable")
+parser_codeql.add_argument("--codeql-pack", help="CodeQL pack")
 parser_codeql.add_argument(
     "--codeql-database",
     help="CodeQL database",
@@ -23,6 +41,9 @@
 
 if __name__ == "__main__":
     args = parser.parse_args()
+    owner, repo = args.github_repo.split("/", 1)
+    # TODO: support GitHub Enterprise
+    github = Octokit(owner=owner, repo=repo, token=args.github_token)
 
     codeql_path = args.codeql_path or find_codeql()
 
@@ -39,6 +60,25 @@
 
     for database in databases:
         codeql = CodeQL(
-            args.codeql_database,
+            database,
             codeql_path,
         )
+        print(codeql)
+
+        codeql_results = codeql.run("Dependencies.ql")
+        results = parseDependencies(codeql_results)
+
+        print(f"Found {len(results)} dependencies.")
+
+        depgraph = exportDependencies(
+            results,
+            # git sha and ref
+            sha=args.sha,
+            ref=args.ref,
+            # source is the codeql database
+            source=database,
+        )
+
+        github.submitDependencies(depgraph)
+
+    print("Done!")

codeqldepgraph/codeql.py

@@ -1,33 +1,47 @@
 import os
 import glob
-import shutil
+import json
+import logging
 import subprocess
 
+__HERE__ = os.path.dirname(os.path.abspath(__file__))
+__ROOT__ = os.path.abspath(os.path.join(__HERE__, ".."))
+
 CODEQL_LOCATIONS = [
     "codeql",
     # gh cli
     "gh codeql",
+]
+# Actions
+CODEQL_LOCATIONS.extend(glob.glob("/opt/hostedtoolcache/CodeQL/*/x64/codeql/codeql"))
+# VSCode install
+CODEQL_LOCATIONS.extend(glob.glob(
+    "/home/codespace/.vscode-remote/data/User/globalStorage/github.vscode-codeql/*/codeql/codeql"
+)),
+print(CODEQL_LOCATIONS)
+
+CODEQL_DATABASE_LOCATIONS = [
+    # local db
+    ".codeql/db",
     # Actions
-    "/opt/hostedtoolcache/CodeQL/*/x64/codeql/codeql",
-    # VSCode install
-    "/home/codespace/.vscode-remote/data/User/globalStorage/github.vscode-codeql/*/codeql/codeql",
+    "/home/runner/work/_temp/codeql_databases/",
 ]
 
-CODEQL_DATABASE_LOCATIONS = [".codeql/db", "/home/runner/work/_temp/codeql_databases/"]
+CODEQL_TEMP = os.path.join("/tmp", "codeqldepgraph")
+
+logger = logging.getLogger("codeql")
 
 
 def find_codeql() -> str:
     """Find the CodeQL executable"""
     for codeql in CODEQL_LOCATIONS:
-        codeql = glob.glob(codeql)
-        # test if the glob found anything
-        if codeql:
-            # test command works
-            try:
-                subprocess.run([codeql[0], "--version"], stdout=subprocess.PIPE)
-                return codeql[0]
-            except Exception as err:
-                pass
+        try:
+            with open(os.devnull) as null:
+                subprocess.run([codeql, "--version"], stdout=null, stderr=null)
+            return codeql
+        except Exception as err:
+            pass
+        print(f" >> {codeql}")
 
     raise Exception("Could not find CodeQL executable")
 
@@ -43,35 +57,94 @@ def find_codeql_databases() -> list:
 
 
 class CodeQL:
-    def __init__(self, database: str, language: str, codeql_path: str = None):
+    def __init__(self, database: str, codeql_path: str = None):
         self.database = database
-        self.language = language
 
         self.codeql_path = codeql_path or find_codeql()
-        self.databases = []
+        self.language = self.find_language()
+
+        self.pack_name = f"codeql-depgraph-{self.language}"
+
+        if not os.path.exists(CODEQL_TEMP):
+            os.makedirs(CODEQL_TEMP)
 
     def find_language(self) -> str:
         """Find the language of the CodeQL database"""
-        # find db folder
-        db = glob.glob(os.path.join(self.database), "db-*")
+        db = glob.glob(os.path.join(self.database, "db-*"))
+
         if db:
-            return db[0].split("-")[1]
+            return db[0].split("-")[-1]
 
         raise Exception("Could not find CodeQL database language")
 
-    def run_query(self, query):
+    def run(self, query):
         """Run a CodeQL query"""
-        return subprocess.run(
-            [
-                self.codeql_path,
-                "query",
-                "run",
-                query,
-                "--database",
-                self.database,
-                "--language",
-                self.language,
-            ],
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
+        local_query = os.path.join(__ROOT__, "ql", self.language, query)
+        if os.path.exists(local_query):
+            full_query = local_query
+        elif os.path.exists(query):
+            full_query = query
+        else:
+            full_query = f"{self.pack_name}:{query}"
+
+        resultBqrs = os.path.join(
+            self.database,
+            "results",
+            self.pack_name,
+            query.replace(":", "/").replace(".ql", ".bqrs"),
         )
+
+        cmd = [
+            self.codeql_path,
+            "database",
+            "run-queries",
+            # use all the threads on system
+            "--threads",
+            "0",
+            self.database,
+            full_query,
+        ]
+        logger.debug(f"Running: {' '.join(cmd)}")
+
+        output_std = os.path.join(CODEQL_TEMP, "runquery.txt")
+        with open(output_std, "wb") as std:
+            subprocess.run(cmd, stdout=std, stderr=std)
+
+        return self.readRows(resultBqrs)
+
+    def readRows(self, bqrsFile: str) -> list:
+        generatedJson = os.path.join(CODEQL_TEMP, "out.json")
+        output_std = os.path.join(CODEQL_TEMP, "rows.txt")
+
+        with open(output_std, "wb") as std:
+            subprocess.run(
+                [
+                    self.codeql_path,
+                    "bqrs",
+                    "decode",
+                    "--format",
+                    "json",
+                    "--output",
+                    generatedJson,
+                    bqrsFile,
+                ],
+                stdout=std,
+                stderr=std,
+            )
+
+        with open(generatedJson) as f:
+            results = json.load(f)
+
+        try:
+            results["#select"]["tuples"]
+        except KeyError:
+            raise Exception("Unexpected JSON output - no tuples found")
+
+        rows = []
+        for tup in results["#select"]["tuples"]:
+            rows.extend(tup)
+
+        return rows
+
+    def __str__(self) -> str:
+        return f"CodeQL(language='{self.language}', path='{self.database}')"

codeqldepgraph/dependencies.py

@@ -0,0 +1,86 @@
+from typing import *
+import os
+from datetime import datetime
+
+from codeqldepgraph import __name__, __version__, __url__
+
+
+class Dependency:
+    def __init__(self, **kwargs):
+        self.namespace = kwargs.get("namespace")
+        self.name = kwargs.get("name")
+        self.version = kwargs.get("version")
+        self.manager = kwargs.get("manager")
+        self.path = kwargs.get("path")
+
+    @staticmethod
+    def parse(data: str) -> "Dependency":
+        filepath, name, version = data.split("<|>")
+
+        dep = Dependency(name=name, version=version, path=filepath).parseJava()
+        return dep
+
+    def parseJava(self):
+        if self.path and self.path.endswith(".jar"):
+            # We assume that we are using Maven
+            self.manager = "maven"
+
+            pathcomps = self.path.split(os.path.sep)
+            jar = pathcomps[-1].replace(".jar", "")
+            # split on last dash
+            self.name, self.version = jar.rsplit("-", 1)
+
+            next = False
+            for part in reversed(pathcomps):
+                if next:
+                    self.namespace = part
+                    break
+                elif part == self.name:
+                    next = True
+        return self
+
+    def getName(self):
+        if self.manager == "maven":
+            return f"{self.namespace}.{self.name}"
+        return self.name
+
+    def getPurl(self):
+        return f"pkg:{self.manager}/{self.namespace}/{self.name}@{self.version}"
+
+    def __str__(self) -> str:
+        """Return a string representation of the dependency"""
+        return self.getPurl()
+
+
+def parseDependencies(data: str) -> list[Dependency]:
+    results = []
+
+    for line in data:
+        results.append(Dependency.parse(line))
+    return results
+
+
+def exportDependencies(dependencies: list[Dependency], **kwargs):
+    resolved = {}
+    for dep in dependencies:
+        name = dep.getName()
+        purl = dep.getPurl()
+        resolved[name] = {"package_url": purl}
+    data = {
+        "version": 0,
+        "sha": kwargs.get("sha"),
+        "ref": kwargs.get("ref"),
+        "job": {"correlator": __name__, "id": __name__},
+        "detector": {"name": __name__, "version": __version__, "url": __url__},
+        "scanned": datetime.now().isoformat(),
+        "manifests": {
+            __name__: {
+                "name": __name__,
+                "file": {
+                    "source_location": "codeql",
+                },
+                "resolved": resolved,
+            }
+        },
+    }
+    return data

codeqldepgraph/octokit.py

@@ -0,0 +1,27 @@
+from typing import *
+
+import requests
+
+
+class Octokit:
+    def __init__(self, owner: str, repo: str, token: str, url="https://api.github.com"):
+        self.owner = owner
+        self.repo = repo
+        self.token = token
+        self.url = url
+
+    def submitDependencies(self, dependencies: dict):
+        """Submit dependencies to GitHub"""
+        url = f"{self.url}/repos/{self.owner}/{self.repo}/dependency-graph/snapshots"
+        resp = requests.post(
+            url,
+            json=dependencies,
+            headers={
+                "Accept": "application/vnd.github+json",
+                "Authorization": f"token {self.token}",
+            },
+        )
+        if resp.status_code != 201:
+            raise Exception(
+                f"Failed to submit dependencies: {resp.status_code} {resp.text}"
+            )