Better logging and documentation support

.github/CODEOWNERS

@@ -0,0 +1,3 @@
+# This project is maintained with love by:
+
+* @geekmasher

.github/workflows/python-build-test.yml

@@ -31,10 +31,13 @@ jobs:
 
     - name: End-2-end Tests
       env:
-        CODEQL_TOKEN: ${{ secrets.CODEQL_TOKEN }}
+        GITHUB_TOKEN: ${{ secrets.CODEQL_TOKEN }}
       run: |
-        ./gh-codeql-summarize --github-token $CODEQL_TOKEN -i ./examples/projects.json -f bundle -o ./examples
-
+        ./gh-codeql-summarize \
+          -i ./examples/projects.json \
+          -f bundle \
+          -o ./examples \
+          --disable-banner
 
     # Pull Request auto-linting
     - name: Lint

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2022 Mathew Payne
+Copyright (c) 2022 GitHub
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

codeqlsummarize/__init__.py

@@ -1,3 +1,7 @@
 import os as _os
 
 __MODULE_PATH__ = _os.path.dirname(_os.path.realpath(__file__))
+
+DOCUMENTATION = {
+    "codeql_setup": "https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/setting-up-code-scanning-for-a-repository"
+}

codeqlsummarize/__main__.py

@@ -7,7 +7,8 @@
 
 sys.path.append(".")
 
-from codeqlsummarize import __MODULE_PATH__
+from codeqlsummarize import __MODULE_PATH__, DOCUMENTATION
+from codeqlsummarize.__version__ import __banner__
 from codeqlsummarize.generator import Generator, QUERIES
 from codeqlsummarize.models import CodeQLDatabase, GitHub
 from codeqlsummarize.exporters import EXPORTERS
@@ -27,9 +28,11 @@
     help="Export format (`json`, `customizations`, `mad`, `bundle`)",
 )
 parser.add_argument("-i", "--input", help="Input / Project File")
-parser.add_argument("-o", "--output", default=os.getcwd(), help="Output directory / file")
-
-parser.add_argument("--disable-cache", action="store_true")
+parser.add_argument(
+    "-o", "--output", default=os.getcwd(), help="Output directory / file"
+)
+parser.add_argument("--disable-banner", action="store_true", help="Disable Banner")
+parser.add_argument("--disable-cache", action="store_true", help="Disable Caching Databases and other files")
 
 parser_codeql = parser.add_argument_group("CodeQL")
 parser_codeql.add_argument("--codeql-base", default="./codeql", help="CodeQL Base Path")
@@ -48,9 +51,9 @@
     "-t", "--github-token", default=os.environ.get("GITHUB_TOKEN")
 )
 
+
 def main(arguments):
-    """ Main workflow
-    """
+    """Main workflow"""
     github = GitHub(token=arguments.github_token)
     languages: list[str] = []
     databases: list[CodeQLDatabase] = []
@@ -59,6 +62,8 @@ def main(arguments):
         level=logging.DEBUG if arguments.debug else logging.INFO,
         format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
     )
+    if not arguments.disable_banner:
+        print(__banner__)
     logger.debug("Debugging is enabled")
 
     if not arguments.format:
@@ -83,8 +88,7 @@ def main(arguments):
             database=arguments.database, project_repo=arguments.project_repo
         )
 
-        if langs:
-            languages.extend(langs)
+        languages.extend(langs)
 
     if arguments.github_repository:
         owner, repo = arguments.github_repository.split("/", 1)
@@ -107,16 +111,19 @@ def main(arguments):
         _, repo = arguments.project_repo.split("/", 1)
 
         for language in languages:
-            logger.info(
-                f"Analysing remote repo: {arguments.project_repo} ({language})"
-            )
+            logger.info(f"Analyzing remote repo: {arguments.project_repo} ({language})")
 
             database = CodeQLDatabase(
                 repo, language=language, repository=arguments.project_repo
             )
 
-            if github.avalible:
-                database.path = database.downloadDatabase(github, temppath)
+            if github.available:
+                try:
+                    database.path = database.downloadDatabase(github, temppath)
+                except Exception as err:
+                    logger.warning(
+                        f"Error encountered while downloading CodeQL Database: {err}"
+                    )
             elif arguments.database:
                 logger.debug("Setting database to arguments.database ")
                 database.path = arguments.database
@@ -142,14 +149,17 @@ def main(arguments):
 
                 db = CodeQLDatabase(name=name, language=lang, repository=repo)
 
-                if github.avalible and db.repository:
+                if github.available and db.repository:
                     logger.info(f"Downloading database for :: {repo}")
 
-                    download_path = db.downloadDatabase(
-                        github, temppath, use_cache=not arguments.disable_cache
-                    )
-
-                    db.path = download_path
+                    try:
+                        db.path = db.downloadDatabase(
+                            github, temppath, use_cache=not arguments.disable_cache
+                        )
+                    except Exception as err:
+                        logger.warning(
+                            f"Error encountered while downloading CodeQL Database: {err}"
+                        )
 
                 if not db.path:
                     logger.warning(f"CodeQL Database path is not set")
@@ -178,7 +188,15 @@ def main(arguments):
         logger.info(f"Database setup complete: {database}")
 
         if not database.exists():
-            raise Exception("CodeQL Database does not exist...")
+            logger.warning(
+                f"Failed to find or download the CodeQL Database for '{database.name}'"
+            )
+            logger.warning(
+                "Please consult the GitHub docs to find out how to build a CodeQL Database"
+            )
+            logger.warning(DOCUMENTATION.get("codeql_setup"))
+            logger.warning("Skipping project until Database is available...")
+            continue
 
         # find codeql
         generator = Generator(database)

codeqlsummarize/__version__.py

@@ -0,0 +1,34 @@
+
+__title__ = "CodeQL Summarize"
+__name__ = "codeqlsummarize"
+__version__ = "0.1.0"
+
+__description__ = "GitHub CodeQL Summaries Toolkit"
+__summary__ = """\
+This is the GitHub CodeQL Summarize project and Actions which allows users to generate Models as Data (MaD) from CodeQL databases.
+"""
+
+__url__ = "https://github.com/advanced-security/gh-codeql-summarize"
+
+__license__ = "MIT License"
+__copyright__ = "Copyright (c) 2022, GitHub"
+
+__author__ = "GitHub Field Team"
+__email__ = ""
+__maintainer__ = "GeekMasher"
+__mEmail__ = ""
+
+__contributors__ = [
+    "GeekMasher",
+    "zbazztian"
+]
+
+__banner__ = f"""\
+ _____           _       _____ _       _____                                      _         
+/  __ \         | |     |  _  | |     /  ___|                                    (_)        
+| /  \/ ___   __| | ___ | | | | |     \ `--. _   _ _ __ ___  _ __ ___   __ _ _ __ _ _______ 
+| |    / _ \ / _` |/ _ \| | | | |      `--. \ | | | '_ ` _ \| '_ ` _ \ / _` | '__| |_  / _ \\
+| \__/\ (_) | (_| |  __/\ \/' / |____ /\__/ / |_| | | | | | | | | | | | (_| | |  | |/ /  __/
+ \____/\___/ \__,_|\___| \_/\_\_____/ \____/ \__,_|_| |_| |_|_| |_| |_|\__,_|_|  |_/___\___|
+                                                        By {__author__} - v{__version__}
+"""

codeqlsummarize/exporters/customizations.py

@@ -116,9 +116,9 @@ def exportCustomizations(
 
 def exportBundle(database: CodeQLDatabase, output: str, github: GitHub, **kargs):
     logger.debug(f"Output directory :: {output}")
-      
+
     owner = github.owner.replace("-", "_")
-    
+
     if not github or not github.owner:
         raise Exception("Failed to export Bundle: No owner / repo name set")
 

codeqlsummarize/exporters/exptjson.py

@@ -9,19 +9,17 @@
 
 
 def exportToJson(database: CodeQLDatabase, output: str, **kargs):
-    """ Export to JSON
-    """
+    """Export to JSON"""
     logger.info("Running export to JSON")
 
     data = {}
     for name, summary in database.summaries.items():
         data[name] = summary.rows
-    
+
     logger.info(f"Saving output to file: {output}")
     with open(output, "w") as handle:
         js.dump(data, handle, indent=2, sort_keys=True)
 
     logger.info("Completed writing to output")
 
     return
-

codeqlsummarize/generator.py

@@ -3,19 +3,15 @@
 
 import json
 import os
-from os.path import (
-  join,
-  exists,
-  realpath
-)
+from os.path import join, exists, realpath
 import shlex
 import tempfile
 import logging
 from typing import *
 from codeqlsummarize.utils import (
-  findCodeQLCli,
-  exec_from_path_env,
-  print_to_stream,
+    findCodeQLCli,
+    exec_from_path_env,
+    print_to_stream,
 )
 from codeqlsummarize import __MODULE_PATH__
 from codeqlsummarize.models import CodeQLDatabase, Summaries
@@ -40,13 +36,10 @@ def __init__(self, database: CodeQLDatabase):
         self.database = database
         self.codeql = findCodeQLCli()
         if not self.codeql:
-            raise Exception('Failed to find CodeQL distribution!')
+            raise Exception("Failed to find CodeQL distribution!")
 
-        self.pack_name = f'codeql/{database.language}-queries'
-        self.codeql(
-            'pack', 'download',
-            self.pack_name
-        )
+        self.pack_name = f"codeql/{database.language}-queries"
+        self.codeql("pack", "download", self.pack_name)
 
     def getModelGeneratorQuery(self, name) -> Optional[str]:
         logger.info(f"Finding query name: {name}")
@@ -63,17 +56,19 @@ def runQuery(self, query: str) -> Summaries:
         logger.info("Running Query :: " + query)
         resultBqrs = join(
             self.database.path,
-            'results',
-            query.replace(':', '/').replace('.ql', '.bqrs')
+            "results",
+            query.replace(":", "/").replace(".ql", ".bqrs"),
         )
 
         output_std = join(Generator.TEMP_PATH, "runquery.txt")
 
         print(f'Running query "{query}"...')
         with open(output_std, "wb") as std:
             self.codeql(
-                "database", "run-queries",
-                "--threads", "0",
+                "database",
+                "run-queries",
+                "--threads",
+                "0",
                 self.database.path,
                 query,
                 outconsumer=print_to_stream(std),
@@ -91,9 +86,12 @@ def readRows(self, bqrsFile):
 
         with open(output_std, "wb") as std:
             self.codeql(
-                "bqrs", "decode",
-                "--format", "json",
-                "--output", generatedJson,
+                "bqrs",
+                "decode",
+                "--format",
+                "json",
+                "--output",
+                generatedJson,
                 bqrsFile,
                 outconsumer=print_to_stream(std),
             )

codeqlsummarize/models.py

@@ -34,9 +34,10 @@ def __post_init__(self):
             logger.debug(f"GitHub Token is set")
 
     @property
-    def avalible(self):
+    def available(self):
         return self.token is not None
 
+
 @dataclass
 class CodeQLDatabase:
     name: str
@@ -134,7 +135,7 @@ def downloadDatabase(
         # SECURITY: Do we trust this DB?
         with zipfile.ZipFile(output_zip) as zf:
             zf.extractall(output_db)
-        
+
         logger.info(f" >>> {output_db}")
         codeql_lang_path = os.path.join(output_db, self.language)
         if os.path.exists(codeql_lang_path):
@@ -144,4 +145,3 @@ def downloadDatabase(
             codeql_dir = os.path.join(output_db, codeql_dir)
             if os.path.isdir(codeql_dir):
                 return codeql_dir
-