From b719350921e258178c774966f8092dbba074117b Mon Sep 17 00:00:00 2001 From: Mathew Payne <2772944+GeekMasher@users.noreply.github.com> Date: Wed, 3 Aug 2022 12:32:37 +0100 Subject: [PATCH 1/4] Update customizations.py --- codeqlsummarize/exporters/customizations.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/codeqlsummarize/exporters/customizations.py b/codeqlsummarize/exporters/customizations.py index 661f816..303ec56 100644 --- a/codeqlsummarize/exporters/customizations.py +++ b/codeqlsummarize/exporters/customizations.py @@ -105,7 +105,8 @@ def exportCustomizations( """ CODEQL_CUSTOMIZATIONS_QLL = """\ -// This file is Automatically Generated +// This file is Automatically Generated based on the files in-side this relative +// directory. This makes it easier to automate this process. import {language} module {owner} {{ @@ -159,8 +160,10 @@ def exportBundle(database: CodeQLDatabase, output: str, github: GitHub, **kargs) for custom in os.listdir(sub): if custom == "Customizations.qll": continue + + custom = custom.replace(".qll", "") - impt = f" private import {owner}.{database.language}.{name}\n" + impt = f" private import {owner}.{database.language}.{custom}\n" customizations_data += impt with open(customizations_path, "w") as handle: From 0451788cb9b5a14e73facd579ab0c871aa047fdf Mon Sep 17 00:00:00 2001 From: GeekMasher Date: Fri, 5 Aug 2022 07:46:30 +0100 Subject: [PATCH 2/4] Finish the Customizations work --- codeqlsummarize/exporters/customizations.py | 63 ++++++++++++--------- codeqlsummarize/models.py | 2 +- 2 files changed, 37 insertions(+), 28 deletions(-) diff --git a/codeqlsummarize/exporters/customizations.py b/codeqlsummarize/exporters/customizations.py index 303ec56..b8a1c19 100644 --- a/codeqlsummarize/exporters/customizations.py +++ b/codeqlsummarize/exporters/customizations.py @@ -5,6 +5,7 @@ from codeqlsummarize.models import CodeQLDatabase, GitHub from codeqlsummarize.generator import QUERIES +from codeqlsummarize.utils import findCodeQLCli logger = logging.getLogger("codeqlsummarize.exporters") @@ -19,7 +20,7 @@ """ CODEQL_CUSTOMIZATION = """\ -private class {name}{type}Custom extends {models} {{ +private class {name}{type}Custom extends {models}Csv {{ override predicate row(string row) {{ row = [ {rows} @@ -33,7 +34,7 @@ def saveQLL( database: CodeQLDatabase, output_customizations: str, github: GitHub, **kargs ): padding = " " * 6 - owner = github.owner.replace("-", "_") + owner = github.owner.replace("-", "_").lower() models = {} # initially populate data @@ -47,7 +48,7 @@ def saveQLL( if len(summary.rows) == 0: models[sname] = f"// No {sname} found\n" continue - for mad in summary.rows: + for mad in sorted(summary.rows): rows += f'{padding}"{mad}"' if len(summary.rows) > counter: @@ -118,35 +119,37 @@ def exportCustomizations( def exportBundle(database: CodeQLDatabase, output: str, github: GitHub, **kargs): logger.debug(f"Output directory :: {output}") - owner = github.owner.replace("-", "_") + owner = github.owner.replace("-", "_").lower() if not github or not github.owner: raise Exception("Failed to export Bundle: No owner / repo name set") + codeql_pack_path = f"{database.language}-summarize" + codeql_pack_name = f"{owner}/{codeql_pack_path}" + # Create root for language - root = os.path.join(output, database.language, owner) - os.makedirs(root, exist_ok=True) - logger.debug(f"Root for language :: {root}") - - # Create language files - codeql_lang_lock = os.path.join(root, "codeql-pack.lock.yml") - if not os.path.exists(codeql_lang_lock): - logger.debug(f"Creating Language Lock file :: {codeql_lang_lock}") - with open(codeql_lang_lock, "w") as handle: - handle.write(CODEQL_LOCK.format(language=database.language)) - - codeql_lang_pack = os.path.join(root, "qlpack.yml") - if not os.path.exists(codeql_lang_pack): - logger.debug(f"Creating Language Pack file :: {codeql_lang_pack}") - with open(codeql_lang_pack, "w") as handle: - handle.write( - CODEQL_PACK.format( - owner=owner, version="0.1.0", language=database.language - ) - ) + root = os.path.join(output, codeql_pack_path) + + codeql = findCodeQLCli() + + if not os.path.exists(root) and codeql: + # Make dirs or codeql cli??? + codeql("pack", "init", "--version=0.0.1", "--extractor", database.language, codeql_pack_path) + + if not os.path.exists(os.path.join(root, "qlpack.yml")): + raise Exception("Pack wasn't found") + + # Create README + readme = os.path.join(root, "README.md") + if not os.path.exists(readme): + with open(readme, "w") as handle: + handle.write("# CodeQL Summarize Pack\n") + + logger.debug(f"Root Pack Path :: {root}") # Create language subfolder (if needed) - sub = os.path.join(root, owner, database.language) + sub = os.path.join(root, owner, codeql_pack_path.replace("-", "_")) + logger.debug(f"Checking sub pack path exists: {sub}") os.makedirs(sub, exist_ok=True) name = database.display_name(owner=owner) + "Generated" @@ -157,13 +160,19 @@ def exportBundle(database: CodeQLDatabase, output: str, github: GitHub, **kargs) # Dynamically update Customizations.qll customizations_path = os.path.join(sub, "Customizations.qll") customizations_data = "" - for custom in os.listdir(sub): + + codeql_files = os.listdir(sub) + if not codeql_files: + logger.error(f"This is a major issue and please report in the GitHub issues") + raise Exception("Something is really wrong here...") + + for custom in codeql_files: if custom == "Customizations.qll": continue custom = custom.replace(".qll", "") - impt = f" private import {owner}.{database.language}.{custom}\n" + impt = f" private import {owner}.{database.language}_summarize.{custom}\n" customizations_data += impt with open(customizations_path, "w") as handle: diff --git a/codeqlsummarize/models.py b/codeqlsummarize/models.py index de570ea..19d7c4b 100644 --- a/codeqlsummarize/models.py +++ b/codeqlsummarize/models.py @@ -16,7 +16,7 @@ @dataclass class Summaries: - rows: List[str] + rows: List[str] = field(default_factory=list) @dataclass From 22eccf635eb2dfe1410905e978cd0868e1f81fd8 Mon Sep 17 00:00:00 2001 From: GeekMasher Date: Fri, 5 Aug 2022 07:55:19 +0100 Subject: [PATCH 3/4] Fix bug with cmd exec cwd --- codeqlsummarize/exporters/customizations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codeqlsummarize/exporters/customizations.py b/codeqlsummarize/exporters/customizations.py index b8a1c19..108ca7a 100644 --- a/codeqlsummarize/exporters/customizations.py +++ b/codeqlsummarize/exporters/customizations.py @@ -133,8 +133,8 @@ def exportBundle(database: CodeQLDatabase, output: str, github: GitHub, **kargs) codeql = findCodeQLCli() if not os.path.exists(root) and codeql: - # Make dirs or codeql cli??? - codeql("pack", "init", "--version=0.0.1", "--extractor", database.language, codeql_pack_path) + logger.info("Generating CodeQL Summarize Pack") + codeql("pack", "init", "--version=0.0.1", "--extractor", database.language, codeql_pack_path, cwd=output) if not os.path.exists(os.path.join(root, "qlpack.yml")): raise Exception("Pack wasn't found") From 4ee1a70d865c7054817a193802c793accc2ce0a4 Mon Sep 17 00:00:00 2001 From: GeekMasher Date: Fri, 5 Aug 2022 07:55:43 +0100 Subject: [PATCH 4/4] Update examples with new pack structure --- examples/java-summarize/README.md | 1 + .../geekmasherorg/java_summarize/Customizations.qll | 8 ++++++++ .../java_summarize}/EsapiEsapiJavaLegacyGenerated.qll | 6 +++--- examples/java-summarize/qlpack.yml | 5 +++++ .../advanced_security/java/Customizations.qll | 7 ------- examples/java/advanced_security/codeql-pack.lock.yml | 6 ------ examples/java/advanced_security/qlpack.yml | 6 ------ 7 files changed, 17 insertions(+), 22 deletions(-) create mode 100644 examples/java-summarize/README.md create mode 100644 examples/java-summarize/geekmasherorg/java_summarize/Customizations.qll rename examples/{java/advanced_security/advanced_security/java => java-summarize/geekmasherorg/java_summarize}/EsapiEsapiJavaLegacyGenerated.qll (99%) create mode 100644 examples/java-summarize/qlpack.yml delete mode 100644 examples/java/advanced_security/advanced_security/java/Customizations.qll delete mode 100644 examples/java/advanced_security/codeql-pack.lock.yml delete mode 100644 examples/java/advanced_security/qlpack.yml diff --git a/examples/java-summarize/README.md b/examples/java-summarize/README.md new file mode 100644 index 0000000..9b6296a --- /dev/null +++ b/examples/java-summarize/README.md @@ -0,0 +1 @@ +# CodeQL Summarize Pack diff --git a/examples/java-summarize/geekmasherorg/java_summarize/Customizations.qll b/examples/java-summarize/geekmasherorg/java_summarize/Customizations.qll new file mode 100644 index 0000000..b7b7c91 --- /dev/null +++ b/examples/java-summarize/geekmasherorg/java_summarize/Customizations.qll @@ -0,0 +1,8 @@ +// This file is Automatically Generated based on the files in-side this relative +// directory. This makes it easier to automate this process. +import java + +module geekmasherorg { + private import geekmasherorg.java_summarize.EsapiEsapiJavaLegacyGenerated + +} diff --git a/examples/java/advanced_security/advanced_security/java/EsapiEsapiJavaLegacyGenerated.qll b/examples/java-summarize/geekmasherorg/java_summarize/EsapiEsapiJavaLegacyGenerated.qll similarity index 99% rename from examples/java/advanced_security/advanced_security/java/EsapiEsapiJavaLegacyGenerated.qll rename to examples/java-summarize/geekmasherorg/java_summarize/EsapiEsapiJavaLegacyGenerated.qll index ff09238..4fc20ac 100644 --- a/examples/java/advanced_security/advanced_security/java/EsapiEsapiJavaLegacyGenerated.qll +++ b/examples/java-summarize/geekmasherorg/java_summarize/EsapiEsapiJavaLegacyGenerated.qll @@ -1,7 +1,7 @@ import java private import semmle.code.java.dataflow.ExternalFlow -private class EsapiEsapiJavaLegacySinkModelCustom extends SinkModel { +private class EsapiEsapiJavaLegacySinkModelCustom extends SinkModelCsv { override predicate row(string row) { row = [ "org.owasp.esapi.codecs;Base64;true;decodeFileToFile;(String,String);;Argument[1];create-file;generated", @@ -27,7 +27,7 @@ private class EsapiEsapiJavaLegacySinkModelCustom extends SinkModel { } } -private class EsapiEsapiJavaLegacySourceModelCustom extends SourceModel { +private class EsapiEsapiJavaLegacySourceModelCustom extends SourceModelCsv { override predicate row(string row) { row = [ "org.owasp.esapi.filters;SecurityWrapperRequest;true;getParameter;(String,boolean);;ReturnValue;remote;generated", @@ -43,7 +43,7 @@ private class EsapiEsapiJavaLegacySourceModelCustom extends SourceModel { } } -private class EsapiEsapiJavaLegacySummaryModelCustom extends SummaryModel { +private class EsapiEsapiJavaLegacySummaryModelCustom extends SummaryModelCsv { override predicate row(string row) { row = [ "java.util;Properties;true;getProperty;(String);;Argument[-1];ReturnValue;taint;generated", diff --git a/examples/java-summarize/qlpack.yml b/examples/java-summarize/qlpack.yml new file mode 100644 index 0000000..d8477a7 --- /dev/null +++ b/examples/java-summarize/qlpack.yml @@ -0,0 +1,5 @@ +--- +library: false +name: java-summarize +version: 0.0.1 +extractor: java diff --git a/examples/java/advanced_security/advanced_security/java/Customizations.qll b/examples/java/advanced_security/advanced_security/java/Customizations.qll deleted file mode 100644 index 3edc7ab..0000000 --- a/examples/java/advanced_security/advanced_security/java/Customizations.qll +++ /dev/null @@ -1,7 +0,0 @@ -// This file is Automatically Generated -import java - -module advanced_security { - private import advanced_security.java.EsapiEsapiJavaLegacyGenerated - -} diff --git a/examples/java/advanced_security/codeql-pack.lock.yml b/examples/java/advanced_security/codeql-pack.lock.yml deleted file mode 100644 index d40266c..0000000 --- a/examples/java/advanced_security/codeql-pack.lock.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -dependencies: - codeql/java-all: - version: 0.0.12 -compiled: false -lockVersion: 1.0.0 diff --git a/examples/java/advanced_security/qlpack.yml b/examples/java/advanced_security/qlpack.yml deleted file mode 100644 index e43efd7..0000000 --- a/examples/java/advanced_security/qlpack.yml +++ /dev/null @@ -1,6 +0,0 @@ -name: advanced_security/java -version: 0.1.0 -dependencies: - codeql/java-all: "*" -library: true -extractor: java