Python Audit Suite and Queries

python/.data/queries.json

@@ -3,7 +3,9 @@
     "python/CWE-078/CommandInjectionAudit.ql",
     "python/CWE-089/SqlInjectionHeuristic.ql",
     "python/CWE-094/CodeInjectionAudit.ql",
-    "python/CWE-502/UnsafeDeserializationAudit.ql"
+    "python/CWE-502/UnsafeDeserializationAudit.ql",
+    "python/CWE-502/XMLLocalFileAudit.ql",
+    "python/CWE-676/DangerousFunctions.ql"
   ],
   "default": [
     "codeql/python/ql/src/Expressions/UseofInput.ql",
@@ -314,27 +316,19 @@
   ],
   "super-extended": [
     "python/CWE-078/CommandInjectionLocal.ql",
-    "python/CWE-078/CommandInjectionStatic.ql",
-    "python/CWE-089/SqlInjectionHeuristic.ql",
     "python/CWE-089/SqlInjectionLocal.ql",
     "python/CWE-094/CodeInjectionLocal.ql",
-    "python/CWE-094/CodeInjectionStatic.ql",
     "python/CWE-133/format_string.ql",
     "python/CWE-327/WeakHMacAlgorithms.ql",
     "python/CWE-327/WeakHashingAlgorithms.ql",
     "python/CWE-338/WeakPRNG.ql",
     "python/CWE-502/UnsafeDeserializationLocal.ql",
-    "python/CWE-502/UnsafeDeserializationStatic.ql",
-    "python/CWE-502/XMLLocalFileStatic.ql",
     "python/CWE-502/XMLLocalFileTaint.ql",
     "python/CWE-502/XMLLocalStringTaint.ql",
-    "python/CWE-676/DangerousFunctions.ql",
     "python/CWE-778/InsufficientLogging.ql",
     "python/CWE-798/HardcodedFrameworkSecrets.ql",
     "python/CWE-915/MassAssignment.ql",
     "python/CWE-915/MassAssignmentLocal.ql",
-    "python/debugging/PartialPathsFromSink.ql",
-    "python/debugging/PartialPathsFromSource.ql",
     "codeql/python/ql/src/Expressions/UseofInput.ql",
     "codeql/python/ql/src/Security/CVE-2018-1281/BindToAllInterfaces.ql",
     "codeql/python/ql/src/Security/CWE-020/IncompleteHostnameRegExp.ql",
@@ -380,11 +374,24 @@
     "codeql/python/ql/src/Diagnostics/SuccessfullyExtractedFiles.ql",
     "codeql/python/ql/src/Summary/LinesOfCode.ql",
     "codeql/python/ql/src/Summary/LinesOfUserCode.ql",
+    "codeql/python/ql/src/experimental/Classes/NamingConventionsClasses.ql",
+    "codeql/python/ql/src/experimental/Functions/NamingConventionsFunctions.ql",
+    "codeql/python/ql/src/experimental/Security/CWE-022/ZipSlip.ql",
     "codeql/python/ql/src/experimental/Security/CWE-074/TemplateInjection.ql",
+    "codeql/python/ql/src/experimental/Security/CWE-079/ReflectedXSS.ql",
     "codeql/python/ql/src/experimental/Security/CWE-091/Xslt.ql",
+    "codeql/python/ql/src/experimental/Security/CWE-113/HeaderInjection.ql",
+    "codeql/python/ql/src/experimental/Security/CWE-1236/CsvInjection.ql",
     "codeql/python/ql/src/experimental/Security/CWE-287/ImproperLdapAuth.ql",
+    "codeql/python/ql/src/experimental/Security/CWE-327/Azure/UnsafeUsageOfClientSideEncryptionVersion.ql",
+    "codeql/python/ql/src/experimental/Security/CWE-338/InsecureRandomness.ql",
+    "codeql/python/ql/src/experimental/Security/CWE-347/JWTEmptyKeyOrAlgorithm.ql",
+    "codeql/python/ql/src/experimental/Security/CWE-347/JWTMissingSecretOrPublicKeyVerification.ql",
     "codeql/python/ql/src/experimental/Security/CWE-348/ClientSuppliedIpUsedInSecurityCheck.ql",
     "codeql/python/ql/src/experimental/Security/CWE-522/LDAPInsecureAuth.ql",
+    "codeql/python/ql/src/experimental/Security/CWE-611/SimpleXmlRpcServer.ql",
+    "codeql/python/ql/src/experimental/Security/CWE-614/CookieInjection.ql",
+    "codeql/python/ql/src/experimental/Security/CWE-614/InsecureCookie.ql",
     "codeql/python/ql/src/experimental/Security/CWE-943/NoSQLInjection.ql"
   ]
 }

python/CWE-078/CommandInjectionStatic.ql → python/CWE-078/CommandInjectionAudit.ql

@@ -7,13 +7,11 @@
  * @security-severity 2.5
  * @sub-severity low
  * @precision very-low
- * @id py/command-line-injection-static
+ * @id py/audit/command-line-injection
  * @tags security
  *       external/cwe/cwe-078
  *       external/cwe/cwe-088
- *       external/owasp/owasp-a1
- *       experimental
- *       static
+ *       audit
  */
 
 import python

python/CWE-089/SqlInjectionHeuristic.ql

@@ -6,12 +6,11 @@
  * @problem.severity error
  * @security-severity 8.8
  * @precision high
- * @id py/sql-injection
+ * @id py/audit/sql-injection
  * @tags security
  *       external/cwe/cwe-089
- *       external/owasp/owasp-a1
- *       audit
  *       heuristic
+ *       audit
  */
 
 import python

python/CWE-094/CodeInjectionStatic.ql → python/CWE-094/CodeInjectionAudit.ql

@@ -7,14 +7,12 @@
  * @security-severity 2.5
  * @sub-severity low
  * @precision very-low
- * @id py/code-injection-static
+ * @id py/audit/code-injection
  * @tags security
  *       external/cwe/cwe-094
  *       external/cwe/cwe-095
  *       external/cwe/cwe-116
- *       external/owasp/owasp-a1
- *       experimental
- *       static
+ *       audit
  */
 
 import python

python/CWE-502/UnsafeDeserializationStatic.ql → python/CWE-502/UnsafeDeserializationAudit.ql

@@ -6,11 +6,10 @@
  * @security-severity 2.5
  * @sub-severity low
  * @precision very-low
- * @id py/unsafe-deserialization-static
+ * @id py/audit/unsafe-deserialization
  * @tags security
  *       external/cwe/cwe-502
- *       experimental
- *       static
+ *       audit
  */
 
 import python

python/CWE-502/XMLLocalFileStatic.ql → python/CWE-502/XMLLocalFileAudit.ql

@@ -6,12 +6,13 @@
  * @problem.severity error
  * @security-severity 6.0
  * @precision high
- * @id python/xxe-local-file-static
+ * @id python/audit/xxe-local-file
  * @tags security
  *       external/cwe/cwe-611
  *       external/cwe/cwe-776
  *       external/cwe/cwe-827
  *       external/cwe/cwe-502
+ *       audit
  */
 
 private import python

python/CWE-676/DangerousFunctions.ql

@@ -9,6 +9,7 @@
  * @precision low
  * @tags security
  *       external/cwe/cwe-676
+ *       audit
  */
 
 import python

python/README.md

@@ -2,44 +2,42 @@
 
 ## Query Suites
 <!-- AUTOMATION-SUITES -->
-| Name             | Queries Count | Description                                                  | Path                                                                          |
-| :--------------- | :------------ | :----------------------------------------------------------- | :---------------------------------------------------------------------------- |
-| `default`        | 38            | Default Query Suite                                          | `codeql/python/ql/src/codeql-suites/code-scanning`                            |
-| `extended`       | 45            | Security Extended Suite                                      | `codeql/python/ql/src/codeql-suites/security-extended`                        |
-| `quality`        | 167           | Security and Quality Extended Suite                          | `codeql/python/ql/src/codeql-suites/security-and-quality`                     |
-| `local-variants` | 49            | Security Extended with local variants enabled                | `advanced-security/codeql-queries/python/suites/codeql-python-local.qls@main` |
-| `super-extended` | 73            | Security Extended with Experimental and Custom Queries Suite | `advanced-security/codeql-queries/python/suites/codeql-python.qls@main`       |
-| `audit`          | 4             | Security Audit Query Suite                                   | `advanced-security/codeql-queries/python/suites/codeql-python-audit.qls@main` |
+| Name | Queries Count | Description | Path |
+| :--- | :---- | :--- | :--- |
+| `default` | 38 | Default Query Suite | `codeql/python/ql/src/codeql-suites/code-scanning` |
+| `extended` | 45 | Security Extended Suite | `codeql/python/ql/src/codeql-suites/security-extended` |
+| `quality` | 167 | Security and Quality Extended Suite | `codeql/python/ql/src/codeql-suites/security-and-quality` |
+| `local-variants` | 49 | Security Extended with local variants enabled | `advanced-security/codeql-queries/python/suites/codeql-python-local.qls@main` |
+| `super-extended` | 78 | Security Extended with Experimental and Custom Queries Suite | `advanced-security/codeql-queries/python/suites/codeql-python.qls@main` |
+| `audit` | 6 | Security Audit Query Suite | `advanced-security/codeql-queries/python/suites/codeql-python-audit.qls@main` |
 
 
 <!-- AUTOMATION-SUITES -->
 
 ## Queries
 <!-- AUTOMATION-QUERIES -->
-| Name                                                           | Severity        | Path                                            |
-| :------------------------------------------------------------- | :-------------- | :---------------------------------------------- |
-| `Uncontrolled command line`                                    | Critical / 10.0 | `python/CWE-078/CommandInjectionLocal.ql`       |
-| `SQL query built from user-controlled sources`                 | Critical / 10.0 | `python/CWE-089/SqlInjectionLocal.ql`           |
-| `Code injection`                                               | Critical / 10.0 | `python/CWE-094/CodeInjectionLocal.ql`          |
-| `Deserializing untrusted input`                                | High / 8.0      | `python/CWE-502/UnsafeDeserializationLocal.ql`  |
-| `Uncontrolled command line`                                    | Low / 2.5       | `python/CWE-078/CommandInjectionStatic.ql`      |
-| `SQL query built from user-controlled sources`                 | Unknown / 8.8   | `python/CWE-089/SqlInjectionHeuristic.ql`       |
-| `Code injection`                                               | Low / 2.5       | `python/CWE-094/CodeInjectionStatic.ql`         |
-| `Python user-controlled format string`                         | Unknown / 1.0   | `python/CWE-133/format_string.ql`               |
-| `Use of Cryptographically Weak HMAC Algorithm`                 | Medium / 5.0    | `python/CWE-327/WeakHMacAlgorithms.ql`          |
-| `Use of a broken or weak cryptographic algorithm`              | Medium / 5.0    | `python/CWE-327/WeakHashingAlgorithms.ql`       |
-| `Use of Cryptographically Weak Pseudo-Random Number Generator` | Medium / 6.0    | `python/CWE-338/WeakPRNG.ql`                    |
-| `Deserializing untrusted input`                                | Low / 2.5       | `python/CWE-502/UnsafeDeserializationStatic.ql` |
-| `Deserializing XML from local file`                            | Unknown / 6.0   | `python/CWE-502/XMLLocalFileStatic.ql`          |
-| `Deserializing XML from user-controlled filename`              | Unknown / 6.0   | `python/CWE-502/XMLLocalFileTaint.ql`           |
-| `Deserializing XML from user-controlled data`                  | Unknown / 6.0   | `python/CWE-502/XMLLocalStringTaint.ql`         |
-| `Dangerous Functions`                                          | Low / 2.5       | `python/CWE-676/DangerousFunctions.ql`          |
-| `Insufficient Logging`                                         | Low / 1.0       | `python/CWE-778/InsufficientLogging.ql`         |
-| `Hard-coded credentials`                                       | Medium / 5.9    | `python/CWE-798/HardcodedFrameworkSecrets.ql`   |
-| `Mass assignment`                                              | High / 8.0      | `python/CWE-915/MassAssignment.ql`              |
-| `Mass assignment`                                              | High / 2.0      | `python/CWE-915/MassAssignmentLocal.ql`         |
-| `Partial Path Query from Sink`                                 | Low / 1.0       | `python/debugging/PartialPathsFromSink.ql`      |
-| `Partial Path Query from Source`                               | Low / 1.0       | `python/debugging/PartialPathsFromSource.ql`    |
+| Name | Severity | Path |
+| :--- | :------- | :--- |
+| `Uncontrolled command line` | Critical / 10.0 | `python/CWE-078/CommandInjectionLocal.ql` |
+| `SQL query built from user-controlled sources` | Critical / 10.0 | `python/CWE-089/SqlInjectionLocal.ql` |
+| `Code injection` | Critical / 10.0 | `python/CWE-094/CodeInjectionLocal.ql` |
+| `Deserializing untrusted input` | High / 8.0 | `python/CWE-502/UnsafeDeserializationLocal.ql` |
+| `Python user-controlled format string` | Unknown / 1.0 | `python/CWE-133/format_string.ql` |
+| `Use of Cryptographically Weak HMAC Algorithm` | Medium / 5.0 | `python/CWE-327/WeakHMacAlgorithms.ql` |
+| `Use of a broken or weak cryptographic algorithm` | Medium / 5.0 | `python/CWE-327/WeakHashingAlgorithms.ql` |
+| `Use of Cryptographically Weak Pseudo-Random Number Generator` | Medium / 6.0 | `python/CWE-338/WeakPRNG.ql` |
+| `Deserializing XML from user-controlled filename` | Unknown / 6.0 | `python/CWE-502/XMLLocalFileTaint.ql` |
+| `Deserializing XML from user-controlled data` | Unknown / 6.0 | `python/CWE-502/XMLLocalStringTaint.ql` |
+| `Insufficient Logging` | Low / 1.0 | `python/CWE-778/InsufficientLogging.ql` |
+| `Hard-coded credentials` | Medium / 5.9 | `python/CWE-798/HardcodedFrameworkSecrets.ql` |
+| `Mass assignment` | High / 8.0 | `python/CWE-915/MassAssignment.ql` |
+| `Mass assignment` | High / 2.0 | `python/CWE-915/MassAssignmentLocal.ql` |
+| `Uncontrolled command line` | Low / 2.5 | `python/CWE-078/CommandInjectionAudit.ql` |
+| `SQL query built from user-controlled sources` | Unknown / 8.8 | `python/CWE-089/SqlInjectionHeuristic.ql` |
+| `Code injection` | Low / 2.5 | `python/CWE-094/CodeInjectionAudit.ql` |
+| `Deserializing untrusted input` | Low / 2.5 | `python/CWE-502/UnsafeDeserializationAudit.ql` |
+| `Deserializing XML from local file` | Unknown / 6.0 | `python/CWE-502/XMLLocalFileAudit.ql` |
+| `Dangerous Functions` | Low / 2.5 | `python/CWE-676/DangerousFunctions.ql` |
 
 
 <!-- AUTOMATION-QUERIES -->

python/suites/codeql-python-audit.qls

@@ -0,0 +1,15 @@
+# This is the field security specialist audit pack
+
+- description: "Python Audit Pack"
+
+# Field query pack with some audit queries
+- qlpack: github-queries-python
+
+- include:
+    kind:
+    - problem
+    - path-problem
+    - metric
+    - diagnostic
+    tags contain:
+    - audit

python/suites/codeql-python.qls

@@ -1,30 +1,34 @@
-#   See https://help.semmle.com/codeql/codeql-cli/procedures/query-suites.html#filtering-the-queries-in-a-query-suite
-#   for additional ways to exclude queries
+# https://codeql.github.com/docs/codeql-cli/creating-codeql-query-suites/
 
 - description: "GitHub's Field Team Python Extended Suite"
 
 - qlpack: github-queries-python
-- exclude:
-    query path:
-      - /^testing\/.*/
-      - /^debugging\/.*/
-    tags contain:
-      - static
-      - debugging
 
 - import: codeql-suites/python-security-extended.qls
   from: codeql/python-queries
 
+# Remove debugging, and audit queries
+- exclude:
+    tags contain:
+      - debugging
+      - audit
+# Remove local testing folders
+- exclude:
+    query path:
+      - /testing\/.*/
+
+# Include Experimental queries
 - queries: '.'
   from: codeql/python-queries
 - include:
-    id:
-      - py/template-injection
-      - py/xslt-injection
-      - py/ip-address-spoofing
-      - py/xpath-injection
-      - py/nosql-injection
-      # LDAP
-      - py/ldap-injection
-      - py/improper-ldap-auth
-      - py/insecure-ldap-auth
+    query path:
+      - /experimental\/.*/
+- exclude:
+    deprecated: //
+- exclude:
+    query path:
+      - Metrics/Summaries/FrameworkCoverage.ql
+      - /Diagnostics/Internal/.*/
+- exclude:
+    tags contain:
+      - model-generator

tests/python-tests/CWE-078/audit/CommandInjectionAudit.expected

@@ -0,0 +1,3 @@
+| cmdi.py:7:17:7:17 | ControlFlowNode for i | Usage of command line |
+| cmdi.py:9:17:9:17 | ControlFlowNode for i | Usage of command line |
+| cmdi.py:11:17:11:30 | ControlFlowNode for Fstring | Usage of command line |

tests/python-tests/CWE-078/audit/CommandInjectionAudit.qlref

@@ -0,0 +1 @@
+CWE-078/CommandInjectionAudit.ql

tests/python-tests/CWE-078/audit/cmdi.py

@@ -0,0 +1,11 @@
+import os
+import subprocess
+
+i = input("Enter command: ")
+
+# direct input
+subprocess.call(i, shell=True)
+# direct input, no shell
+subprocess.call(i)
+# format string
+subprocess.call(f"bash -c {i}", shell=True)

tests/python-tests/CWE-078/audit/options

@@ -0,0 +1 @@
+semmle-extractor-options: --max-import-depth=0

tests/python-tests/CWE-078/local/options

@@ -0,0 +1 @@
+semmle-extractor-options: --max-import-depth=0

tests/python-tests/CWE-502/audit/UnsafeDeserializationAudit.qlref

@@ -0,0 +1 @@
+CWE-502/UnsafeDeserializationAudit.ql

tests/python-tests/CWE-502/audit/options

@@ -0,0 +1 @@
+semmle-extractor-options: --max-import-depth=0

tests/python-tests/CWE-502/local/options

@@ -0,0 +1 @@
+semmle-extractor-options: --max-import-depth=0

tests/python-tests/CWE-915/local/options

@@ -1 +1 @@
-semmle-extractor-options: --max-import-depth=2
+semmle-extractor-options: --max-import-depth=0