annotate .qll files in java and shared via script

Author: ginsbach

config/annotate-overlay-local.py

@@ -0,0 +1,175 @@
+# This script is used to annotate .qll files with overlay[local?] annotations.
+# It will walk the directory tree and annotate most .qll files, skipping only
+# some specific cases (e.g., empty files, files that configure dataflow for queries).
+# It will also add overlay[caller] annotations to predicates that are pragma[inline]
+# and either not private or in a hardcoded list of predicates.
+
+# The script takes a list of languages and processes the corresponding directories.
+
+# Usage: python3 annotate-overlay-local.py <language1> <language2> ...
+
+# The script will modify the files in place and print the changes made.
+# The script is designed to be run from the root of the repository.
+
+#!/usr/bin/python3
+import sys
+import os
+from difflib import *
+
+# These are the only two predicates that are pragma[inline], private, and must be
+# overlay[caller] in order to successfully compile our internal java queries.
+hardcoded_overlay_caller_preds = [
+    "fwdFlowInCand", "fwdFlowInCandTypeFlowDisabled"]
+
+
+def filter_out_annotations(filename):
+    '''
+    Read the file and strip all existing overlay[...] annotations from the contents.
+    Return the file modified file content as a list of lines.
+    '''
+    overlays = ["local", "local?", "global", "caller"]
+    annotations = [f"overlay[{t}]" for t in overlays]
+    with open(filename, 'r') as file_in:
+        lines = [l for l in file_in if not l.strip() in annotations]
+    for ann in annotations:
+        if any(line for line in lines if ann in line):
+            raise Exception(f"Failed to filter out {ann} from {filename}.")
+    return lines
+
+
+def insert_toplevel_maybe_local_anntotation(filename, lines):
+    '''
+    Find a suitable place to insert an overlay[local?] annotation at the top of the file.
+    Return a pair: (string describing action taken, modified content as list of lines).
+    '''
+    out_lines = []
+    status = 0
+
+    for line in lines:
+        if status == 0 and line.rstrip().endswith("module;"):
+            out_lines.append("overlay[local?]\n")
+            status = 1
+        out_lines.append(line)
+
+    if status == 1:
+        return (f"Annotating \"{filename}\" via existing file-level module statement", out_lines)
+
+    out_lines = []
+    empty_line_buffer = []
+    status = 0
+    for line in lines:
+        trimmed = line.strip()
+        if not trimmed:
+            empty_line_buffer.append(line)
+            continue
+        if status <= 1 and trimmed.endswith("*/"):
+            status = 2
+        elif status == 0 and trimmed.startswith("/**"):
+            status = 1
+        elif status == 0 and not trimmed.startswith("/*"):
+            out_lines.append("overlay[local?]\n")
+            out_lines.append("module;\n")
+            out_lines.append("\n")
+            status = 3
+        elif status == 2 and (trimmed.startswith("import ") or trimmed.startswith("private import ")):
+            out_lines.append("overlay[local?]\n")
+            out_lines.append("module;\n")
+            status = 3
+        elif status == 2 and (trimmed.startswith("class ") or trimmed.startswith("predicate ")
+                              or trimmed.startswith("module ") or trimmed.startswith("signature ")):
+            out_lines = ["overlay[local?]\n", "module;\n", "\n"] + out_lines
+            status = 3
+        elif status == 2 and trimmed.startswith("/*"):
+            out_lines.append("overlay[local?]\n")
+            out_lines.append("module;\n")
+            status = 3
+        elif status == 2:
+            status = 4
+        if empty_line_buffer:
+            out_lines += empty_line_buffer
+            empty_line_buffer = []
+        out_lines.append(line)
+    if status == 3:
+        out_lines += empty_line_buffer
+
+    if status == 3:
+        return (f"Annotating \"{filename}\" after file-level module qldoc", out_lines)
+
+    raise Exception(f"Failed to annotate \"{filename}\" as overlay[local?].")
+
+
+def insert_overlay_caller_annotations(lines):
+    '''
+    Mark pragma[inline] predicates as overlay[caller] if they are not declared private
+    or if they are private but are in the list of hardcoded_overlay_caller_preds.
+    '''
+    out_lines = []
+    for i, line in enumerate(lines):
+        trimmed = line.strip()
+        if trimmed == "pragma[inline]":
+            if (not "private" in lines[i+1] or
+                    any(pred in lines[i+1] for pred in hardcoded_overlay_caller_preds)):
+                whitespace = line[0: line.find(trimmed)]
+                out_lines.append(f"{whitespace}overlay[caller]\n")
+        out_lines.append(line)
+    return out_lines
+
+
+def annotate_as_appropriate(filename):
+    '''
+    Read file and strip all existing overlay[...] annotations from the contents;
+    then insert new overlay[...] annotations according to heuristics.
+    Return a pair: (string describing action taken, modified content as list of lines).
+    '''
+    lines = filter_out_annotations(filename)
+    lines = insert_overlay_caller_annotations(lines)
+
+    # These simple heuristics filter out those .qll files that we no _not_ want to annotate
+    # as overlay[local?].  It is not clear that these heuristics are exactly what we want,
+    # but they seem to work well enough for now (as determined by speed and accuracy numbers).
+    if (filename.endswith("Test.qll") or
+        ((filename.endswith("Query.qll") or filename.endswith("Config.qll")) and
+         any("implements DataFlow::ConfigSig" in line for line in lines))):
+        return (f"Keeping \"{filename}\" global because it configures dataflow for a query", lines)
+    elif not any(line for line in lines if line.strip()):
+        return (f"Keeping \"{filename}\" global because it is empty", lines)
+
+    return insert_toplevel_maybe_local_anntotation(filename, lines)
+
+
+def process_single_file(filename):
+    '''
+    Process a single file, annotating it as appropriate and writing the changes back to the file.
+    '''
+    annotate_result = annotate_as_appropriate(filename)
+
+    old = [line for line in open(filename)]
+    new = annotate_result[1]
+
+    if old != new:
+        diff = context_diff(old, new, fromfile=filename, tofile=filename)
+        diff = [line for line in diff]
+        if diff:
+            print(annotate_result[0])
+            for line in diff:
+                print(line.rstrip())
+            with open(filename, "w") as out_file:
+                for line in new:
+                    out_file.write(line)
+
+
+dirs = []
+for lang in sys.argv[1:]:
+    if lang in ["cpp", "go", "csharp", "java", "javascript", "python", "ruby", "rust", "swift"]:
+        dirs.append(f"{lang}/ql/lib")
+    else:
+        raise Exception(f"Unknown language \"{lang}\".")
+
+if dirs:
+    dirs.append("shared")
+
+for roots in dirs:
+    for dirpath, dirnames, filenames in os.walk(roots):
+        for filename in filenames:
+            if filename.endswith(".qll") and not dirpath.endswith("tutorial"):
+                process_single_file(os.path.join(dirpath, filename))

java/ql/lib/Customizations.qll

@@ -8,5 +8,7 @@
  * the `RemoteFlowSource` and `AdditionalTaintStep` classes associated with the security queries
  * to model frameworks that are not covered by the standard library.
  */
+overlay[local?]
+module;
 
 import java

java/ql/lib/IDEContextual.qll

@@ -1,6 +1,8 @@
 /**
  * Provides shared predicates related to contextual queries in the code viewer.
  */
+overlay[local?]
+module;
 
 import semmle.files.FileSystem
 private import codeql.util.FileSystem

java/ql/lib/default.qll

@@ -1,3 +1,5 @@
 /** DEPRECATED: use `java.qll` instead. */
+overlay[local?]
+module;
 
 import java

java/ql/lib/definitions.qll

@@ -2,6 +2,8 @@
  * Provides classes and predicates related to jump-to-definition links
  * in the code viewer.
  */
+overlay[local?]
+module;
 
 import java
 import IDEContextual

java/ql/lib/external/ExternalArtifact.qll

@@ -1,3 +1,6 @@
+overlay[local?]
+module;
+
 import java
 
 class ExternalData extends @externalDataElement {

java/ql/lib/java.qll

@@ -1,4 +1,6 @@
 /** Provides all default Java QL imports. */
+overlay[local?]
+module;
 
 import Customizations
 import semmle.code.FileSystem

java/ql/lib/semmle/code/FileSystem.qll

@@ -1,4 +1,6 @@
 /** Provides classes for working with files and folders. */
+overlay[local?]
+module;
 
 import Location
 private import codeql.util.FileSystem

java/ql/lib/semmle/code/Location.qll

@@ -3,6 +3,8 @@
  *
  * Locations represent parts of files and are used to map elements to their source location.
  */
+overlay[local?]
+module;
 
 import FileSystem
 import semmle.code.java.Element

java/ql/lib/semmle/code/SMAP.qll

@@ -1,6 +1,8 @@
 /**
  * Provides classes and predicates for working with SMAP files (see JSR-045).
  */
+overlay[local?]
+module;
 
 import java
 

java/ql/lib/semmle/code/Unit.qll

@@ -1,3 +1,5 @@
 /** Provides the `Unit` class. */
+overlay[local?]
+module;
 
 import codeql.util.Unit

java/ql/lib/semmle/code/configfiles/ConfigFiles.qll

@@ -2,6 +2,8 @@
  * Provides classes and predicates for working with configuration files, such
  * as Java `.properties` or `.ini` files.
  */
+overlay[local?]
+module;
 
 import semmle.code.Location
 

java/ql/lib/semmle/code/java/Annotation.qll

@@ -8,6 +8,8 @@
  * Each annotation type has zero or more annotation elements that contain a
  * name and possibly a value.
  */
+overlay[local?]
+module;
 
 import Element
 import Expr

java/ql/lib/semmle/code/java/Collections.qll

@@ -2,6 +2,8 @@
  * Provides classes and predicates for reasoning about instances of
  * `java.util.Collection` and their methods.
  */
+overlay[local?]
+module;
 
 import java
 

java/ql/lib/semmle/code/java/Compilation.qll

@@ -1,6 +1,8 @@
 /**
  * Provides a class representing individual compiler invocations that occurred during the build.
  */
+overlay[local?]
+module;
 
 import semmle.code.FileSystem
 

java/ql/lib/semmle/code/java/CompilationUnit.qll

@@ -1,6 +1,8 @@
 /**
  * Provides classes and predicates for working with Java compilation units.
  */
+overlay[local?]
+module;
 
 import Element
 import Package

java/ql/lib/semmle/code/java/Completion.qll

@@ -1,6 +1,8 @@
 /**
  * Provides classes and predicates for representing completions.
  */
+overlay[local?]
+module;
 
 /*
  * A completion represents how a statement or expression terminates.

java/ql/lib/semmle/code/java/Concurrency.qll

@@ -1,3 +1,6 @@
+overlay[local?]
+module;
+
 import java
 
 /**

java/ql/lib/semmle/code/java/Constants.qll

@@ -1,6 +1,8 @@
 /**
  * Provdides a module to calculate constant integer and boolean values.
  */
+overlay[local?]
+module;
 
 import java
 

java/ql/lib/semmle/code/java/ControlFlowGraph.qll

@@ -7,6 +7,8 @@
  * statement, an expression, or an exit node for a callable, indicating that
  * execution of the callable terminates.
  */
+overlay[local?]
+module;
 
 /*
  * The implementation is centered around the concept of a _completion_, which

java/ql/lib/semmle/code/java/Conversions.qll

@@ -4,6 +4,8 @@
  *
  * See the Java Language Specification, Section 5, for details.
  */
+overlay[local?]
+module;
 
 import java
 import semmle.code.java.arithmetic.Overflow

java/ql/lib/semmle/code/java/Dependency.qll

@@ -1,6 +1,8 @@
 /**
  * Provides utility predicates for representing dependencies between types.
  */
+overlay[local?]
+module;
 
 import Type
 import Generics

java/ql/lib/semmle/code/java/DependencyCounts.qll

@@ -1,6 +1,8 @@
 /**
  * This library provides utility predicates for representing the number of dependencies between types.
  */
+overlay[local?]
+module;
 
 import Type
 import Generics

java/ql/lib/semmle/code/java/Diagnostics.qll

@@ -1,6 +1,8 @@
 /**
  * Provides classes representing warnings generated during compilation.
  */
+overlay[local?]
+module;
 
 import java
 

java/ql/lib/semmle/code/java/Element.qll

@@ -1,6 +1,8 @@
 /**
  * Provides a class that represents named elements in Java programs.
  */
+overlay[local?]
+module;
 
 import CompilationUnit
 import semmle.code.Location

java/ql/lib/semmle/code/java/Exception.qll

@@ -1,6 +1,8 @@
 /**
  * Provides classes and predicates for working with Java exceptions.
  */
+overlay[local?]
+module;
 
 import Element
 import Type

java/ql/lib/semmle/code/java/Expr.qll

@@ -1,6 +1,8 @@
 /**
  * Provides classes for working with Java expressions.
  */
+overlay[local?]
+module;
 
 import java
 private import semmle.code.java.frameworks.android.Compose