Go: overlay workaround for cgo-processed files

Author: nickrolfe

go/extractor/extractor.go

@@ -732,8 +732,13 @@ func (extraction *Extraction) extractFile(ast *ast.File, pkg *packages.Package)
 		return nil
 	}
 	path := normalizedPath(ast, fset)
-	if extraction.OverlayChanges != nil && !extraction.OverlayChanges[path] {
-		// This file did not change since the base was extracted
+	// If we're extracting an overlay, we want to skip extraction of files that haven't changed.
+	// Since some files may be outside the source directory (e.g. files preprocessed by cgo) we
+	// can't easily know if they have changed (or came from source files that changed), so we always
+	// extract a file if it's not in the package directory.
+	if extraction.OverlayChanges != nil &&
+		!extraction.OverlayChanges[path] &&
+		strings.HasPrefix(path+string(filepath.Separator), pkg.Dir) {
 		return nil
 	}
 

go/ql/lib/semmle/go/Overlay.qll

@@ -23,13 +23,30 @@ private predicate discardableLocatable(@file file, @locatable locatable) {
   file = getFile(locatable)
 }
 
+/**
+ * Holds if the given `path` is for a file in the base database whose entities should be discarded.
+ */
+bindingset[path]
+private predicate discardableFile(string path) {
+  isOverlay() and
+  (
+    overlayChangedFiles(path)
+    or
+    // The extractor unconditionally extracts files outside of the source directory (these are
+    // typically cgo-processed source files), so all entities in such files should be discarded.
+    not exists(string srcLoc | sourceLocationPrefix(srcLoc) |
+      path.substring(0, srcLoc.length()) = srcLoc
+    )
+  )
+}
+
 /**
  * Holds if the given `locatable` should be discarded, because it is part of the overlay base and is
  * in a file that was also extracted as part of the overlay database.
  */
 overlay[discard_entity]
 private predicate discardLocatable(@locatable locatable) {
   exists(@file file, string path | files(file, path) |
-    discardableLocatable(file, locatable) and overlayChangedFiles(path)
+    discardableLocatable(file, locatable) and discardableFile(path)
   )
 }