Build tracing

Makefile

@@ -14,11 +14,11 @@ CODEQL_PLATFORM = osx64
 endif
 endif
 
-CODEQL_TOOLS = $(addprefix codeql-tools/,autobuild.cmd autobuild.sh index.cmd index.sh)
+CODEQL_TOOLS = $(addprefix codeql-tools/,autobuild.cmd autobuild.sh index.cmd index.sh linux64 osx64 win64)
 
 EXTRACTOR_PACK_OUT = build/codeql-extractor-go
 
-BINARIES = go-extractor go-tokenizer go-autobuilder go-bootstrap go-gen-dbscheme
+BINARIES = go-extractor go-tokenizer go-autobuilder go-build-runner go-bootstrap go-gen-dbscheme
 
 .PHONY: tools tools-codeql tools-codeql-full clean autoformat \
 	tools-linux64 tools-osx64 tools-win64 check-formatting

change-notes/2020-06-11-build-tracing.md

@@ -0,0 +1,5 @@
+lgtm,codescanning
+* The Go extractor now supports build tracing, allowing users to supply a build command when
+  creating databases with the CodeQL CLI or via configuration. It currently only supports projects
+  that use Go modules. To opt-in, set the environment variable `CODEQL_EXTRACTOR_GO_BUILD_TRACING`
+  to `on`, or supply a build command.

codeql-tools/autobuild.cmd

@@ -4,7 +4,12 @@ SETLOCAL EnableDelayedExpansion
 rem Some legacy environment variables for the autobuilder.
 set LGTM_SRC=%CD%
 
-type NUL && "%CODEQL_EXTRACTOR_GO_ROOT%/tools/%CODEQL_PLATFORM%/go-autobuilder.exe"
+if "%CODEQL_EXTRACTOR_GO_BUILD_TRACING%"=="on" (
+  echo "Tracing enabled"
+  type NUL && "%CODEQL_EXTRACTOR_GO_ROOT%/tools/%CODEQL_PLATFORM%/go-build-runner.exe"
+) else (
+  type NUL && "%CODEQL_EXTRACTOR_GO_ROOT%/tools/%CODEQL_PLATFORM%/go-autobuilder.exe"
+)
 exit /b %ERRORLEVEL%
 
 ENDLOCAL

codeql-tools/autobuild.sh

@@ -11,4 +11,9 @@ fi
 LGTM_SRC="$(pwd)"
 export LGTM_SRC
 
-"$CODEQL_EXTRACTOR_GO_ROOT/tools/$CODEQL_PLATFORM/go-autobuilder"
+if [ "${CODEQL_EXTRACTOR_GO_BUILD_TRACING:-}" == "on" ]; then
+  echo "Tracing enabled"
+  "$CODEQL_EXTRACTOR_GO_ROOT/tools/$CODEQL_PLATFORM/go-build-runner"
+else
+  "$CODEQL_EXTRACTOR_GO_ROOT/tools/$CODEQL_PLATFORM/go-autobuilder"
+fi

codeql-tools/linux64/compiler-tracing.spec

@@ -0,0 +1,7 @@
+**/go-autobuilder:
+  order compiler
+  trace no
+**/go:
+  invoke ${config_dir}/go-extractor
+  prepend --mimic
+  prepend "${compiler}"

codeql-tools/osx64/compiler-tracing.spec

@@ -0,0 +1,22 @@
+**/go-autobuilder:
+  order compiler
+  trace no
+**/go:
+  invoke ${config_dir}/go-extractor
+  prepend --mimic
+  prepend "${compiler}"
+/usr/bin/codesign:
+  replace yes
+  invoke /usr/bin/env
+  prepend /usr/bin/codesign
+  trace no
+/usr/bin/pkill:
+  replace yes
+  invoke /usr/bin/env
+  prepend /usr/bin/pkill
+  trace no
+/usr/bin/pgrep:
+  replace yes
+  invoke /usr/bin/env
+  prepend /usr/bin/pgrep
+  trace no

codeql-tools/win64/compiler-tracing.spec

@@ -0,0 +1,7 @@
+**/go-autobuilder.exe:
+  order compiler
+  trace no
+**/go.exe:
+  invoke ${config_dir}/go-extractor.exe
+  prepend --mimic
+  prepend "${compiler}"

extractor/autobuilder/autobuilder.go

@@ -0,0 +1,81 @@
+// Package autobuilder implements a simple system that attempts to run build commands for common
+// build frameworks, if the relevant files exist.
+package autobuilder
+
+import (
+	"log"
+	"os"
+	"os/exec"
+
+	"github.com/github/codeql-go/extractor/util"
+)
+
+// CheckExtracted sets whether the autobuilder should check whether source files have been extracted
+// to the CodeQL source directory as well as whether the build command executed successfully.
+var CheckExtracted = false
+
+// checkEmpty checks whether a directory either doesn't exist or is empty.
+func checkEmpty(dir string) (bool, error) {
+	if !util.DirExists(dir) {
+		return true, nil
+	}
+
+	d, err := os.Open(dir)
+	if err != nil {
+		return false, err
+	}
+	defer d.Close()
+
+	names, err := d.Readdirnames(-1)
+	if err != nil {
+		return false, err
+	}
+	return len(names) == 0, nil
+}
+
+// checkExtractorRun checks whether the CodeQL Go extractor has run, by checking if the source
+// archive directory is empty or not.
+func checkExtractorRun() bool {
+	srcDir := os.Getenv("CODEQL_EXTRACTOR_GO_SOURCE_ARCHIVE_DIR")
+	if srcDir != "" {
+		empty, err := checkEmpty(srcDir)
+		if err != nil {
+			log.Fatalf("Unable to read source archive directory %s.", srcDir)
+		}
+		if empty {
+			log.Printf("No Go code seen; continuing to try other builds.")
+			return false
+		}
+		return true
+	} else {
+		log.Fatalf("No source directory set.\nThis binary should not be run manually; instead, use the CodeQL CLI or VSCode extension. See https://securitylab.github.com/tools/codeql.")
+		return false
+	}
+}
+
+// tryBuildIfExists tries to run the command `cmd args...` if the file `buildFile` exists and is not
+// a directory. Returns true if the command was successful and false if not.
+func tryBuildIfExists(buildFile, cmd string, args ...string) bool {
+	if util.FileExists(buildFile) {
+		log.Printf("%s found.\n", buildFile)
+		return tryBuild(cmd, args...)
+	}
+	return false
+}
+
+// tryBuild tries to run `cmd args...`, returning true if successful and false if not.
+func tryBuild(cmd string, args ...string) bool {
+	log.Printf("Trying build command %s %v", cmd, args)
+	res := util.RunCmd(exec.Command(cmd, args...))
+	return res && (!CheckExtracted || checkExtractorRun())
+}
+
+// Autobuild attempts to detect build system and run the corresponding command.
+func Autobuild() bool {
+	return tryBuildIfExists("Makefile", "make") ||
+		tryBuildIfExists("makefile", "make") ||
+		tryBuildIfExists("GNUmakefile", "make") ||
+		tryBuildIfExists("build.ninja", "ninja") ||
+		tryBuildIfExists("build", "./build") ||
+		tryBuildIfExists("build.sh", "./build.sh")
+}

extractor/cli/go-autobuilder/go-autobuilder.go

@@ -13,6 +13,7 @@ import (
 	"runtime"
 	"strings"
 
+	"github.com/github/codeql-go/extractor/autobuilder"
 	"github.com/github/codeql-go/extractor/util"
 )
 
@@ -68,29 +69,10 @@ func getEnvGoSemVer() string {
 	return "v" + goVersion[2:]
 }
 
-func run(cmd *exec.Cmd) bool {
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	in, _ := cmd.StdinPipe()
-	err := cmd.Start()
-	if err != nil {
-		log.Printf("Running %s failed, continuing anyway: %s\n", cmd.Path, err.Error())
-		return false
-	}
-	in.Close()
-	err = cmd.Wait()
-	if err != nil {
-		log.Printf("Running %s failed, continuing anyway: %s\n", cmd.Path, err.Error())
-		return false
-	}
-
-	return true
-}
-
 func tryBuild(buildFile, cmd string, args ...string) bool {
 	if util.FileExists(buildFile) {
 		log.Printf("%s found, running %s\n", buildFile, cmd)
-		return run(exec.Command(cmd, args...))
+		return util.RunCmd(exec.Command(cmd, args...))
 	}
 	return false
 }
@@ -209,7 +191,7 @@ func (m ModMode) argsForGoVersion(version string) []string {
 // addVersionToMod add a go version directive, e.g. `go 1.14` to a `go.mod` file.
 func addVersionToMod(goMod []byte, version string) bool {
 	cmd := exec.Command("go", "mod", "edit", "-go="+version)
-	return run(cmd)
+	return util.RunCmd(cmd)
 }
 
 // checkVendor tests to see whether a vendor directory is inconsistent according to the go frontend
@@ -422,13 +404,8 @@ func main() {
 	inst := util.Getenv("CODEQL_EXTRACTOR_GO_BUILD_COMMAND", "LGTM_INDEX_BUILD_COMMAND")
 	shouldInstallDependencies := false
 	if inst == "" {
-		// if there is a build file, run the corresponding build tool
-		buildSucceeded := tryBuild("Makefile", "make") ||
-			tryBuild("makefile", "make") ||
-			tryBuild("GNUmakefile", "make") ||
-			tryBuild("build.ninja", "ninja") ||
-			tryBuild("build", "./build") ||
-			tryBuild("build.sh", "./build.sh")
+		// try to build the project
+		buildSucceeded := autobuilder.Autobuild()
 
 		if !buildSucceeded {
 			// Build failed; we'll try to install dependencies ourselves
@@ -464,7 +441,7 @@ func main() {
 		}
 		os.Chmod(script.Name(), 0700)
 		log.Println("Installing dependencies using custom build command.")
-		run(exec.Command(script.Name()))
+		util.RunCmd(exec.Command(script.Name()))
 	}
 
 	if modMode == ModVendor {
@@ -525,7 +502,7 @@ func main() {
 				install = exec.Command("go", "get", "-v", "./...")
 				log.Println("Installing dependencies using `go get -v ./...`.")
 			}
-			run(install)
+			util.RunCmd(install)
 		}
 	}
 

extractor/cli/go-build-runner/go-build-runner.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+	"github.com/github/codeql-go/extractor/util"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+
+	"github.com/github/codeql-go/extractor/autobuilder"
+)
+
+func main() {
+	// check if a build command has successfully extracted something
+	autobuilder.CheckExtracted = true
+	if autobuilder.Autobuild() {
+		return
+	}
+
+	// if the autobuilder fails, invoke the extractor manually
+	// we cannot simply call `go build` here, because the tracer is not able to trace calls made by
+	// this binary
+	log.Printf("No build commands succeeded, falling back to go build ./...")
+
+	mypath, err := os.Executable()
+	if err != nil {
+		log.Fatalf("Could not determine path of extractor: %v.\n", err)
+	}
+	extractor := filepath.Join(filepath.Dir(mypath), "go-extractor")
+	if runtime.GOOS == "windows" {
+		extractor = extractor + ".exe"
+	}
+
+	util.RunCmd(exec.Command(extractor, "./..."))
+}

extractor/cli/go-extractor/go-extractor.go

@@ -20,23 +20,70 @@ func usage() {
 	fmt.Fprintf(os.Stderr, "--help                Print this help.\n")
 }
 
-func parseFlags(args []string) ([]string, []string) {
+func parseFlags(args []string, mimic bool) ([]string, []string) {
 	i := 0
 	buildFlags := []string{}
-	for i < len(args) && strings.HasPrefix(args[i], "-") {
+	for ; i < len(args) && strings.HasPrefix(args[i], "-"); i++ {
 		if args[i] == "--" {
 			i++
 			break
 		}
 
-		if args[i] == "--help" {
-			usage()
-			os.Exit(0)
-		} else {
-			buildFlags = append(buildFlags, args[i])
+		if !mimic {
+			// we're not in mimic mode, try to parse our arguments
+			switch args[i] {
+			case "--help":
+				usage()
+				os.Exit(0)
+			case "--mimic":
+				if i+1 < len(args) {
+					i++
+					compiler := args[i]
+					log.Printf("Compiler: %s", compiler)
+					if i+1 < len(args) {
+						i++
+						command := args[i]
+						if command == "build" || command == "install" || command == "run" {
+							log.Printf("Intercepting build")
+							return parseFlags(args[i+1:], true)
+						} else {
+							log.Printf("Non-build command '%s'; skipping", strings.Join(args[1:], " "))
+							os.Exit(0)
+						}
+					} else {
+						log.Printf("Non-build command '%s'; skipping", strings.Join(args[1:], " "))
+						os.Exit(0)
+					}
+				} else {
+					log.Fatalf("--mimic requires an argument, e.g. --mimic go")
+				}
+			}
 		}
 
-		i++
+		// parse go build flags
+		switch args[i] {
+		// skip `-o output` and `-i`, if applicable
+		case "-o":
+			if i+1 < len(args) {
+				i++
+			}
+		case "-i":
+		case "-p", "-asmflags", "-buildmode", "-compiler", "-gccgoflags", "-gcflags", "-installsuffix",
+			"-ldflags", "-mod", "-modfile", "-pkgdir", "-tags", "-toolexec":
+			if i+1 < len(args) {
+				buildFlags = append(buildFlags, args[i], args[i+1])
+				i++
+			} else {
+				buildFlags = append(buildFlags, args[i])
+			}
+		default:
+			if strings.HasPrefix(args[i], "-") {
+				buildFlags = append(buildFlags, args[i])
+			} else {
+				// stop parsing if the argument is not a flag (and so is positional)
+				break
+			}
+		}
 	}
 
 	cpuprofile = os.Getenv("CODEQL_EXTRACTOR_GO_CPU_PROFILE")
@@ -46,7 +93,7 @@ func parseFlags(args []string) ([]string, []string) {
 }
 
 func main() {
-	buildFlags, patterns := parseFlags(os.Args[1:])
+	buildFlags, patterns := parseFlags(os.Args[1:], false)
 
 	if cpuprofile != "" {
 		f, err := os.Create(cpuprofile)
@@ -63,9 +110,10 @@ func main() {
 	if len(patterns) == 0 {
 		log.Println("Nothing to extract.")
 	} else {
+		log.Printf("Build flags: '%s'; patterns: '%s'\n", strings.Join(buildFlags, " "), strings.Join(patterns, " "))
 		err := extractor.ExtractWithFlags(buildFlags, patterns)
 		if err != nil {
-			log.Fatal(err)
+			log.Fatalf("Error running go tooling: %s\n", err.Error())
 		}
 	}
 

extractor/extractor.go

@@ -122,6 +122,7 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
 			log.Fatalf("Unable to get a source directory for input package %s.", pkg.PkgPath)
 		}
 		wantedRoots[pkgRoots[pkg.PkgPath]] = true
+		wantedRoots[pkgDirs[pkg.PkgPath]] = true
 	}
 
 	log.Println("Done processing dependencies.")