From 16796529fcad518ffa50dca7eb5dbc71493c5ab7 Mon Sep 17 00:00:00 2001 From: Sauyon Lee Date: Wed, 23 Sep 2020 04:25:01 -0700 Subject: [PATCH 01/10] Add package directory as a wanted root --- extractor/extractor.go | 1 + 1 file changed, 1 insertion(+) diff --git a/extractor/extractor.go b/extractor/extractor.go index 814ed9fe0..4661534c4 100644 --- a/extractor/extractor.go +++ b/extractor/extractor.go @@ -122,6 +122,7 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error { log.Fatalf("Unable to get a source directory for input package %s.", pkg.PkgPath) } wantedRoots[pkgRoots[pkg.PkgPath]] = true + wantedRoots[pkgDirs[pkg.PkgPath]] = true } log.Println("Done processing dependencies.") From eaf5342b7d31305ec00117d7d35d04cf6c5add4f Mon Sep 17 00:00:00 2001 From: Sauyon Lee Date: Wed, 23 Sep 2020 04:27:09 -0700 Subject: [PATCH 02/10] Enable Go modules while determining module directory --- extractor/util/util.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/extractor/util/util.go b/extractor/util/util.go index a59222467..6d0db2f67 100644 --- a/extractor/util/util.go +++ b/extractor/util/util.go @@ -28,9 +28,14 @@ func Getenv(key string, aliases ...string) string { // runGoList is a helper function for running go list with format `format` and flags `flags` on // package `pkgpath`. func runGoList(format string, pkgpath string, flags ...string) (string, error) { + return runGoListWithEnv(format, pkgpath, nil, flags...) +} + +func runGoListWithEnv(format string, pkgpath string, additionalEnv []string, flags ...string) (string, error) { args := append([]string{"list", "-e", "-f", format}, flags...) args = append(args, pkgpath) cmd := exec.Command("go", args...) + cmd.Env = append(os.Environ(), additionalEnv...) out, err := cmd.Output() if err != nil { @@ -48,13 +53,15 @@ func runGoList(format string, pkgpath string, flags ...string) (string, error) { // GetModDir gets the absolute directory of the module containing the package with path // `pkgpath`. It passes the `go list` the flags specified by `flags`. func GetModDir(pkgpath string, flags ...string) string { - mod, err := runGoList("{{.Module}}", pkgpath, flags...) + // enable module mode so that we can find a module root if it exists, even if go module support is + // disabled by a build + mod, err := runGoListWithEnv("{{.Module}}", pkgpath, []string{"GO111MODULE=on"}, flags...) if err != nil || mod == "" { // if the command errors or modules aren't being used, return the empty string return "" } - modDir, err := runGoList("{{.Module.Dir}}", pkgpath, flags...) + modDir, err := runGoListWithEnv("{{.Module.Dir}}", pkgpath, []string{"GO111MODULE=on"}, flags...) if err != nil { return "" } From 2da89c65276723f2891b5c82b9c96219d1273107 Mon Sep 17 00:00:00 2001 From: Sauyon Lee Date: Thu, 10 Sep 2020 00:45:52 -0700 Subject: [PATCH 03/10] extractor: factor out `run` from autobuilder --- .../cli/go-autobuilder/go-autobuilder.go | 27 +++---------------- extractor/util/util.go | 19 +++++++++++++ 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/extractor/cli/go-autobuilder/go-autobuilder.go b/extractor/cli/go-autobuilder/go-autobuilder.go index fade9194f..086017222 100644 --- a/extractor/cli/go-autobuilder/go-autobuilder.go +++ b/extractor/cli/go-autobuilder/go-autobuilder.go @@ -68,29 +68,10 @@ func getEnvGoSemVer() string { return "v" + goVersion[2:] } -func run(cmd *exec.Cmd) bool { - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - in, _ := cmd.StdinPipe() - err := cmd.Start() - if err != nil { - log.Printf("Running %s failed, continuing anyway: %s\n", cmd.Path, err.Error()) - return false - } - in.Close() - err = cmd.Wait() - if err != nil { - log.Printf("Running %s failed, continuing anyway: %s\n", cmd.Path, err.Error()) - return false - } - - return true -} - func tryBuild(buildFile, cmd string, args ...string) bool { if util.FileExists(buildFile) { log.Printf("%s found, running %s\n", buildFile, cmd) - return run(exec.Command(cmd, args...)) + return util.RunCmd(exec.Command(cmd, args...)) } return false } @@ -209,7 +190,7 @@ func (m ModMode) argsForGoVersion(version string) []string { // addVersionToMod add a go version directive, e.g. `go 1.14` to a `go.mod` file. func addVersionToMod(goMod []byte, version string) bool { cmd := exec.Command("go", "mod", "edit", "-go="+version) - return run(cmd) + return util.RunCmd(cmd) } // checkVendor tests to see whether a vendor directory is inconsistent according to the go frontend @@ -464,7 +445,7 @@ func main() { } os.Chmod(script.Name(), 0700) log.Println("Installing dependencies using custom build command.") - run(exec.Command(script.Name())) + util.RunCmd(exec.Command(script.Name())) } if modMode == ModVendor { @@ -525,7 +506,7 @@ func main() { install = exec.Command("go", "get", "-v", "./...") log.Println("Installing dependencies using `go get -v ./...`.") } - run(install) + util.RunCmd(install) } } diff --git a/extractor/util/util.go b/extractor/util/util.go index 6d0db2f67..9bd253d54 100644 --- a/extractor/util/util.go +++ b/extractor/util/util.go @@ -107,3 +107,22 @@ func DirExists(filename string) bool { } return err == nil && info.IsDir() } + +func RunCmd(cmd *exec.Cmd) bool { + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + in, _ := cmd.StdinPipe() + err := cmd.Start() + if err != nil { + log.Printf("Running %s failed, continuing anyway: %s\n", cmd.Path, err.Error()) + return false + } + in.Close() + err = cmd.Wait() + if err != nil { + log.Printf("Running %s failed, continuing anyway: %s\n", cmd.Path, err.Error()) + return false + } + + return true +} From cd63ea84aabb6414f3471cd4cb99480c245a148f Mon Sep 17 00:00:00 2001 From: Sauyon Lee Date: Thu, 10 Sep 2020 00:46:36 -0700 Subject: [PATCH 04/10] extractor: revamp argument parsing --- extractor/cli/go-extractor/go-extractor.go | 68 ++++++++++++++++++---- 1 file changed, 58 insertions(+), 10 deletions(-) diff --git a/extractor/cli/go-extractor/go-extractor.go b/extractor/cli/go-extractor/go-extractor.go index 8ab34255e..4b2dbe229 100644 --- a/extractor/cli/go-extractor/go-extractor.go +++ b/extractor/cli/go-extractor/go-extractor.go @@ -20,23 +20,70 @@ func usage() { fmt.Fprintf(os.Stderr, "--help Print this help.\n") } -func parseFlags(args []string) ([]string, []string) { +func parseFlags(args []string, mimic bool) ([]string, []string) { i := 0 buildFlags := []string{} - for i < len(args) && strings.HasPrefix(args[i], "-") { + for ; i < len(args) && strings.HasPrefix(args[i], "-"); i++ { if args[i] == "--" { i++ break } - if args[i] == "--help" { - usage() - os.Exit(0) - } else { - buildFlags = append(buildFlags, args[i]) + if !mimic { + // we're not in mimic mode, try to parse our arguments + switch args[i] { + case "--help": + usage() + os.Exit(0) + case "--mimic": + if i+1 < len(args) { + i++ + compiler := args[i] + log.Printf("Compiler: %s", compiler) + if i+1 < len(args) { + i++ + command := args[i] + if command == "build" || command == "install" || command == "run" { + log.Printf("Intercepting build") + return parseFlags(args[i+1:], true) + } else { + log.Printf("Non-build command '%s'; skipping", strings.Join(args[1:], " ")) + os.Exit(0) + } + } else { + log.Printf("Non-build command '%s'; skipping", strings.Join(args[1:], " ")) + os.Exit(0) + } + } else { + log.Fatalf("Invalid --mimic: no compiler specified") + } + } } - i++ + // parse go build flags + switch args[i] { + // skip `-o output` and `-i`, if applicable + case "-o": + if i+1 < len(args) { + i++ + } + case "-i": + case "-p", "-asmflags", "-buildmode", "-compiler", "-gccgoflags", "-gcflags", "-installsuffix", + "-ldflags", "-mod", "-modfile", "-pkgdir", "-tags", "-toolexec": + if i+1 < len(args) { + buildFlags = append(buildFlags, args[i], args[i+1]) + i++ + } else { + buildFlags = append(buildFlags, args[i]) + } + default: + if strings.HasPrefix(args[i], "-") { + buildFlags = append(buildFlags, args[i]) + } else { + // stop parsing if the argument is not a flag (and so is positional) + break + } + } } cpuprofile = os.Getenv("CODEQL_EXTRACTOR_GO_CPU_PROFILE") @@ -46,7 +93,7 @@ func parseFlags(args []string) ([]string, []string) { } func main() { - buildFlags, patterns := parseFlags(os.Args[1:]) + buildFlags, patterns := parseFlags(os.Args[1:], false) if cpuprofile != "" { f, err := os.Create(cpuprofile) @@ -63,9 +110,10 @@ func main() { if len(patterns) == 0 { log.Println("Nothing to extract.") } else { + log.Printf("Build flags: '%s'; patterns: '%s'\n", strings.Join(buildFlags, " "), strings.Join(patterns, " ")) err := extractor.ExtractWithFlags(buildFlags, patterns) if err != nil { - log.Fatal(err) + log.Fatalf("Error running go tooling: %s\n", err.Error()) } } From de0582a67ff6be962dfceda58fa32ae279d41152 Mon Sep 17 00:00:00 2001 From: Sauyon Lee Date: Thu, 10 Sep 2020 00:47:07 -0700 Subject: [PATCH 05/10] autobuilder: extract out attempted build commands --- extractor/autobuilder/autobuilder.go | 81 +++++++++++++++++++ .../cli/go-autobuilder/go-autobuilder.go | 10 +-- 2 files changed, 84 insertions(+), 7 deletions(-) create mode 100644 extractor/autobuilder/autobuilder.go diff --git a/extractor/autobuilder/autobuilder.go b/extractor/autobuilder/autobuilder.go new file mode 100644 index 000000000..f31e3b6ff --- /dev/null +++ b/extractor/autobuilder/autobuilder.go @@ -0,0 +1,81 @@ +// Package autobuilder implements a simple system that attempts to run build commands for common +// build frameworks, if the relevant files exist. +package autobuilder + +import ( + "log" + "os" + "os/exec" + + "github.com/github/codeql-go/extractor/util" +) + +// CheckExtracted sets whether the autobuilder should check whether source files have been extracted +// to the CodeQL source directory as well as whether the build command executed successfully. +var CheckExtracted = false + +// checkEmpty checks whether a directory either doesn't exist or is empty. +func checkEmpty(dir string) (bool, error) { + if !util.DirExists(dir) { + return true, nil + } + + d, err := os.Open(dir) + if err != nil { + return false, err + } + defer d.Close() + + names, err := d.Readdirnames(-1) + if err != nil { + return false, err + } + return len(names) == 0, nil +} + +// checkExtractorRun checks whether the CodeQL Go extractor has run, by checking if the source +// archive directory is empty or not. +func checkExtractorRun() bool { + srcDir := os.Getenv("CODEQL_EXTRACTOR_GO_SOURCE_ARCHIVE_DIR") + if srcDir != "" { + empty, err := checkEmpty(srcDir) + if err != nil { + log.Fatalf("Unable to read source archive directory %s.", srcDir) + } + if empty { + log.Printf("No Go code seen; continuing to try other builds.") + return false + } + return true + } else { + log.Fatalf("No source directory set.") + return false + } +} + +// tryBuildIfExists tries to run the command `cmd args...` if the file `buildFile` exists and is not +// a directory. Returns true if the command was successful and false if not. +func tryBuildIfExists(buildFile, cmd string, args ...string) bool { + if util.FileExists(buildFile) { + log.Printf("%s found.\n", buildFile) + return tryBuild(cmd, args...) + } + return false +} + +// tryBuild tries to run `cmd args...`, returning true if successful and false if not. +func tryBuild(cmd string, args ...string) bool { + log.Printf("Trying build command %s %v", cmd, args) + res := util.RunCmd(exec.Command(cmd, args...)) + return res && (!CheckExtracted || checkExtractorRun()) +} + +// Autobuild attempts to detect build system and run the corresponding command. +func Autobuild() bool { + return tryBuildIfExists("Makefile", "make") || + tryBuildIfExists("makefile", "make") || + tryBuildIfExists("GNUmakefile", "make") || + tryBuildIfExists("build.ninja", "ninja") || + tryBuildIfExists("build", "./build") || + tryBuildIfExists("build.sh", "./build.sh") +} diff --git a/extractor/cli/go-autobuilder/go-autobuilder.go b/extractor/cli/go-autobuilder/go-autobuilder.go index 086017222..4344187cb 100644 --- a/extractor/cli/go-autobuilder/go-autobuilder.go +++ b/extractor/cli/go-autobuilder/go-autobuilder.go @@ -13,6 +13,7 @@ import ( "runtime" "strings" + "github.com/github/codeql-go/extractor/autobuilder" "github.com/github/codeql-go/extractor/util" ) @@ -403,13 +404,8 @@ func main() { inst := util.Getenv("CODEQL_EXTRACTOR_GO_BUILD_COMMAND", "LGTM_INDEX_BUILD_COMMAND") shouldInstallDependencies := false if inst == "" { - // if there is a build file, run the corresponding build tool - buildSucceeded := tryBuild("Makefile", "make") || - tryBuild("makefile", "make") || - tryBuild("GNUmakefile", "make") || - tryBuild("build.ninja", "ninja") || - tryBuild("build", "./build") || - tryBuild("build.sh", "./build.sh") + // try to build the project + buildSucceeded := autobuilder.Autobuild() if !buildSucceeded { // Build failed; we'll try to install dependencies ourselves From 85c92251d63af9279d124ab60474725e2a1e0696 Mon Sep 17 00:00:00 2001 From: Sauyon Lee Date: Thu, 10 Sep 2020 00:50:22 -0700 Subject: [PATCH 06/10] Add a new binary for tracing --- Makefile | 4 +-- codeql-tools/autobuild.cmd | 7 +++- codeql-tools/autobuild.sh | 7 +++- codeql-tools/linux64/compiler-tracing.spec | 7 ++++ codeql-tools/osx64/compiler-tracing.spec | 7 ++++ codeql-tools/win64/compiler-tracing.spec | 7 ++++ .../cli/go-build-runner/go-build-runner.go | 36 +++++++++++++++++++ 7 files changed, 71 insertions(+), 4 deletions(-) create mode 100644 codeql-tools/linux64/compiler-tracing.spec create mode 100644 codeql-tools/osx64/compiler-tracing.spec create mode 100644 codeql-tools/win64/compiler-tracing.spec create mode 100644 extractor/cli/go-build-runner/go-build-runner.go diff --git a/Makefile b/Makefile index f209a0e9f..0017dc0c0 100644 --- a/Makefile +++ b/Makefile @@ -14,11 +14,11 @@ CODEQL_PLATFORM = osx64 endif endif -CODEQL_TOOLS = $(addprefix codeql-tools/,autobuild.cmd autobuild.sh index.cmd index.sh) +CODEQL_TOOLS = $(addprefix codeql-tools/,autobuild.cmd autobuild.sh index.cmd index.sh linux64 osx64 win64) EXTRACTOR_PACK_OUT = build/codeql-extractor-go -BINARIES = go-extractor go-tokenizer go-autobuilder go-bootstrap go-gen-dbscheme +BINARIES = go-extractor go-tokenizer go-autobuilder go-build-runner go-bootstrap go-gen-dbscheme .PHONY: tools tools-codeql tools-codeql-full clean autoformat \ tools-linux64 tools-osx64 tools-win64 check-formatting diff --git a/codeql-tools/autobuild.cmd b/codeql-tools/autobuild.cmd index 17f9022ac..aed999876 100644 --- a/codeql-tools/autobuild.cmd +++ b/codeql-tools/autobuild.cmd @@ -4,7 +4,12 @@ SETLOCAL EnableDelayedExpansion rem Some legacy environment variables for the autobuilder. set LGTM_SRC=%CD% -type NUL && "%CODEQL_EXTRACTOR_GO_ROOT%/tools/%CODEQL_PLATFORM%/go-autobuilder.exe" +if "%CODEQL_EXTRACTOR_GO_BUILD_TRACING%"=="on" ( + echo "Tracing enabled" + type NUL && "%CODEQL_EXTRACTOR_GO_ROOT%/tools/%CODEQL_PLATFORM%/go-build-runner.exe" +) else ( + type NUL && "%CODEQL_EXTRACTOR_GO_ROOT%/tools/%CODEQL_PLATFORM%/go-autobuilder.exe" +) exit /b %ERRORLEVEL% ENDLOCAL diff --git a/codeql-tools/autobuild.sh b/codeql-tools/autobuild.sh index e22f3e6fa..466841f79 100755 --- a/codeql-tools/autobuild.sh +++ b/codeql-tools/autobuild.sh @@ -11,4 +11,9 @@ fi LGTM_SRC="$(pwd)" export LGTM_SRC -"$CODEQL_EXTRACTOR_GO_ROOT/tools/$CODEQL_PLATFORM/go-autobuilder" +if [ "${CODEQL_EXTRACTOR_GO_BUILD_TRACING:-}" == "on" ]; then + echo "Tracing enabled" + "$CODEQL_EXTRACTOR_GO_ROOT/tools/$CODEQL_PLATFORM/go-build-runner" +else + "$CODEQL_EXTRACTOR_GO_ROOT/tools/$CODEQL_PLATFORM/go-autobuilder" +fi diff --git a/codeql-tools/linux64/compiler-tracing.spec b/codeql-tools/linux64/compiler-tracing.spec new file mode 100644 index 000000000..2055555c2 --- /dev/null +++ b/codeql-tools/linux64/compiler-tracing.spec @@ -0,0 +1,7 @@ +**/go-autobuilder: + order compiler + trace no +**/go: + invoke ${config_dir}/go-extractor + prepend --mimic + prepend "${compiler}" diff --git a/codeql-tools/osx64/compiler-tracing.spec b/codeql-tools/osx64/compiler-tracing.spec new file mode 100644 index 000000000..2055555c2 --- /dev/null +++ b/codeql-tools/osx64/compiler-tracing.spec @@ -0,0 +1,7 @@ +**/go-autobuilder: + order compiler + trace no +**/go: + invoke ${config_dir}/go-extractor + prepend --mimic + prepend "${compiler}" diff --git a/codeql-tools/win64/compiler-tracing.spec b/codeql-tools/win64/compiler-tracing.spec new file mode 100644 index 000000000..76a6b0114 --- /dev/null +++ b/codeql-tools/win64/compiler-tracing.spec @@ -0,0 +1,7 @@ +**/go-autobuilder.exe: + order compiler + trace no +**/go.exe: + invoke ${config_dir}/go-extractor.exe + prepend --mimic + prepend "${compiler}" diff --git a/extractor/cli/go-build-runner/go-build-runner.go b/extractor/cli/go-build-runner/go-build-runner.go new file mode 100644 index 000000000..118de5caf --- /dev/null +++ b/extractor/cli/go-build-runner/go-build-runner.go @@ -0,0 +1,36 @@ +package main + +import ( + "github.com/github/codeql-go/extractor/util" + "log" + "os" + "os/exec" + "path/filepath" + "runtime" + + "github.com/github/codeql-go/extractor/autobuilder" +) + +func main() { + // check if a build command has successfully extracted something + autobuilder.CheckExtracted = true + if autobuilder.Autobuild() { + return + } + + // if the autobuilder fails, invoke the extractor manually + // we cannot simply call `go build` here, because the tracer is not able to trace calls made by + // this binary + log.Printf("No build commands succeeded, falling back to go build ./...") + + mypath, err := os.Executable() + if err != nil { + log.Fatalf("Could not determine path of extractor: %v.\n", err) + } + extractor := filepath.Join(filepath.Dir(mypath), "go-extractor") + if runtime.GOOS == "windows" { + extractor = extractor + ".exe" + } + + util.RunCmd(exec.Command(extractor, "./...")) +} From 3addb962a9d4c7e4e58c758149ab5db82c018fff Mon Sep 17 00:00:00 2001 From: Sauyon Lee Date: Thu, 10 Sep 2020 00:50:32 -0700 Subject: [PATCH 07/10] Add change note for build tracing --- change-notes/2020-06-11-build-tracing.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 change-notes/2020-06-11-build-tracing.md diff --git a/change-notes/2020-06-11-build-tracing.md b/change-notes/2020-06-11-build-tracing.md new file mode 100644 index 000000000..b500742db --- /dev/null +++ b/change-notes/2020-06-11-build-tracing.md @@ -0,0 +1,5 @@ +lgtm,codescanning +* The Go extractor now supports build tracing, allowing users to supply a build command when + creating databases with the CodeQL CLI or via configuration. It currently only supports projects + that use Go modules. To opt-in, set the environment variable `CODEQL_EXTRACTOR_GO_BUILD_TRACING` + to `on`, or supply a build command. From 3c6626c604ab9b69b56b9a7cef8ffa5ca208a844 Mon Sep 17 00:00:00 2001 From: Sauyon Lee Date: Tue, 15 Sep 2020 07:40:41 -0700 Subject: [PATCH 08/10] Don't trace through problem binaries on OS X See https://github.com/github/semmle-code/pull/37764 --- codeql-tools/osx64/compiler-tracing.spec | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/codeql-tools/osx64/compiler-tracing.spec b/codeql-tools/osx64/compiler-tracing.spec index 2055555c2..9403ade33 100644 --- a/codeql-tools/osx64/compiler-tracing.spec +++ b/codeql-tools/osx64/compiler-tracing.spec @@ -5,3 +5,18 @@ invoke ${config_dir}/go-extractor prepend --mimic prepend "${compiler}" +/usr/bin/codesign: + replace yes + invoke /usr/bin/env + prepend /usr/bin/codesign + trace no +/usr/bin/pkill: + replace yes + invoke /usr/bin/env + prepend /usr/bin/pkill + trace no +/usr/bin/pgrep: + replace yes + invoke /usr/bin/env + prepend /usr/bin/pgrep + trace no From 25eebe95e4c27b3f08ec017338b142f187d5f8ac Mon Sep 17 00:00:00 2001 From: Sauyon Lee Date: Wed, 14 Oct 2020 09:39:34 -0700 Subject: [PATCH 09/10] autobuilder: Clarify error message --- extractor/autobuilder/autobuilder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/autobuilder/autobuilder.go b/extractor/autobuilder/autobuilder.go index f31e3b6ff..11c49735e 100644 --- a/extractor/autobuilder/autobuilder.go +++ b/extractor/autobuilder/autobuilder.go @@ -48,7 +48,7 @@ func checkExtractorRun() bool { } return true } else { - log.Fatalf("No source directory set.") + log.Fatalf("No source directory set.\nThis binary should not be run manually; instead, use the CodeQL CLI or VSCode extension. See https://securitylab.github.com/tools/codeql.") return false } } From e5afd1dcb6c655311ed418c14b3d43d20a5c593f Mon Sep 17 00:00:00 2001 From: Sauyon Lee Date: Wed, 14 Oct 2020 09:43:10 -0700 Subject: [PATCH 10/10] go-extractor: clarify --mimic error message Co-authored-by: Chris Smowton --- extractor/cli/go-extractor/go-extractor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/cli/go-extractor/go-extractor.go b/extractor/cli/go-extractor/go-extractor.go index 4b2dbe229..52fd8ede6 100644 --- a/extractor/cli/go-extractor/go-extractor.go +++ b/extractor/cli/go-extractor/go-extractor.go @@ -55,7 +55,7 @@ func parseFlags(args []string, mimic bool) ([]string, []string) { os.Exit(0) } } else { - log.Fatalf("Invalid --mimic: no compiler specified") + log.Fatalf("--mimic requires an argument, e.g. --mimic go") } } }