instrgen : Almost complete rewrite of instrgen internals, new way of loading and analyzing packages used by whole go program. utilizing toolexec

.github/dependabot.yml

@@ -190,15 +190,6 @@ updates:
     schedule:
       interval: weekly
       day: sunday
-  - package-ecosystem: gomod
-    directory: /instrgen/driver/testdata/interface
-    labels:
-      - dependencies
-      - go
-      - Skip Changelog
-    schedule:
-      interval: weekly
-      day: sunday
   - package-ecosystem: gomod
     directory: /instrumentation/github.com/aws/aws-lambda-go/otellambda
     labels:

instrgen/README.md

@@ -7,31 +7,62 @@ If you are looking for more details about internal working, see [How it works](.
 
 :construction: This package is currently work in progress.
 
-## How to use it
+## Build
 
-In order to instrument your project you have to add following call in your entry point function, usually main
-(you can look at testdata directory for reference) and invoke instrgen tool.
+From driver directory execute:
 
 ```
-func main() {
-    rtlib.AutotelEntryPoint()
+go build
 ```
 
-Instrgen requires three parameters: command, path to project and package(s) pattern we
-would like to instrument.
+## Prerequisites
+
+`instrgen` driver utility needs to be on your PATH environment variable.
+
+## How to use it
+
+Instrgen has to be invoked from main module directory and
+requires three parameters: command, directory (files from specified directory will be rewritten).
 
 ```
-./instrgen --inject [path to your go project] [package(s) pattern]
+./driver --inject [file pattern] [replace input source] [entry point]
 ```
 
 Below concrete example with one of test instrumentation that is part of the project.
 
 ```
-./instrgen --inject ./testdata/basic ./...
+driver --inject  /testdata/basic yes main.main
+```
+
+Above command will invoke golang compiler under the hood:
+
 ```
+go build -work -a -toolexec driver
+```
+
+which means that the above command can be executed directly, however first `instrgen_cmd.json`
+configuration file needs to be provided. This file is created internally by `driver` based on provided
+command line.
+
+Below example content of `instrgen_cmd.json`:
+
+```
+{
+"ProjectPath": ".",
+"FilePattern": "/testdata/basic",
+"Cmd": "inject",
+"Replace": "yes",
+"EntryPoint": {
+    "Pkg": "main",
+    "FunName": "main"
+ }
+}
+```
+
+### Work in progress:
 
-```./...``` works like wildcard in this case and it will instrument all packages in this path, but it can be invoked with
-specific package as well.
+Library instrumentation:
+- HTTP
 
 ### Compatibility
 

instrgen/docs/how-it-works.md

@@ -3,10 +3,9 @@
 `instrgen` adds OpenTelemetry instrumentation to source code by directly modifying it.
 It uses the AST (Abstract Syntax Tree) representation of the code to determine its operational flow and injects necessary OpenTelemetry functionality into the AST.
 
+`instrgen` utilizes toolexec golang compiler switch. It means that it has access to all files
+that takes part in the compilation process.
+
 The AST modification algorithm is the following:
-1. Search for the entry point: a function definition with `AutotelEntryPoint()`.
-2. Build the call graph. Traverse all calls from the entry point through all function definitions.
-3. Inject OpenTelemetry instrumentation into functions bodies.
-4. Context propagation. Adding an additional context parameter to all function declarations and function call expressions that are visible
-   (it will not add a context argument to call expressions if they are not reachable from the entry point).
-![image info](./flow.png)
+1. Rewrites go runtime package in order to provide correct context propagation.
+2. Inject OpenTelemetry instrumentation into functions bodies.

instrgen/driver/go.mod

@@ -7,13 +7,12 @@ replace go.opentelemetry.io/contrib/instrgen => ../
 require (
 	github.com/stretchr/testify v1.8.4
 	go.opentelemetry.io/contrib/instrgen v0.0.0-00010101000000-000000000000
+	golang.org/x/tools v0.14.0
 )
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	golang.org/x/mod v0.13.0 // indirect
 	golang.org/x/sys v0.13.0 // indirect
-	golang.org/x/tools v0.14.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

instrgen/driver/go.sum

@@ -5,8 +5,6 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY=
-golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
 golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc=

instrgen/driver/instrgen_test.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -27,70 +28,56 @@ import (
 	"github.com/stretchr/testify/require"
 
 	alib "go.opentelemetry.io/contrib/instrgen/lib"
+	"go.opentelemetry.io/contrib/instrgen/rewriters"
 )
 
 var testcases = map[string]string{
-	"./testdata/basic":     "./testdata/expected/basic",
-	"./testdata/selector":  "./testdata/expected/selector",
-	"./testdata/interface": "./testdata/expected/interface",
+	"testdata/basic":     "testdata/expected/basic",
+	"testdata/interface": "testdata/expected/interface",
 }
 
 var failures []string
 
-func inject(t *testing.T, root string, packagePattern string) {
-	err := executeCommand("--inject-dump-ir", root, packagePattern)
-	require.NoError(t, err)
+func TestCommand(t *testing.T) {
+	executor := &NullExecutor{}
+	err := executeCommand("--unknown", "./testdata/basic", "testdata/basic", "yes", "main.main", executor)
+	assert.Error(t, err)
 }
 
-func TestCommands(t *testing.T) {
-	err := executeCommand("--dumpcfg", "./testdata/dummy", "./...")
-	require.NoError(t, err)
-	err = executeCommand("--rootfunctions", "./testdata/dummy", "./...")
-	require.NoError(t, err)
-	err = executeCommand("--prune", "./testdata/dummy", "./...")
-	require.NoError(t, err)
-	err = executeCommand("--inject", "./testdata/dummy", "./...")
-	require.NoError(t, err)
-	err = usage()
-	require.NoError(t, err)
-}
-
-func TestCallGraph(t *testing.T) {
-	cg := makeCallGraph("./testdata/dummy", "./...")
-	dumpCallGraph(cg)
-	assert.Equal(t, len(cg), 0, "callgraph should contain 0 elems")
-	rf := makeRootFunctions("./testdata/dummy", "./...")
-	dumpRootFunctions(rf)
-	assert.Equal(t, len(rf), 0, "rootfunctions set should be empty")
-}
+func TestInstrumentation(t *testing.T) {
+	cwd, _ := os.Getwd()
+	var args []string
+	for k := range testcases {
+		filePaths := make(map[string]int)
 
-func TestArgs(t *testing.T) {
-	err := checkArgs(nil)
-	require.Error(t, err)
-	args := []string{"driver", "--inject", "", "./..."}
-	err = checkArgs(args)
-	require.NoError(t, err)
-}
+		files := alib.SearchFiles(k, ".go")
+		for index, file := range files {
+			filePaths[file] = index
+		}
+		pruner := rewriters.OtelPruner{
+			FilePattern: k, Replace: true}
+		analyzePackage(pruner, "main", filePaths, nil, "", args)
 
-func TestUnknownCommand(t *testing.T) {
-	err := executeCommand("unknown", "a", "b")
-	require.Error(t, err)
-}
+		rewriter := rewriters.BasicRewriter{
+			FilePattern: k, Replace: "yes", Pkg: "main", Fun: "main"}
+		analyzePackage(rewriter, "main", filePaths, nil, "", args)
+	}
+	fmt.Println(cwd)
 
-func TestInstrumentation(t *testing.T) {
 	for k, v := range testcases {
-		inject(t, k, "./...")
-		files := alib.SearchFiles(k, ".go_pass_tracing")
-		expectedFiles := alib.SearchFiles(v, ".go")
+		files := alib.SearchFiles(cwd+"/"+k, ".go")
+		expectedFiles := alib.SearchFiles(cwd+"/"+v, ".go")
 		numOfFiles := len(expectedFiles)
 		fmt.Println("Go Files:", len(files))
 		fmt.Println("Expected Go Files:", len(expectedFiles))
+		assert.True(t, len(files) > 0)
 		numOfComparisons := 0
 		for _, file := range files {
 			fmt.Println(filepath.Base(file))
 			for _, expectedFile := range expectedFiles {
 				fmt.Println(filepath.Base(expectedFile))
-				if filepath.Base(file) == filepath.Base(expectedFile+"_pass_tracing") {
+				if filepath.Base(file) == filepath.Base(expectedFile) {
+					fmt.Println(file, " : ", expectedFile)
 					f1, err1 := os.ReadFile(file)
 					require.NoError(t, err1)
 					f2, err2 := os.ReadFile(expectedFile)
@@ -106,12 +93,120 @@ func TestInstrumentation(t *testing.T) {
 			fmt.Println("numberOfComparisons:", numOfComparisons)
 			panic("not all files were compared")
 		}
-		_, err := Prune(k, "./...", false)
-		if err != nil {
-			fmt.Println("Prune failed")
-		}
 	}
-	for _, f := range failures {
-		fmt.Println("FAILURE : ", f)
+}
+
+type NullExecutor struct {
+}
+
+func (executor *NullExecutor) Execute(_ string, _ []string) {
+}
+
+func (executor *NullExecutor) Run() error {
+	return nil
+}
+
+func TestToolExecMain(t *testing.T) {
+	for k := range testcases {
+		var args []string
+		files := alib.SearchFiles(k, ".go")
+		args = append(args, []string{"-o", "/tmp/go-build", "-p", "main", "-pack", "-asmhdr", "go_asm.h"}...)
+		args = append(args, files...)
+		instrgenCfg := InstrgenCmd{FilePattern: k, Cmd: "prune", Replace: "yes",
+			EntryPoint: EntryPoint{Pkg: "main", FunName: "main"}}
+		rewriterS := makeRewriters(instrgenCfg)
+		analyze(args, rewriterS)
+		instrgenCfg.Cmd = "inject"
+		rewriterS = makeRewriters(instrgenCfg)
+		analyze(args, rewriterS)
+	}
+	for k := range testcases {
+		var args []string
+		files := alib.SearchFiles(k, ".go")
+		args = append(args, []string{"-pack", "-asmhdr", "go_asm.h"}...)
+		args = append(args, files...)
+		instrgenCfg := InstrgenCmd{FilePattern: k, Cmd: "prune", Replace: "no",
+			EntryPoint: EntryPoint{Pkg: "main", FunName: "main"}}
+		rewriterS := makeRewriters(instrgenCfg)
+		analyze(args, rewriterS)
+		instrgenCfg.Cmd = "inject"
+		rewriterS = makeRewriters(instrgenCfg)
+		analyze(args, rewriterS)
+	}
+	for k := range testcases {
+		instrgenCfg := InstrgenCmd{FilePattern: k, Cmd: "prune", Replace: "yes",
+			EntryPoint: EntryPoint{Pkg: "main", FunName: "main"}}
+		rewriterS := makeRewriters(instrgenCfg)
+		var args []string
+		executor := &NullExecutor{}
+		err := toolExecMain(args, rewriterS, executor)
+		assert.Error(t, err)
+	}
+}
+
+func TestGetCommandName(t *testing.T) {
+	cmd := GetCommandName([]string{"/usr/local/go/compile"})
+	assert.True(t, cmd == "compile")
+	cmd = GetCommandName([]string{"/usr/local/go/compile.exe"})
+	assert.True(t, cmd == "compile")
+	cmd = GetCommandName([]string{})
+	assert.True(t, cmd == "")
+}
+
+func TestExecutePass(t *testing.T) {
+	executor := &ToolExecutor{}
+	require.NoError(t, executePass([]string{"go", "version"}, executor))
+}
+
+func TestDriverMain(t *testing.T) {
+	executor := &NullExecutor{}
+	{
+		err := os.Remove("instrgen_cmd.json")
+		_ = err
+		var args []string
+		args = append(args, "compile")
+		err = driverMain(args, executor)
+		require.Error(t, err)
+	}
+	for k := range testcases {
+		var args []string
+		files := alib.SearchFiles(k, ".go")
+		args = append(args, []string{"-o", "/tmp/go-build", "-p", "main", "-pack", "-asmhdr", "go_asm.h"}...)
+		args = append(args, files...)
+		instrgenCfg := InstrgenCmd{FilePattern: k, Cmd: "prune", Replace: "yes",
+			EntryPoint: EntryPoint{Pkg: "main", FunName: "main"}}
+		err := driverMain(args, executor)
+		assert.NoError(t, err)
+		instrgenCfg.Cmd = "inject"
+		err = driverMain(args, executor)
+		assert.NoError(t, err)
+	}
+	{
+		var args []string
+		args = append(args, "compile")
+		instrgenCfg := InstrgenCmd{FilePattern: "/testdata/basic", Cmd: "inject", Replace: "yes",
+			EntryPoint: EntryPoint{Pkg: "main", FunName: "main"}}
+		file, _ := json.MarshalIndent(instrgenCfg, "", " ")
+		err := os.WriteFile("instrgen_cmd.json", file, 0644)
+		require.NoError(t, err)
+		err = driverMain(args, executor)
+		require.NoError(t, err)
+	}
+	for k := range testcases {
+		var args []string
+		args = append(args, []string{"--inject", k, "yes", "main.main"}...)
+		err := driverMain(args, executor)
+		assert.NoError(t, err)
+	}
+	{
+		var args []string
+		args = append(args, "--inject")
+		err := driverMain(args, executor)
+		assert.Error(t, err)
+	}
+	{
+		var args []string
+		err := driverMain(args, executor)
+		assert.NoError(t, err)
 	}
 }