chore: make integration tests more robust

- Add generic functions to check received events in different manners:
  - ExpectAllInOrder
  - ExpectAllEqualTo
  - ExpectAtLeastOneOfEach
  - ExpectAnyOfEach
- Add waitForTraceeOutputEvents function to wait for tracee output
  events until buffer fills with certain number of events or
  timeout occurs.
- Add waitforTraceeStop function to wait for tracee to stop or until
  timeout occurs.
- Remove tester.sh concentrating all event filter tests in
  event_filters_test.go.
- Create syscaller tool to easily generate syscalls from the command
  line (strictly for integration tests). It reuses the tracee events
  package.
- Fix goroutine leaks, deadlocks and race conditions.

.gitignore

@@ -26,6 +26,7 @@ compile_commands.json
 
 # binaries and build files
 dist
+tests/integration/syscaller/cmd/syscaller
 
 # release files
 release_notes.txt

Makefile

@@ -36,6 +36,7 @@ CMD_STATICCHECK ?= staticcheck
 CMD_STRIP ?= llvm-strip
 CMD_TOUCH ?= touch
 CMD_TR ?= tr
+CMD_SYSCALLER ?= ./tests/integration/syscaller/cmd/syscaller
 
 .check_%:
 #
@@ -727,9 +728,22 @@ test-types: \
 		-coverprofile=coverage.txt \
 		./...
 
+#
+# syscaller (required for integration tests)
+#
+
+.PHONY: $(CMD_SYSCALLER)
+$(CMD_SYSCALLER): \
+	$(OUTPUT_DIR)/libbpf/libbpf.a \
+	| .check_$(CMD_GO)
+#
+	$(GO_ENV_EBPF) \
+	$(CMD_GO) build -o $(CMD_SYSCALLER) $(dir $(CMD_SYSCALLER))
+
 .PHONY: test-integration
 test-integration: \
 	.checkver_$(CMD_GO) \
+	$(CMD_SYSCALLER) \
 	tracee-ebpf
 #
 	$(GO_ENV_EBPF) \
@@ -739,6 +753,7 @@ test-integration: \
 			-extldflags \"$(CGO_EXT_LDFLAGS_EBPF)\" \
 			-X main.version=\"$(VERSION)\" \
 			" \
+		-race \
 		-v \
 		-p 1 \
 		-count=1 \
@@ -829,3 +844,4 @@ clean:
 	$(CMD_RM) -f .*.md5
 	$(CMD_RM) -f .check*
 	$(CMD_RM) -f .*-pkgs*
+	$(CMD_RM) -f $(CMD_SYSCALLER)

go.mod

@@ -23,6 +23,7 @@ require (
 	github.com/spf13/viper v1.15.0
 	github.com/stretchr/testify v1.8.2
 	github.com/urfave/cli/v2 v2.3.0
+	go.uber.org/goleak v1.1.12
 	go.uber.org/zap v1.24.0
 	golang.org/x/sys v0.8.0
 	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1

go.sum

@@ -464,6 +464,7 @@ go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqe
 go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
 go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
 go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA=
+go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
@@ -504,6 +505,7 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu
 golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug=
 golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=

tests/integration/cpu/cpu.go

@@ -0,0 +1,18 @@
+package cpu
+
+import "golang.org/x/sys/unix"
+
+const CPUForTests = 0 // CPU to pin test processes to
+
+// SetCPUs pins the current process to a specific CPU
+func SetCPUs(id ...int) {
+	if len(id) == 0 {
+		id = append(id, CPUForTests)
+	}
+
+	cpuMask := unix.CPUSet{}
+	for _, i := range id {
+		cpuMask.Set(i)
+	}
+	_ = unix.SchedSetaffinity(0, &cpuMask)
+}