[jaeger-v2] add GRPC storage backend integration test (#5259)

## Which problem is this PR solving?
- Resolves GRPC integration test sub-task at #5254

## Description of the changes
- Created a `grpc-integration-test.sh` script to run a jaeger remote
storage and execute the end-to-end test through the OpenTelemetry
Collector pipeline with the jaeger storage extension inside connected to
the remote storage. To have a visualization of this architecture, see
the proposal at #5254
- Separate the GRPC and Badger integration test CI because GRPC need to
be run twice for the v1 and v2 versions.

## How was this change tested?
- Run `./scripts/grpc-integration-test.sh latest` and the whole remote
storage and pipeline will be built and executed for you.
- I also ran a `jaeger-query`component to query traces from the remote
storage for manual checks.
<img width="1440" alt="Screenshot 2024-03-08 at 09 58 13"
src="https://github.com/jaegertracing/jaeger/assets/46216691/2388e8bc-baf9-41bd-8baa-c3d51703fc8e">

## Checklist
- [x] I have read
https://github.com/jaegertracing/jaeger/blob/master/CONTRIBUTING_GUIDELINES.md
- [x] I have signed all commits
- [x] I have added unit tests for the new functionality
- [x] I have run lint and test steps successfully
  - for `jaeger`: `make lint test`
  - for `jaeger-ui`: `yarn lint` and `yarn test`

---------

Signed-off-by: James Ryans <james.ryans2012@gmail.com>

.codecov.yml

@@ -11,6 +11,7 @@ ignore:
   - "thrift-gen/*/*"
   - "**/thrift-0.9.2/*"
   - "**/main.go"
+  - "cmd/jaeger/internal/integration/datareceivers"
   - "examples/hotrod"
 
 coverage:

.github/workflows/ci-grpc-badger.yml → .github/workflows/ci-badger.yaml

@@ -1,4 +1,4 @@
-name: CIT gRPC And Badger
+name: CIT Badger
 
 on:
   push:
@@ -16,7 +16,7 @@ permissions:  # added using https://github.com/step-security/secure-workflows
   contents: read
 
 jobs:
-  grpc-and-badger:
+  badger:
     runs-on: ubuntu-latest
     steps:
     - name: Harden Runner
@@ -33,17 +33,14 @@ jobs:
     - name: Run Badger storage integration tests
       run: make badger-storage-integration-test
 
-    - name: Run gRPC storage integration tests
-      run: make grpc-storage-integration-test
-
     - name: Setup CODECOV_TOKEN
       uses: ./.github/actions/setup-codecov
 
     - name: Upload coverage to codecov
       uses: codecov/codecov-action@54bcd8715eee62d40e33596ef5e8f0f48dbbccab # v4.1.0
       with:
-        files: cover.out,cover-badger.out
+        files: cover.out
         verbose: true
-        flags: grpc-badger
+        flags: badger
         fail_ci_if_error: true
         token: ${{ env.CODECOV_TOKEN }}

.github/workflows/ci-grpc.yml

@@ -0,0 +1,57 @@
+name: CIT gRPC
+
+on:
+  push:
+    branches: [main]
+
+  pull_request:
+    branches: [main]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ (github.event.pull_request && github.event.pull_request.number) || github.ref || github.run_id }}
+  cancel-in-progress: true
+
+# See https://github.com/ossf/scorecard/blob/main/docs/checks.md#token-permissions
+permissions:  # added using https://github.com/step-security/secure-workflows
+  contents: read
+
+jobs:
+  grpc:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        version: [v1, v2]
+    steps:
+    - name: Harden Runner
+      uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
+      with:
+        egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
+
+    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+    - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+      with:
+        go-version: 1.22.x
+
+    - name: Run gRPC storage integration tests
+      run: |
+        case ${{ matrix.version }} in
+          v1)
+            make grpc-storage-integration-test
+            ;;
+          v2)
+            bash scripts/grpc-integration-test.sh latest
+            ;;
+        esac
+
+    - name: Setup CODECOV_TOKEN
+      uses: ./.github/actions/setup-codecov
+
+    - name: Upload coverage to codecov
+      uses: codecov/codecov-action@54bcd8715eee62d40e33596ef5e8f0f48dbbccab # v4.1.0
+      with:
+        files: cover.out
+        verbose: true
+        flags: grpc
+        fail_ci_if_error: true
+        token: ${{ env.CODECOV_TOKEN }}

.gitignore

@@ -27,6 +27,7 @@ cmd/collector/collector
 cmd/collector/collector-*
 cmd/ingester/ingester
 cmd/ingester/ingester-*
+cmd/jaeger/internal/integration/results
 cmd/remote-storage/remote-storage
 cmd/remote-storage/remote-storage-*
 cmd/es-index-cleaner/es-index-cleaner-*

Makefile

@@ -4,6 +4,7 @@
 SHELL := /bin/bash
 JAEGER_IMPORT_PATH = github.com/jaegertracing/jaeger
 STORAGE_PKGS = ./plugin/storage/integration/...
+JAEGER_STORAGE_PKGS = ./cmd/jaeger/internal/integration
 
 # These DOCKER_xxx vars are used when building Docker images.
 DOCKER_NAMESPACE?=jaegertracing
@@ -118,9 +119,21 @@ storage-integration-test:
 	go clean -testcache
 	bash -c "set -e; set -o pipefail; $(GOTEST) -coverpkg=./... -coverprofile cover.out $(STORAGE_PKGS) $(COLORIZE)"
 
+# A general integration tests for jaeger-v2 storage backends,
+# these tests placed at `./cmd/jaeger/internal/integration/*_test.go`.
+# The integration tests are filtered by STORAGE env,
+# currently the available STORAGE variable is:
+#  - grpc
+.PHONY: jaeger-storage-integration-test
+jaeger-storage-integration-test:
+	# Expire tests results for jaeger storage integration tests since the environment might change
+	# even though the code remains the same.
+	go clean -testcache
+	bash -c "set -e; set -o pipefail; $(GOTEST) -coverpkg=./... -coverprofile cover.out $(JAEGER_STORAGE_PKGS) $(COLORIZE)"
+
 .PHONY: badger-storage-integration-test
 badger-storage-integration-test:
-	bash -c "set -e; set -o pipefail; $(GOTEST) -tags=badger_storage_integration -coverpkg=./... -coverprofile cover-badger.out $(STORAGE_PKGS) $(COLORIZE)"
+	bash -c "set -e; set -o pipefail; $(GOTEST) -tags=badger_storage_integration -coverpkg=./... -coverprofile cover.out $(STORAGE_PKGS) $(COLORIZE)"
 
 .PHONY: grpc-storage-integration-test
 grpc-storage-integration-test:

cmd/jaeger/internal/command.go

@@ -29,7 +29,7 @@ func Command() *cobra.Command {
 
 	settings := otelcol.CollectorSettings{
 		BuildInfo: info,
-		Factories: components,
+		Factories: Components,
 	}
 
 	cmd := otelcol.NewCommand(settings)

cmd/jaeger/internal/components.go

@@ -116,6 +116,6 @@ func (b builders) build() (otelcol.Factories, error) {
 	return factories, nil
 }
 
-func components() (otelcol.Factories, error) {
+func Components() (otelcol.Factories, error) {
 	return defaultBuilders().build()
 }

cmd/jaeger/internal/components_test.go

@@ -29,7 +29,7 @@ import (
 )
 
 func TestComponents(t *testing.T) {
-	factories, err := components()
+	factories, err := Components()
 
 	require.NoError(t, err)
 

cmd/jaeger/internal/extension/jaegerstorage/extension_test.go

@@ -21,6 +21,7 @@ import (
 	esCfg "github.com/jaegertracing/jaeger/pkg/es/config"
 	memoryCfg "github.com/jaegertracing/jaeger/pkg/memory/config"
 	"github.com/jaegertracing/jaeger/pkg/metrics"
+	"github.com/jaegertracing/jaeger/pkg/testutils"
 	badgerCfg "github.com/jaegertracing/jaeger/plugin/storage/badger"
 	"github.com/jaegertracing/jaeger/storage"
 	"github.com/jaegertracing/jaeger/storage/dependencystore"
@@ -181,17 +182,19 @@ func TestESStorageExtension(t *testing.T) {
 }
 
 func TestESStorageExtensionError(t *testing.T) {
+	defer testutils.VerifyGoLeaksOnce(t)
+
 	ext := makeStorageExtenion(t, &Config{
 		Elasticsearch: map[string]esCfg.Configuration{
 			"foo": {
-				Servers:  []string{"http://badurl"},
+				Servers:  []string{"http://127.0.0.1:65535"},
 				LogLevel: "error",
 			},
 		},
 	})
 	err := ext.Start(context.Background(), componenttest.NewNopHost())
 	require.ErrorContains(t, err, "failed to initialize elasticsearch storage")
-	require.ErrorContains(t, err, "badurl")
+	require.ErrorContains(t, err, "http://127.0.0.1:65535")
 }
 
 func noopTelemetrySettings() component.TelemetrySettings {

cmd/jaeger/internal/integration/README.md

@@ -0,0 +1,27 @@
+# Integration
+
+Jaeger v2 integration tests are built on top of [OTEL Testbed module](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/testbed). OTEL Testbed provide comprehensive tools for conducting end-to-end tests for the OTEL Collector, such as reproducible short-term benchmarks, correctness tests, long-running stability tests and maximum load stress tests. However, we only utilize the correctness tests from testbed, it generates and sends every combinatorial trace attributes and matches every single of them with the received traces from another end. To learn more about OTEL Testbed, please refer to the their [README](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/testbed/README.md).
+
+## Architecture
+
+Here's the architecture to test the OpenTelemetry Collector pipeline from end-to-end with the designated storage backends.
+![integration diagram](integration-diagram.png)
+
+Testbed components:
+| Component | Description |
+|-----------|-------------|
+| **LoadGenerator** | Encapsulates DataProvider and DataSender in order to generate and send data. |
+| Golden DataProvider | Generates traces from the "Golden" dataset generated using pairwise combinatorial testing techniques. Testbed example uses [PICT](https://github.com/microsoft/pict/) to generate the test data, e.g. [testdata](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/internal/coreinternal/goldendataset/testdata). |
+| OTLP Trace DataSender | With the generated traces from DataProvider, the DataSender sends traces to OTLP receiver in the collector instance. |
+| **Mockbackend** | Encapsulates DataReceiver and provides consume functionality. |
+| DataReceiver | A custom DataReceiver that will host a Jaeger storage extension to retrieve traces from the database by pulling them using our artificial Jaeger storage receiver. |
+| Consumer | Consumer does not actually a thing in MockBackend but only to make the diagram intuitive, the traces received from our artificial receiver will be stored inside MockBackend. |
+| **Correctness Test Validator** | Checks if the traces received from MockBackend are all matches with the generated traces from DataProvider. |
+
+## gRPC Integration Test
+
+To conduct the tests, run the following command:
+
+```
+scripts/grpc-integration-test.sh <remote_storage_image_version>
+```

cmd/jaeger/internal/integration/datareceivers/.nocover

@@ -0,0 +1 @@
+A custom testbed data receiver for integration testing purpose

cmd/jaeger/internal/integration/datareceivers/jaegerstorage.go

@@ -0,0 +1,87 @@
+// Copyright (c) 2024 The Jaeger Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package datareceivers
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/extension/storage/storagetest"
+	"github.com/open-telemetry/opentelemetry-collector-contrib/testbed/testbed"
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/component/componenttest"
+	"go.opentelemetry.io/collector/consumer"
+	"go.opentelemetry.io/collector/extension"
+	"go.opentelemetry.io/collector/receiver"
+	"go.opentelemetry.io/collector/receiver/receivertest"
+
+	"github.com/jaegertracing/jaeger/cmd/jaeger/internal/extension/jaegerstorage"
+	"github.com/jaegertracing/jaeger/cmd/jaeger/internal/integration/receivers/storagereceiver"
+)
+
+type jaegerStorageDataReceiver struct {
+	TraceStorage  string
+	StorageConfig *jaegerstorage.Config
+	host          *storagetest.StorageHost
+	receiver      receiver.Traces
+}
+
+func NewJaegerStorageDataReceiver(traceStorage string, storageConfig *jaegerstorage.Config) testbed.DataReceiver {
+	return &jaegerStorageDataReceiver{
+		TraceStorage:  traceStorage,
+		StorageConfig: storageConfig,
+	}
+}
+
+func (dr *jaegerStorageDataReceiver) Start(tc consumer.Traces, _ consumer.Metrics, _ consumer.Logs) error {
+	ctx := context.Background()
+
+	extFactory := jaegerstorage.NewFactory()
+	ext, err := extFactory.CreateExtension(ctx, extension.CreateSettings{
+		TelemetrySettings: componenttest.NewNopTelemetrySettings(),
+		BuildInfo:         component.NewDefaultBuildInfo(),
+	}, dr.StorageConfig)
+	if err != nil {
+		return err
+	}
+
+	rcvSet := receivertest.NewNopCreateSettings()
+	rcvFactory := storagereceiver.NewFactory()
+	rcvCfg := rcvFactory.CreateDefaultConfig().(*storagereceiver.Config)
+	rcvCfg.TraceStorage = dr.TraceStorage
+	rcv, err := rcvFactory.CreateTracesReceiver(ctx, rcvSet, rcvCfg, tc)
+	if err != nil {
+		return err
+	}
+	dr.receiver = rcv
+
+	dr.host = storagetest.NewStorageHost()
+	dr.host.WithExtension(jaegerstorage.ID, ext)
+
+	err = dr.host.GetExtensions()[jaegerstorage.ID].Start(ctx, dr.host)
+	if err != nil {
+		return err
+	}
+	return dr.receiver.Start(ctx, dr.host)
+}
+
+func (dr *jaegerStorageDataReceiver) Stop() error {
+	ctx := context.Background()
+	err := dr.receiver.Shutdown(ctx)
+	if err != nil {
+		return err
+	}
+	return dr.host.GetExtensions()[jaegerstorage.ID].Shutdown(ctx)
+}
+
+func (dr *jaegerStorageDataReceiver) GenConfigYAMLStr() string {
+	return fmt.Sprintf(`
+  jaeger_storage_receiver:
+    trace_storage: %s
+`, dr.TraceStorage)
+}
+
+func (dr *jaegerStorageDataReceiver) ProtocolName() string {
+	return "jaeger_storage_receiver"
+}