test: integrate otel into our StorageITRunner

google-cloud-storage/pom.xml

@@ -164,6 +164,12 @@
       <groupId>com.google.cloud.opentelemetry</groupId>
       <artifactId>exporter-metrics</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.google.cloud.opentelemetry</groupId>
+      <artifactId>exporter-trace</artifactId>
+      <version>0.32.0</version>
+      <scope>test</scope>
+    </dependency>
 
     <dependency>
       <groupId>io.opentelemetry.contrib</groupId>

google-cloud-storage/src/main/java/com/google/cloud/storage/OtelStorageDecorator.java

@@ -31,6 +31,7 @@
 import com.google.cloud.storage.PostPolicyV4.PostConditionsV4;
 import com.google.cloud.storage.PostPolicyV4.PostFieldsV4;
 import com.google.cloud.storage.TransportCompatibility.Transport;
+import com.google.common.annotations.VisibleForTesting;
 import io.opentelemetry.api.OpenTelemetry;
 import io.opentelemetry.api.common.Attributes;
 import io.opentelemetry.api.trace.Span;
@@ -58,7 +59,7 @@ final class OtelStorageDecorator implements Storage {
   /** Becomes the {@code otel.scope.name} attribute in a span */
   private static final String OTEL_SCOPE_NAME = "cloud.google.com/java/storage";
 
-  private final Storage delegate;
+  @VisibleForTesting final Storage delegate;
   private final OpenTelemetry otel;
   private final Attributes baseAttributes;
   private final Tracer tracer;
@@ -1501,9 +1502,10 @@ public SpanBuilder spanBuilder(String spanName) {
     }
   }
 
-  private static final class OtelDecoratedReadChannel implements ReadChannel {
+  @VisibleForTesting
+  static final class OtelDecoratedReadChannel implements ReadChannel {
 
-    private final ReadChannel reader;
+    @VisibleForTesting final ReadChannel reader;
     private final Span span;
 
     private OtelDecoratedReadChannel(ReadChannel reader, Span span) {
@@ -1620,8 +1622,9 @@ public void close() throws IOException {
     }
   }
 
-  private static final class OtelDecoratedWriteChannel implements WriteChannel {
-    private final WriteChannel delegate;
+  @VisibleForTesting
+  static final class OtelDecoratedWriteChannel implements WriteChannel {
+    @VisibleForTesting final WriteChannel delegate;
     private final Span openSpan;
 
     private OtelDecoratedWriteChannel(WriteChannel delegate, Span openSpan) {

google-cloud-storage/src/test/java/com/google/cloud/storage/PackagePrivateMethodWorkarounds.java

@@ -20,6 +20,8 @@
 import com.google.api.core.ApiFutures;
 import com.google.cloud.ReadChannel;
 import com.google.cloud.WriteChannel;
+import com.google.cloud.storage.OtelStorageDecorator.OtelDecoratedReadChannel;
+import com.google.cloud.storage.OtelStorageDecorator.OtelDecoratedWriteChannel;
 import com.google.common.collect.ImmutableList;
 import com.google.storage.v2.StorageClient;
 import java.util.Optional;
@@ -49,6 +51,10 @@ public static Blob blobCopyWithStorage(Blob b, Storage s) {
 
   public static Function<WriteChannel, Optional<BlobInfo>> maybeGetBlobInfoFunction() {
     return (w) -> {
+      if (w instanceof OtelDecoratedWriteChannel) {
+        OtelDecoratedWriteChannel odwc = (OtelDecoratedWriteChannel) w;
+        w = odwc.delegate;
+      }
       if (w instanceof BlobWriteChannelV2) {
         BlobWriteChannelV2 blobWriteChannel = (BlobWriteChannelV2) w;
         return Optional.ofNullable(blobWriteChannel.getResolvedObject())
@@ -71,13 +77,16 @@ public static Function<WriteChannel, Optional<BlobInfo>> maybeGetBlobInfoFunctio
   }
 
   public static ApiFuture<BlobInfo> getBlobInfoFromReadChannelFunction(ReadChannel c) {
+    if (c instanceof OtelDecoratedReadChannel) {
+      OtelDecoratedReadChannel odrc = (OtelDecoratedReadChannel) c;
+      c = odrc.reader;
+    }
     if (c instanceof StorageReadChannel) {
       StorageReadChannel src = (StorageReadChannel) c;
       return src.getObject();
-    } else {
-      return ApiFutures.immediateFailedFuture(
-          new IllegalStateException("Unsupported ReadChannel Type " + c.getClass().getName()));
     }
+    return ApiFutures.immediateFailedFuture(
+        new IllegalStateException("Unsupported ReadChannel Type " + c.getClass().getName()));
   }
 
   @Nullable
@@ -87,6 +96,10 @@ public static StorageClient maybeGetStorageClient(Storage s) {
     }
     // handle instances of AbstractStorageProxy
     Storage service = s.getOptions().getService();
+    if (service instanceof OtelStorageDecorator) {
+      OtelStorageDecorator osd = (OtelStorageDecorator) service;
+      service = osd.delegate;
+    }
     if (service instanceof GrpcStorageImpl) {
       return ((GrpcStorageImpl) service).storageClient;
     }

google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BackendResources.java

@@ -83,7 +83,7 @@ public String toString() {
   }
 
   @SuppressWarnings("SwitchStatementWithTooFewBranches")
-  static BackendResources of(Backend backend) {
+  static BackendResources of(Backend backend, TestRunScopedInstance<OtelSdkShim> otelSdk) {
     ProtectedBucketNames protectedBucketNames = new ProtectedBucketNames();
     TestRunScopedInstance<StorageInstance> storageJson =
         TestRunScopedInstance.of(
@@ -99,7 +99,8 @@ static BackendResources of(Backend backend) {
                           .setProjectId("test-project-id");
                   break;
                 default: // PROD, java8 doesn't have exhaustive checking for enum switch
-                  optionsBuilder = StorageOptions.http();
+                  // Register the exporters with OpenTelemetry
+                  optionsBuilder = StorageOptions.http().setOpenTelemetry(otelSdk.get().get());
                   break;
               }
               HttpStorageOptions built = optionsBuilder.build();
@@ -121,7 +122,8 @@ static BackendResources of(Backend backend) {
                           .setProjectId("test-project-id");
                   break;
                 default: // PROD, java8 doesn't have exhaustive checking for enum switch
-                  optionsBuilder = StorageOptions.grpc();
+                  // Register the exporters with OpenTelemetry
+                  optionsBuilder = StorageOptions.grpc().setOpenTelemetry(otelSdk.get().get());
                   break;
               }
               GrpcStorageOptions built =

google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/ObjectsFixture.java

@@ -102,7 +102,8 @@ public void start() {
     BlobInfo info2 = BlobInfo.newBuilder(blobId2).setMetadata(ImmutableMap.of("pow", "2")).build();
     BlobInfo info3 = BlobInfo.newBuilder(blobId3).setMetadata(ImmutableMap.of("pow", "3")).build();
     BlobInfo info4 = BlobInfo.newBuilder(blobId4).setMetadata(ImmutableMap.of("pow", "4")).build();
-    s.create(info1, "A".getBytes(StandardCharsets.UTF_8), blobTargetOptions);
+    this.info1 =
+        s.create(info1, "A".getBytes(StandardCharsets.UTF_8), blobTargetOptions).asBlobInfo();
 
     ComposeRequest c2 =
         ComposeRequest.newBuilder()
@@ -122,14 +123,9 @@ public void start() {
             .setTarget(info4)
             .setTargetOptions(blobTargetOptions)
             .build();
-    s.compose(c2);
-    s.compose(c3);
-    s.compose(c4);
-
-    this.info1 = s.get(blobId1, blobGetOptions).asBlobInfo();
-    this.info2 = s.get(blobId2, blobGetOptions).asBlobInfo();
-    this.info3 = s.get(blobId3, blobGetOptions).asBlobInfo();
-    this.info4 = s.get(blobId4, blobGetOptions).asBlobInfo();
+    this.info2 = s.compose(c2).asBlobInfo();
+    this.info3 = s.compose(c3).asBlobInfo();
+    this.info4 = s.compose(c4).asBlobInfo();
 
     byte[] bytes = DataGenerator.base64Characters().genBytes(512 * 1024);
     Blob obj512KiB =

google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/OtelSdkShim.java

@@ -0,0 +1,102 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage.it.runner.registry;
+
+import com.google.cloud.opentelemetry.metric.GoogleCloudMetricExporter;
+import com.google.cloud.opentelemetry.metric.MetricConfiguration;
+import com.google.cloud.opentelemetry.trace.TraceConfiguration;
+import com.google.cloud.opentelemetry.trace.TraceExporter;
+import io.opentelemetry.api.OpenTelemetry;
+import io.opentelemetry.sdk.OpenTelemetrySdk;
+import io.opentelemetry.sdk.metrics.SdkMeterProvider;
+import io.opentelemetry.sdk.metrics.export.MetricExporter;
+import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.export.BatchSpanProcessor;
+import io.opentelemetry.sdk.trace.export.SpanExporter;
+import io.opentelemetry.sdk.trace.samplers.Sampler;
+import java.time.Duration;
+import java.util.Arrays;
+
+public final class OtelSdkShim implements ManagedLifecycle {
+  private static final boolean STORAGE_IT_OTEL_ENABLE =
+      Arrays.asList(
+              System.getProperty("STORAGE_IT_OTEL_ENABLE"), System.getenv("STORAGE_IT_OTEL_ENABLE"))
+          .contains("true");
+  private final String projectId;
+
+  private OpenTelemetrySdk otelSdk;
+
+  OtelSdkShim(String projectId) {
+    this.projectId = projectId;
+  }
+
+  @Override
+  public OpenTelemetry get() {
+    if (otelSdk == null) {
+      return OpenTelemetry.noop();
+    }
+    return otelSdk;
+  }
+
+  @Override
+  public void start() {
+    if (!STORAGE_IT_OTEL_ENABLE) {
+      otelSdk = null;
+      return;
+    }
+    MetricConfiguration metricConfiguration =
+        MetricConfiguration.builder()
+            .setProjectId(projectId)
+            .setDeadline(Duration.ofSeconds(30))
+            .build();
+    TraceConfiguration traceConfiguration =
+        TraceConfiguration.builder()
+            .setProjectId(projectId)
+            .setDeadline(Duration.ofSeconds(30))
+            .build();
+    MetricExporter metricExporter =
+        GoogleCloudMetricExporter.createWithConfiguration(metricConfiguration);
+    SpanExporter spanExporter = TraceExporter.createWithConfiguration(traceConfiguration);
+
+    SdkMeterProvider meterProvider =
+        SdkMeterProvider.builder()
+            .registerMetricReader(
+                PeriodicMetricReader.builder(metricExporter)
+                    .setInterval(Duration.ofSeconds(60))
+                    .build())
+            .build();
+    SdkTracerProvider tracerProvider =
+        SdkTracerProvider.builder()
+            .setSampler(Sampler.traceIdRatioBased(1.0))
+            .addSpanProcessor(
+                BatchSpanProcessor.builder(spanExporter).setMeterProvider(meterProvider).build())
+            .build();
+    otelSdk =
+        OpenTelemetrySdk.builder()
+            .setTracerProvider(tracerProvider)
+            .setMeterProvider(meterProvider)
+            .build();
+  }
+
+  @Override
+  public void stop() {
+    if (otelSdk != null) {
+      otelSdk.close();
+    }
+  }
+}