Capture HTTP2 headers in netty gRPC client/server

instrumentation/grpc-1.5/build.gradle.kts

@@ -16,6 +16,7 @@ muzzle {
         versions = "[1.5.0, 1.33.0)" // see https://github.com/open-telemetry/opentelemetry-java-instrumentation/issues/1453
         // for body capture via com.google.protobuf.util.JsonFormat
         extraDependency("io.grpc:grpc-protobuf:1.5.0")
+        extraDependency("io.grpc:grpc-netty:1.5.0")
     }
 }
 
@@ -57,11 +58,12 @@ val versions: Map<String, String> by extra
 
 dependencies {
     api("io.opentelemetry.javaagent.instrumentation:opentelemetry-javaagent-grpc-1.5:${versions["opentelemetry_java_agent"]}")
-    api("io.opentelemetry.instrumentation:opentelemetry-grpc-1.5:0.11.0")
+    api("io.opentelemetry.instrumentation:opentelemetry-grpc-1.5:${versions["opentelemetry_java_agent"]}")
 
     compileOnly("io.grpc:grpc-core:1.5.0")
     compileOnly("io.grpc:grpc-protobuf:1.5.0")
     compileOnly("io.grpc:grpc-stub:1.5.0")
+    compileOnly("io.grpc:grpc-netty:1.5.0")
 
     implementation("javax.annotation:javax.annotation-api:1.3.2")
 

instrumentation/grpc-1.5/src/main/java/io/opentelemetry/instrumentation/hypertrace/grpc/v1_5/GrpcSemanticAttributes.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright The Hypertrace Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.opentelemetry.instrumentation.hypertrace.grpc.v1_5;
+
+import io.grpc.Metadata;
+
+public class GrpcSemanticAttributes {
+  private GrpcSemanticAttributes() {}
+
+  public static final String SCHEME = ":scheme";
+  public static final String PATH = ":path";
+  public static final String AUTHORITY = ":authority";
+  public static final String METHOD = ":method";
+
+  /**
+   * These metadata headers are added in Http2Headers instrumentation. We use different names than
+   * original HTTP2 header names to avoid any collisions with app code.
+   *
+   * <p>We cannot use prefix because e.g. ht.:path is not a valid key.
+   */
+  private static final String SUFFIX = ".ht";
+
+  public static final Metadata.Key<String> SCHEME_METADATA_KEY =
+      Metadata.Key.of(SCHEME + SUFFIX, Metadata.ASCII_STRING_MARSHALLER);
+  public static final Metadata.Key<String> PATH_METADATA_KEY =
+      Metadata.Key.of(PATH + SUFFIX, Metadata.ASCII_STRING_MARSHALLER);
+  public static final Metadata.Key<String> AUTHORITY_METADATA_KEY =
+      Metadata.Key.of(AUTHORITY + SUFFIX, Metadata.ASCII_STRING_MARSHALLER);
+  public static final Metadata.Key<String> METHOD_METADATA_KEY =
+      Metadata.Key.of(METHOD + SUFFIX, Metadata.ASCII_STRING_MARSHALLER);
+
+  public static String removeHypertracePrefix(String key) {
+    if (key.endsWith(SUFFIX)) {
+      return key.replace(SUFFIX, "");
+    }
+    return key;
+  }
+}

instrumentation/grpc-1.5/src/main/java/io/opentelemetry/instrumentation/hypertrace/grpc/v1_5/GrpcSpanDecorator.java

@@ -58,6 +58,7 @@ public static void addMetadataAttributes(
       Key<String> stringKey = Key.of(key, Metadata.ASCII_STRING_MARSHALLER);
       Iterable<String> stringValues = metadata.getAll(stringKey);
       for (String stringValue : stringValues) {
+        key = GrpcSemanticAttributes.removeHypertracePrefix(key);
         span.setAttribute(keySupplier.apply(key), stringValue);
       }
     }
@@ -79,6 +80,7 @@ public static Map<String, String> metadataToMap(Metadata metadata) {
       Key<String> stringKey = Key.of(key, Metadata.ASCII_STRING_MARSHALLER);
       Iterable<String> stringValues = metadata.getAll(stringKey);
       for (String stringValue : stringValues) {
+        key = GrpcSemanticAttributes.removeHypertracePrefix(key);
         mapHeaders.put(key, stringValue);
       }
     }

instrumentation/grpc-1.5/src/main/java/io/opentelemetry/instrumentation/hypertrace/grpc/v1_5/NettyHttp2HeadersInstrumentationModule.java

@@ -0,0 +1,148 @@
+/*
+ * Copyright The Hypertrace Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.opentelemetry.instrumentation.hypertrace.grpc.v1_5;
+
+import static net.bytebuddy.matcher.ElementMatchers.failSafe;
+import static net.bytebuddy.matcher.ElementMatchers.isMethod;
+import static net.bytebuddy.matcher.ElementMatchers.named;
+import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
+
+import com.google.auto.service.AutoService;
+import io.grpc.Metadata;
+import io.netty.handler.codec.http2.Http2Headers;
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.javaagent.instrumentation.api.Java8BytecodeBridge;
+import io.opentelemetry.javaagent.tooling.InstrumentationModule;
+import io.opentelemetry.javaagent.tooling.TypeInstrumentation;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import net.bytebuddy.asm.Advice;
+import net.bytebuddy.description.method.MethodDescription;
+import net.bytebuddy.description.type.TypeDescription;
+import net.bytebuddy.matcher.ElementMatcher;
+import org.hypertrace.agent.core.HypertraceSemanticAttributes;
+
+@AutoService(InstrumentationModule.class)
+public class NettyHttp2HeadersInstrumentationModule extends InstrumentationModule {
+
+  private static final List<String> INSTRUMENTATION_NAMES = new ArrayList<>();
+
+  static {
+    INSTRUMENTATION_NAMES.add(GrpcInstrumentationName.PRIMARY);
+    INSTRUMENTATION_NAMES.addAll(Arrays.asList(GrpcInstrumentationName.OTHER));
+    INSTRUMENTATION_NAMES.add("grpc-netty-ht");
+  }
+
+  public NettyHttp2HeadersInstrumentationModule() {
+    super(INSTRUMENTATION_NAMES);
+  }
+
+  @Override
+  public List<TypeInstrumentation> typeInstrumentations() {
+    return Arrays.asList(new NettyUtilsInstrumentation());
+  }
+
+  /**
+   * The server side HTTP2 headers are added in tracing gRPC interceptor. The headers are added to
+   * metadata in {@link GrpcUtils_convertHeaders_Advice}.
+   *
+   * <p>The client side HTTP2 headers are added directly to span in {@link
+   * Utils_convertClientHeaders_Advice}. TODO However it does not work for the first request
+   * https://github.com/hypertrace/javaagent/issues/109#issuecomment-740918018.
+   */
+  class NettyUtilsInstrumentation implements TypeInstrumentation {
+    @Override
+    public ElementMatcher<? super TypeDescription> typeMatcher() {
+      return failSafe(named("io.grpc.netty.Utils"));
+    }
+
+    @Override
+    public Map<? extends ElementMatcher<? super MethodDescription>, String> transformers() {
+      Map<ElementMatcher<? super MethodDescription>, String> transformers = new HashMap<>();
+      transformers.put(
+          isMethod().and(named("convertClientHeaders")).and(takesArguments(6)),
+          Utils_convertClientHeaders_Advice.class.getName());
+      transformers.put(
+          isMethod().and(named("convertHeaders")).and(takesArguments(1)),
+          GrpcUtils_convertHeaders_Advice.class.getName());
+      return transformers;
+    }
+  }
+
+  static class Utils_convertClientHeaders_Advice {
+    @Advice.OnMethodExit(suppress = Throwable.class)
+    public static void exit(
+        @Advice.Argument(1) Object scheme,
+        @Advice.Argument(2) Object defaultPath,
+        @Advice.Argument(3) Object authority,
+        @Advice.Argument(4) Object method) {
+
+      Span currentSpan = Java8BytecodeBridge.currentSpan();
+      if (scheme != null) {
+        currentSpan.setAttribute(
+            HypertraceSemanticAttributes.rpcRequestMetadata(GrpcSemanticAttributes.SCHEME),
+            scheme.toString());
+      }
+      if (defaultPath != null) {
+        currentSpan.setAttribute(
+            HypertraceSemanticAttributes.rpcRequestMetadata(GrpcSemanticAttributes.PATH),
+            defaultPath.toString());
+      }
+      if (authority != null) {
+        currentSpan.setAttribute(
+            HypertraceSemanticAttributes.rpcRequestMetadata(GrpcSemanticAttributes.AUTHORITY),
+            authority.toString());
+      }
+      if (method != null) {
+        currentSpan.setAttribute(
+            HypertraceSemanticAttributes.rpcRequestMetadata(GrpcSemanticAttributes.METHOD),
+            method.toString());
+      }
+    }
+  }
+
+  /**
+   * There are multiple implementations of {@link Http2Headers}. Only some of them support getting
+   * authority, path etc. For instance {@code GrpcHttp2ResponseHeaders} throws unsupported exception
+   * when accessing authority etc. This header is used client response.
+   *
+   * @see {@link io.grpc.netty.GrpcHttp2HeadersUtils}
+   */
+  static class GrpcUtils_convertHeaders_Advice {
+    @Advice.OnMethodExit(suppress = Throwable.class)
+    public static void exit(
+        @Advice.Argument(0) Http2Headers http2Headers, @Advice.Return Metadata metadata) {
+
+      if (http2Headers.authority() != null) {
+        metadata.put(
+            GrpcSemanticAttributes.AUTHORITY_METADATA_KEY, http2Headers.authority().toString());
+      }
+      if (http2Headers.path() != null) {
+        metadata.put(GrpcSemanticAttributes.PATH_METADATA_KEY, http2Headers.path().toString());
+      }
+      if (http2Headers.method() != null) {
+        metadata.put(GrpcSemanticAttributes.METHOD_METADATA_KEY, http2Headers.method().toString());
+      }
+      if (http2Headers.scheme() != null) {
+        metadata.put(GrpcSemanticAttributes.SCHEME_METADATA_KEY, http2Headers.scheme().toString());
+      }
+    }
+  }
+}

instrumentation/grpc-1.5/src/test/java/io/opentelemetry/instrumentation/hypertrace/grpc/v1_5/GrpcInstrumentationTest.java

@@ -48,8 +48,16 @@
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.MethodOrderer.OrderAnnotation;
+import org.junit.jupiter.api.Order;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
 
+/**
+ * TODO the HTTP2 headers for client does not work for the first request - therefore the explicit
+ * ordering https://github.com/hypertrace/javaagent/issues/109#issuecomment-740918018
+ */
+@TestMethodOrder(OrderAnnotation.class)
 public class GrpcInstrumentationTest extends AbstractInstrumenterTest {
 
   private static final Helloworld.Request REQUEST =
@@ -113,6 +121,7 @@ public void afterEach() {
   }
 
   @Test
+  @Order(2)
   public void blockingStub() throws IOException, TimeoutException, InterruptedException {
     Metadata headers = new Metadata();
     headers.put(CLIENT_STRING_METADATA_KEY, "clientheader");
@@ -135,9 +144,13 @@ public void blockingStub() throws IOException, TimeoutException, InterruptedExce
     assertBodiesAndHeaders(clientSpan, requestJson, responseJson);
     SpanData serverSpan = spans.get(1);
     assertBodiesAndHeaders(serverSpan, requestJson, responseJson);
+
+    assertHttp2HeadersForSayHelloMethod(serverSpan);
+    assertHttp2HeadersForSayHelloMethod(clientSpan);
   }
 
   @Test
+  @Order(1)
   public void serverRequestBlocking() throws TimeoutException, InterruptedException {
     Metadata blockHeaders = new Metadata();
     blockHeaders.put(Metadata.Key.of("mockblock", Metadata.ASCII_STRING_MARSHALLER), "true");
@@ -167,9 +180,11 @@ public void serverRequestBlocking() throws TimeoutException, InterruptedExceptio
         serverSpan
             .getAttributes()
             .get(HypertraceSemanticAttributes.rpcRequestMetadata("mockblock")));
+    assertHttp2HeadersForSayHelloMethod(serverSpan);
   }
 
   @Test
+  @Order(3)
   public void disabledInstrumentation_dynamicConfig()
       throws TimeoutException, InterruptedException {
     URL configUrl = getClass().getClassLoader().getResource("ht-config-all-disabled.yaml");
@@ -215,4 +230,24 @@ private void assertBodiesAndHeaders(SpanData span, String requestJson, String re
                 HypertraceSemanticAttributes.rpcResponseMetadata(
                     SERVER_STRING_METADATA_KEY.name())));
   }
+
+  private void assertHttp2HeadersForSayHelloMethod(SpanData span) {
+    Assertions.assertEquals(
+        "http",
+        span.getAttributes()
+            .get(HypertraceSemanticAttributes.rpcRequestMetadata(GrpcSemanticAttributes.SCHEME)));
+    Assertions.assertEquals(
+        "POST",
+        span.getAttributes()
+            .get(HypertraceSemanticAttributes.rpcRequestMetadata(GrpcSemanticAttributes.METHOD)));
+    Assertions.assertEquals(
+        String.format("localhost:%d", SERVER.getPort()),
+        span.getAttributes()
+            .get(
+                HypertraceSemanticAttributes.rpcRequestMetadata(GrpcSemanticAttributes.AUTHORITY)));
+    Assertions.assertEquals(
+        "/org.hypertrace.example.Greeter/SayHello",
+        span.getAttributes()
+            .get(HypertraceSemanticAttributes.rpcRequestMetadata(GrpcSemanticAttributes.PATH)));
+  }
 }