Instrumentation for Elasticsearch 8+

docs/supported-libraries.md

@@ -53,8 +53,9 @@ These are the supported libraries and frameworks:
 | [Eclipse Jetty HTTP Client](https://www.eclipse.org/jetty/javadoc/jetty-9/org/eclipse/jetty/client/HttpClient.html)                         | 9.2+ (not including 10+ yet)  | [opentelemetry-jetty-httpclient-9.2](../instrumentation/jetty-httpclient/jetty-httpclient-9.2/library)                                                                                                                                                                                                                                                                                  | [HTTP Client Spans], [HTTP Client Metrics]                                             |
 | [Eclipse Metro](https://projects.eclipse.org/projects/ee4j.metro)                                                                           | 2.2+ (not including 3.x yet)  | N/A                                                                                                                                                                                                                                                                                                                                                                                     | Provides `http.route` [2], Controller Spans [3]                                        |
 | [Eclipse Mojarra](https://projects.eclipse.org/projects/ee4j.mojarra)                                                                       | 1.2+ (not including 3.x yet)  | N/A                                                                                                                                                                                                                                                                                                                                                                                     | Provides `http.route` [2], Controller Spans [3]                                        |
-| [Elasticsearch API](https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/index.html)                                       | 5.0+                          | N/A                                                                                                                                                                                                                                                                                                                                                                                     | [Database Client Spans]                                                                |
+| [Elasticsearch API Client](https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/current/index.html)                         | 7.16+ and 8.0+                | N/A                                                                                                                                                                                                                                                                                                                                                                                     | [Elasticsearch Client Spans]                                                           |
 | [Elasticsearch REST Client](https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/index.html)                              | 5.0+                          | N/A                                                                                                                                                                                                                                                                                                                                                                                     | [Database Client Spans]                                                                |
+| [Elasticsearch Transport Client](https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/index.html)                          | 5.0+                          | N/A                                                                                                                                                                                                                                                                                                                                                                                     | [Database Client Spans]                                                                |
 | [Finatra](https://github.com/twitter/finatra)                                                                                               | 2.9+                          | N/A                                                                                                                                                                                                                                                                                                                                                                                     | Provides `http.route` [2], Controller Spans [3]                                        |
 | [Geode Client](https://geode.apache.org/)                                                                                                   | 1.4+                          | N/A                                                                                                                                                                                                                                                                                                                                                                                     | [Database Client Spans]                                                                |
 | [Google HTTP Client](https://github.com/googleapis/google-http-java-client)                                                                 | 1.19+                         | N/A                                                                                                                                                                                                                                                                                                                                                                                     | [HTTP Client Spans], [HTTP Client Metrics]                                             |
@@ -141,6 +142,7 @@ These are the supported libraries and frameworks:
 
 **[3]** Controller Spans are `INTERNAL` spans capturing the controller and/or view execution. See [Suppressing controller and/or view spans](https://opentelemetry.io/docs/instrumentation/java/automatic/agent-config/#suppressing-controller-andor-view-spans).
 
+[Elasticsearch Client Spans]: https://github.com/open-telemetry/semantic-conventions/blob/main/specification/database/elasticsearch.md
 [HTTP Server Spans]: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/http.md#http-server-semantic-conventions
 [HTTP Client Spans]: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/http.md#http-client
 [HTTP Server Metrics]: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/semantic_conventions/http-metrics.md#http-server

instrumentation/elasticsearch/README.md

@@ -1,5 +1,12 @@
 # Settings for the elasticsearch instrumentation
 
+## Settings for the [Elasticsearch Java API Client](https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/current/index.html) instrumentation
+| System property | Type | Default | Description                                                                                                                |
+|---|---|---|----------------------------------------------------------------------------------------------------------------------------|
+| `otel.instrumentation.elasticsearch.capture-search-query` | `Boolean | `false` | Enable the capture of search query bodies. Attention: Elasticsearch queries may contain personal or sensitive information. |
+
+
+## Settings for the [Elasticsearch Transport Client](https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/index.html) instrumentation
 | System property | Type | Default | Description |
 |---|---|---|---|
 | `otel.instrumentation.elasticsearch.experimental-span-attributes` | `Boolean | `false` | Enable the capture of experimental span attributes. |

instrumentation/elasticsearch/elasticsearch-api-client-7.16/javaagent-unit-tests/build.gradle.kts

@@ -0,0 +1,8 @@
+plugins {
+  id("otel.java-conventions")
+}
+
+dependencies {
+  testImplementation(project(":instrumentation:elasticsearch:elasticsearch-rest-common:javaagent"))
+  testImplementation(project(":instrumentation:elasticsearch:elasticsearch-api-client-7.16:javaagent"))
+}

instrumentation/elasticsearch/elasticsearch-api-client-7.16/javaagent-unit-tests/src/test/java/io/opentelemetry/javaagent/instrumentation/elasticsearch/rest/ElasticsearchEndpointMapTest.java

@@ -0,0 +1,123 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.instrumentation.elasticsearch.rest;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import io.opentelemetry.javaagent.instrumentation.elasticsearch.apiclient.ElasticsearchEndpointMap;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.junit.jupiter.api.Test;
+
+public class ElasticsearchEndpointMapTest {
+
+  private static final Set<String> SEARCH_ENDPOINTS =
+      new HashSet<>(
+          Arrays.asList(
+              "search",
+              "async_search.submit",
+              "msearch",
+              "eql.search",
+              "terms_enum",
+              "search_template",
+              "msearch_template",
+              "render_search_template"));
+
+  private static List<String> getPathParts(String route) {
+    List<String> pathParts = new ArrayList<>();
+    String routeFragment = route;
+    int paramStartIndex = routeFragment.indexOf('{');
+    while (paramStartIndex >= 0) {
+      int paramEndIndex = routeFragment.indexOf('}');
+      if (paramEndIndex < 0 || paramEndIndex <= paramStartIndex + 1) {
+        throw new IllegalStateException("Invalid route syntax!");
+      }
+      pathParts.add(routeFragment.substring(paramStartIndex + 1, paramEndIndex));
+
+      int nextIdx = paramEndIndex + 1;
+      if (nextIdx >= routeFragment.length()) {
+        break;
+      }
+
+      routeFragment = routeFragment.substring(nextIdx);
+      paramStartIndex = routeFragment.indexOf('{');
+    }
+    return pathParts;
+  }
+
+  @Test
+  public void testIsSearchEndpoint() {
+    for (ElasticsearchEndpointDefinition esEndpointDefinition :
+        ElasticsearchEndpointMap.getAllEndpoints()) {
+      String endpointId = esEndpointDefinition.getEndpointName();
+      assertEquals(SEARCH_ENDPOINTS.contains(endpointId), esEndpointDefinition.isSearchEndpoint());
+    }
+  }
+
+  @Test
+  public void testProcessPathParts() {
+    for (ElasticsearchEndpointDefinition esEndpointDefinition :
+        ElasticsearchEndpointMap.getAllEndpoints()) {
+      for (String route :
+          esEndpointDefinition.getRoutes().stream()
+              .map(ElasticsearchEndpointDefinition.Route::getName)
+              .collect(Collectors.toList())) {
+        List<String> pathParts = getPathParts(route);
+        String resolvedRoute = route.replace("{", "").replace("}", "");
+        Map<String, String> observedParams = new HashMap<>();
+        esEndpointDefinition.processPathParts(resolvedRoute, (k, v) -> observedParams.put(k, v));
+
+        Map<String, String> expectedMap = new HashMap<>();
+        pathParts.forEach(part -> expectedMap.put(part, part));
+
+        assertEquals(expectedMap, observedParams);
+      }
+    }
+  }
+
+  @Test
+  public void testSearchEndpoint() {
+    ElasticsearchEndpointDefinition esEndpoint = ElasticsearchEndpointMap.get("search");
+    Map<String, String> observedParams = new HashMap<>();
+    esEndpoint.processPathParts(
+        "/test-index-1,test-index-2/_search", (k, v) -> observedParams.put(k, v));
+
+    assertEquals("test-index-1,test-index-2", observedParams.get("index"));
+  }
+
+  @Test
+  public void testBuildRegexPattern() {
+    Pattern pattern =
+        ElasticsearchEndpointDefinition.EndpointPattern.buildRegexPattern(
+            "/_nodes/{node_id}/shutdown");
+    assertEquals("^/_nodes/(?<node0id>[^/]+)/shutdown$", pattern.pattern());
+
+    pattern =
+        ElasticsearchEndpointDefinition.EndpointPattern.buildRegexPattern(
+            "/_snapshot/{repository}/{snapshot}/_mount");
+    assertEquals("^/_snapshot/(?<repository>[^/]+)/(?<snapshot>[^/]+)/_mount$", pattern.pattern());
+
+    pattern =
+        ElasticsearchEndpointDefinition.EndpointPattern.buildRegexPattern(
+            "/_security/profile/_suggest");
+    assertEquals("^/_security/profile/_suggest$", pattern.pattern());
+
+    pattern =
+        ElasticsearchEndpointDefinition.EndpointPattern.buildRegexPattern(
+            "/_application/search_application/{name}");
+    assertEquals("^/_application/search_application/(?<name>[^/]+)$", pattern.pattern());
+
+    pattern = ElasticsearchEndpointDefinition.EndpointPattern.buildRegexPattern("/");
+    assertEquals("^/$", pattern.pattern());
+  }
+}

instrumentation/elasticsearch/elasticsearch-api-client-7.16/javaagent/build.gradle.kts

@@ -0,0 +1,31 @@
+plugins {
+  id("otel.javaagent-instrumentation")
+}
+
+muzzle {
+  pass {
+    group.set("co.elastic.clients")
+    module.set("elasticsearch-java")
+    versions.set("[7.16,)")
+    assertInverse.set(true)
+  }
+}
+
+dependencies {
+  library("co.elastic.clients:elasticsearch-java:7.16.0")
+
+  implementation(project(":instrumentation:elasticsearch:elasticsearch-rest-common:javaagent"))
+
+  testInstrumentation(project(":instrumentation:elasticsearch:elasticsearch-rest-7.0:javaagent"))
+  testInstrumentation(project(":instrumentation:apache-httpclient:apache-httpclient-4.0:javaagent"))
+  testInstrumentation(project(":instrumentation:apache-httpasyncclient-4.1:javaagent"))
+
+  testImplementation("com.fasterxml.jackson.core:jackson-databind:2.14.2")
+  testImplementation("org.testcontainers:elasticsearch")
+}
+
+tasks {
+  test {
+    usesService(gradle.sharedServices.registrations["testcontainersBuildService"].service)
+  }
+}

instrumentation/elasticsearch/elasticsearch-api-client-7.16/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/elasticsearch/apiclient/ApiClientInstrumentation.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.instrumentation.elasticsearch.apiclient;
+
+import static net.bytebuddy.matcher.ElementMatchers.isMethod;
+import static net.bytebuddy.matcher.ElementMatchers.named;
+import static net.bytebuddy.matcher.ElementMatchers.returns;
+import static net.bytebuddy.matcher.ElementMatchers.takesArgument;
+
+import co.elastic.clients.transport.Endpoint;
+import io.opentelemetry.instrumentation.api.util.VirtualField;
+import io.opentelemetry.javaagent.extension.instrumentation.TypeInstrumentation;
+import io.opentelemetry.javaagent.extension.instrumentation.TypeTransformer;
+import io.opentelemetry.javaagent.instrumentation.elasticsearch.rest.ElasticsearchEndpointDefinition;
+import net.bytebuddy.asm.Advice;
+import net.bytebuddy.description.type.TypeDescription;
+import net.bytebuddy.matcher.ElementMatcher;
+import org.elasticsearch.client.Request;
+
+public class ApiClientInstrumentation implements TypeInstrumentation {
+
+  @Override
+  public ElementMatcher<TypeDescription> typeMatcher() {
+    return named("co.elastic.clients.transport.rest_client.RestClientTransport");
+  }
+
+  @Override
+  public void transform(TypeTransformer transformer) {
+    transformer.applyAdviceToMethod(
+        isMethod()
+            .and(named("prepareLowLevelRequest"))
+            .and(takesArgument(1, named("co.elastic.clients.transport.Endpoint")))
+            .and(returns(named("org.elasticsearch.client.Request"))),
+        this.getClass().getName() + "$RestClientTransportAdvice");
+  }
+
+  @SuppressWarnings("unused")
+  public static class RestClientTransportAdvice {
+
+    @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class)
+    public static void onPrepareLowLevelRequest(
+        @Advice.Argument(1) Endpoint<?, ?, ?> endpoint, @Advice.Return Request request) {
+      VirtualField<Request, ElasticsearchEndpointDefinition> virtualField =
+          VirtualField.find(Request.class, ElasticsearchEndpointDefinition.class);
+      String endpointId = endpoint.id();
+      if (endpointId.startsWith("es/") && endpointId.length() > 3) {
+        endpointId = endpointId.substring(3);
+      }
+      virtualField.set(request, ElasticsearchEndpointMap.get(endpointId));
+    }
+  }
+}

instrumentation/elasticsearch/elasticsearch-api-client-7.16/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/elasticsearch/apiclient/ElasticsearchApiClientInstrumentationModule.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.instrumentation.elasticsearch.apiclient;
+
+import static java.util.Collections.singletonList;
+
+import com.google.auto.service.AutoService;
+import io.opentelemetry.javaagent.extension.instrumentation.InstrumentationModule;
+import io.opentelemetry.javaagent.extension.instrumentation.TypeInstrumentation;
+import java.util.List;
+
+@AutoService(InstrumentationModule.class)
+public class ElasticsearchApiClientInstrumentationModule extends InstrumentationModule {
+  public ElasticsearchApiClientInstrumentationModule() {
+    super("elasticsearch-api-client-7.16", "elasticsearch");
+  }
+
+  @Override
+  public List<TypeInstrumentation> typeInstrumentations() {
+    return singletonList(new ApiClientInstrumentation());
+  }
+}