From d1d76d2ea98bc2a124bd0c81d6ac9e52f8ef396f Mon Sep 17 00:00:00 2001 From: Jonas Kunz Date: Thu, 19 Oct 2023 15:20:12 +0200 Subject: [PATCH 1/6] Fix aws sdk instrumentation (#3373) * Fix aws sdk instrumentation * Updated changelog and docs * Removed argument added only for testing --- CHANGELOG.asciidoc | 1 + .../apm-aws-sdk/apm-aws-sdk-2-plugin/pom.xml | 2 +- ...BaseAsyncClientHandlerInstrumentation.java | 14 ++- .../BaseSyncClientHandlerInstrumentation.java | 15 ++- .../ClientHandlerConfigInstrumentation.java | 95 +++++++++++++++++++ ...ic.apm.agent.sdk.ElasticApmInstrumentation | 3 +- docs/supported-technologies.asciidoc | 6 +- pom.xml | 2 +- 8 files changed, 130 insertions(+), 8 deletions(-) create mode 100644 apm-agent-plugins/apm-aws-sdk/apm-aws-sdk-2-plugin/src/main/java/co/elastic/apm/agent/awssdk/v2/ClientHandlerConfigInstrumentation.java diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 0a8fb52ffb..c55082b337 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -34,6 +34,7 @@ Use subheadings with the "=====" level for adding notes for unreleased changes: [float] ===== Features * Added protection against invalid timestamps provided by manual instrumentation - {pull}3363[#3363] +* Added support for AWS SDK 2.21 - {pull}3373[#3373] [float] ===== Bug fixes diff --git a/apm-agent-plugins/apm-aws-sdk/apm-aws-sdk-2-plugin/pom.xml b/apm-agent-plugins/apm-aws-sdk/apm-aws-sdk-2-plugin/pom.xml index 531467cc31..8b8d57c35a 100644 --- a/apm-agent-plugins/apm-aws-sdk/apm-aws-sdk-2-plugin/pom.xml +++ b/apm-agent-plugins/apm-aws-sdk/apm-aws-sdk-2-plugin/pom.xml @@ -13,7 +13,7 @@ ${project.basedir}/../../.. - 2.20.157 + 2.21.1 2.0.0 8 diff --git a/apm-agent-plugins/apm-aws-sdk/apm-aws-sdk-2-plugin/src/main/java/co/elastic/apm/agent/awssdk/v2/BaseAsyncClientHandlerInstrumentation.java b/apm-agent-plugins/apm-aws-sdk/apm-aws-sdk-2-plugin/src/main/java/co/elastic/apm/agent/awssdk/v2/BaseAsyncClientHandlerInstrumentation.java index 8fc839664d..103ff16596 100644 --- a/apm-agent-plugins/apm-aws-sdk/apm-aws-sdk-2-plugin/src/main/java/co/elastic/apm/agent/awssdk/v2/BaseAsyncClientHandlerInstrumentation.java +++ b/apm-agent-plugins/apm-aws-sdk/apm-aws-sdk-2-plugin/src/main/java/co/elastic/apm/agent/awssdk/v2/BaseAsyncClientHandlerInstrumentation.java @@ -22,6 +22,8 @@ import co.elastic.apm.agent.awssdk.v2.helper.S3Helper; import co.elastic.apm.agent.awssdk.v2.helper.SQSHelper; import co.elastic.apm.agent.sdk.ElasticApmInstrumentation; +import co.elastic.apm.agent.sdk.logging.Logger; +import co.elastic.apm.agent.sdk.logging.LoggerFactory; import co.elastic.apm.agent.tracer.GlobalTracer; import co.elastic.apm.agent.tracer.Span; import co.elastic.apm.agent.tracer.Tracer; @@ -35,6 +37,7 @@ import software.amazon.awssdk.core.client.config.SdkClientOption; import software.amazon.awssdk.core.client.handler.ClientExecutionParams; import software.amazon.awssdk.core.http.ExecutionContext; +import software.amazon.awssdk.core.internal.handler.BaseAsyncClientHandler; import software.amazon.awssdk.core.internal.http.TransformingAsyncResponseHandler; import java.net.URI; @@ -70,12 +73,21 @@ public Collection getInstrumentationGroupNames() { @SuppressWarnings({"unchecked", "rawtypes"}) public static class AdviceClass { + private static final Logger logger = LoggerFactory.getLogger(AdviceClass.class); + @Advice.OnMethodEnter(suppress = Throwable.class, inline = false) @Advice.AssignReturned.ToArguments(@Advice.AssignReturned.ToArguments.ToArgument(value = 2)) public static TransformingAsyncResponseHandler enterDoExecute(@Advice.Argument(value = 0) ClientExecutionParams clientExecutionParams, @Advice.Argument(value = 1) ExecutionContext executionContext, @Advice.Argument(value = 2) TransformingAsyncResponseHandler responseHandler, - @Advice.FieldValue("clientConfiguration") SdkClientConfiguration clientConfiguration) { + @Advice.This BaseAsyncClientHandler handler) { + + SdkClientConfiguration clientConfiguration = ClientHandlerConfigInstrumentation.AdviceClass.getConfig(handler); + if(clientConfiguration == null) { + logger.warn("Not tracing AWS request due to being unable to resolve the configuration"); + return responseHandler; + } + String awsService = executionContext.executionAttributes().getAttribute(AwsSignerExecutionAttribute.SERVICE_NAME); SdkRequest sdkRequest = clientExecutionParams.getInput(); URI uri = clientConfiguration.option(SdkClientOption.ENDPOINT); diff --git a/apm-agent-plugins/apm-aws-sdk/apm-aws-sdk-2-plugin/src/main/java/co/elastic/apm/agent/awssdk/v2/BaseSyncClientHandlerInstrumentation.java b/apm-agent-plugins/apm-aws-sdk/apm-aws-sdk-2-plugin/src/main/java/co/elastic/apm/agent/awssdk/v2/BaseSyncClientHandlerInstrumentation.java index b49d4a009b..ccc7dfa58a 100644 --- a/apm-agent-plugins/apm-aws-sdk/apm-aws-sdk-2-plugin/src/main/java/co/elastic/apm/agent/awssdk/v2/BaseSyncClientHandlerInstrumentation.java +++ b/apm-agent-plugins/apm-aws-sdk/apm-aws-sdk-2-plugin/src/main/java/co/elastic/apm/agent/awssdk/v2/BaseSyncClientHandlerInstrumentation.java @@ -24,6 +24,8 @@ import co.elastic.apm.agent.awssdk.v2.helper.sqs.wrapper.MessageListWrapper; import co.elastic.apm.agent.common.JvmRuntimeInfo; import co.elastic.apm.agent.sdk.ElasticApmInstrumentation; +import co.elastic.apm.agent.sdk.logging.Logger; +import co.elastic.apm.agent.sdk.logging.LoggerFactory; import co.elastic.apm.agent.tracer.GlobalTracer; import co.elastic.apm.agent.tracer.Outcome; import co.elastic.apm.agent.tracer.Span; @@ -39,6 +41,8 @@ import software.amazon.awssdk.core.client.config.SdkClientOption; import software.amazon.awssdk.core.client.handler.ClientExecutionParams; import software.amazon.awssdk.core.http.ExecutionContext; +import software.amazon.awssdk.core.internal.handler.BaseAsyncClientHandler; +import software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler; import javax.annotation.Nullable; import java.net.URI; @@ -81,11 +85,20 @@ public Collection getInstrumentationGroupNames() { @SuppressWarnings("rawtypes") public static class AdviceClass { + private static final Logger logger = LoggerFactory.getLogger(AdviceClass.class); + @Nullable @Advice.OnMethodEnter(suppress = Throwable.class, inline = false) public static Object enterDoExecute(@Advice.Argument(value = 0) ClientExecutionParams clientExecutionParams, @Advice.Argument(value = 1) ExecutionContext executionContext, - @Advice.FieldValue("clientConfiguration") SdkClientConfiguration clientConfiguration) { + @Advice.This BaseSyncClientHandler handler) { + + SdkClientConfiguration clientConfiguration = ClientHandlerConfigInstrumentation.AdviceClass.getConfig(handler); + if(clientConfiguration == null) { + logger.warn("Not tracing AWS request due to being unable to resolve the configuration"); + return null; + } + String awsService = executionContext.executionAttributes().getAttribute(AwsSignerExecutionAttribute.SERVICE_NAME); SdkRequest sdkRequest = clientExecutionParams.getInput(); URI uri = clientConfiguration.option(SdkClientOption.ENDPOINT); diff --git a/apm-agent-plugins/apm-aws-sdk/apm-aws-sdk-2-plugin/src/main/java/co/elastic/apm/agent/awssdk/v2/ClientHandlerConfigInstrumentation.java b/apm-agent-plugins/apm-aws-sdk/apm-aws-sdk-2-plugin/src/main/java/co/elastic/apm/agent/awssdk/v2/ClientHandlerConfigInstrumentation.java new file mode 100644 index 0000000000..88f45bd43d --- /dev/null +++ b/apm-agent-plugins/apm-aws-sdk/apm-aws-sdk-2-plugin/src/main/java/co/elastic/apm/agent/awssdk/v2/ClientHandlerConfigInstrumentation.java @@ -0,0 +1,95 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package co.elastic.apm.agent.awssdk.v2; + +import co.elastic.apm.agent.awssdk.v2.helper.DynamoDbHelper; +import co.elastic.apm.agent.awssdk.v2.helper.S3Helper; +import co.elastic.apm.agent.awssdk.v2.helper.SQSHelper; +import co.elastic.apm.agent.awssdk.v2.helper.sqs.wrapper.MessageListWrapper; +import co.elastic.apm.agent.common.JvmRuntimeInfo; +import co.elastic.apm.agent.sdk.ElasticApmInstrumentation; +import co.elastic.apm.agent.sdk.weakconcurrent.WeakConcurrent; +import co.elastic.apm.agent.sdk.weakconcurrent.WeakMap; +import co.elastic.apm.agent.tracer.GlobalTracer; +import co.elastic.apm.agent.tracer.Outcome; +import co.elastic.apm.agent.tracer.Span; +import co.elastic.apm.agent.tracer.Tracer; +import net.bytebuddy.asm.Advice; +import net.bytebuddy.description.method.MethodDescription; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.matcher.ElementMatcher; +import software.amazon.awssdk.auth.signer.AwsSignerExecutionAttribute; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.SdkResponse; +import software.amazon.awssdk.core.client.config.SdkClientConfiguration; +import software.amazon.awssdk.core.client.config.SdkClientOption; +import software.amazon.awssdk.core.client.handler.ClientExecutionParams; +import software.amazon.awssdk.core.http.ExecutionContext; + +import javax.annotation.Nullable; +import java.net.URI; +import java.util.Collection; +import java.util.Collections; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import static net.bytebuddy.matcher.ElementMatchers.isConstructor; +import static net.bytebuddy.matcher.ElementMatchers.nameStartsWith; +import static net.bytebuddy.matcher.ElementMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; + +public class ClientHandlerConfigInstrumentation extends ElasticApmInstrumentation { + + + @Override + public ElementMatcher getTypeMatcher() { + return named("software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler") + .or(named("software.amazon.awssdk.core.internal.handler.BaseAsyncClientHandler")); + } + + @Override + public ElementMatcher getMethodMatcher() { + return isConstructor() + .and(takesArgument(0, named("software.amazon.awssdk.core.client.config.SdkClientConfiguration"))); + } + + @Override + public Collection getInstrumentationGroupNames() { + return Collections.singleton("aws-sdk"); + } + + + @SuppressWarnings("rawtypes") + public static class AdviceClass { + + private static final WeakMap configMap = WeakConcurrent.buildMap(); + + @Nullable + public static SdkClientConfiguration getConfig(Object handler) { + return configMap.get(handler); + } + + + @Advice.OnMethodExit(suppress = Throwable.class, inline = false) + public static void exit(@Advice.This Object handler, @Advice.Argument(value = 0) SdkClientConfiguration config) { + configMap.put(handler, config); + } + + } +} diff --git a/apm-agent-plugins/apm-aws-sdk/apm-aws-sdk-2-plugin/src/main/resources/META-INF/services/co.elastic.apm.agent.sdk.ElasticApmInstrumentation b/apm-agent-plugins/apm-aws-sdk/apm-aws-sdk-2-plugin/src/main/resources/META-INF/services/co.elastic.apm.agent.sdk.ElasticApmInstrumentation index 5dcc2147ec..aa3c1f1b14 100644 --- a/apm-agent-plugins/apm-aws-sdk/apm-aws-sdk-2-plugin/src/main/resources/META-INF/services/co.elastic.apm.agent.sdk.ElasticApmInstrumentation +++ b/apm-agent-plugins/apm-aws-sdk/apm-aws-sdk-2-plugin/src/main/resources/META-INF/services/co.elastic.apm.agent.sdk.ElasticApmInstrumentation @@ -1,4 +1,5 @@ co.elastic.apm.agent.awssdk.v2.BaseSyncClientHandlerInstrumentation co.elastic.apm.agent.awssdk.v2.BaseAsyncClientHandlerInstrumentation +co.elastic.apm.agent.awssdk.v2.ClientHandlerConfigInstrumentation co.elastic.apm.agent.awssdk.v2.GetMessagesInstrumentation -co.elastic.apm.agent.awssdk.v2.AmazonSQSMessagingClientWrapperInstrumentation \ No newline at end of file +co.elastic.apm.agent.awssdk.v2.AmazonSQSMessagingClientWrapperInstrumentation diff --git a/docs/supported-technologies.asciidoc b/docs/supported-technologies.asciidoc index fe223cfc0e..06c35e2260 100644 --- a/docs/supported-technologies.asciidoc +++ b/docs/supported-technologies.asciidoc @@ -294,12 +294,12 @@ This provides support for `com.datastax.cassandra:cassandra-driver-core` and |AWS DynamoDB |1.x, 2.x |The agent creates spans for interactions with the AWS DynamoDb service through the AWS Java SDK. -|1.31.0 +|1.31.0, 2.21+ since 1.44.0 |AWS S3 |1.x, 2.x |The agent creates spans for interactions with the AWS S3 service through the AWS Java SDK. -|1.31.0 +|1.31.0, 2.21+ since 1.44.0 |=== @@ -459,7 +459,7 @@ same trace. |AWS SQS |1.x, 2.x |The agent captures SQS Message sends and polling as well as SQS message sends and consumption through JMS. -|1.34.0 +|1.34.0, 2.21+ since 1.44.0 |=== diff --git a/pom.xml b/pom.xml index a658c40a84..338f7d30c6 100644 --- a/pom.xml +++ b/pom.xml @@ -140,7 +140,7 @@ 1.17 - 1.16.3 + 1.19.1 2.38.0 From 4f4b1c71af89fb4fd3e839b8329034e50563b1f8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Oct 2023 14:25:22 +0000 Subject: [PATCH 2/6] Bump version.log4j from 2.12.4 to 2.21.0 (#3366) * Bump version.log4j from 2.12.4 to 2.21.0 Bumps `version.log4j` from 2.12.4 to 2.21.0. Updates `org.apache.logging.log4j:log4j-core` from 2.12.4 to 2.21.0 Updates `org.apache.logging.log4j:log4j-slf4j-impl` from 2.12.4 to 2.21.0 --- updated-dependencies: - dependency-name: org.apache.logging.log4j:log4j-core dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: org.apache.logging.log4j:log4j-slf4j-impl dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update pom.xml * Update pom.xml --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Jonas Kunz Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- apm-agent-plugins/apm-ecs-logging-plugin/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apm-agent-plugins/apm-ecs-logging-plugin/pom.xml b/apm-agent-plugins/apm-ecs-logging-plugin/pom.xml index e61eda6307..e0a68cbc7a 100644 --- a/apm-agent-plugins/apm-ecs-logging-plugin/pom.xml +++ b/apm-agent-plugins/apm-ecs-logging-plugin/pom.xml @@ -48,7 +48,7 @@ org.apache.logging.log4j log4j-core - 2.20.0 + 2.21.0 provided true From 5fcba1047ae2ccff77d0ddb139143c2f6c8bd166 Mon Sep 17 00:00:00 2001 From: Nugusbayev Kanagat <48118512+videnkz@users.noreply.github.com> Date: Tue, 24 Oct 2023 15:37:03 +0600 Subject: [PATCH 3/6] added capturing s3 otel attributes from lamba S3Event (#3364) * added capturing s3 otel attributes from lamba S3Event * added entry to changelog * minor polish --- CHANGELOG.asciidoc | 1 + .../awslambda/helper/S3TransactionHelper.java | 18 +++++++++++++----- .../apm/agent/awslambda/S3EventLambdaTest.java | 13 ++++++++++++- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index c55082b337..c8f21ef12b 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -35,6 +35,7 @@ Use subheadings with the "=====" level for adding notes for unreleased changes: ===== Features * Added protection against invalid timestamps provided by manual instrumentation - {pull}3363[#3363] * Added support for AWS SDK 2.21 - {pull}3373[#3373] +* Capture bucket and object key to Lambda transaction as OTel attributes - `aws.s3.bueckt`, `aws.s3.key` - {pull}3364[#3364] [float] ===== Bug fixes diff --git a/apm-agent-plugins/apm-awslambda-plugin/src/main/java/co/elastic/apm/agent/awslambda/helper/S3TransactionHelper.java b/apm-agent-plugins/apm-awslambda-plugin/src/main/java/co/elastic/apm/agent/awslambda/helper/S3TransactionHelper.java index fbfda924cd..e2203a665d 100644 --- a/apm-agent-plugins/apm-awslambda-plugin/src/main/java/co/elastic/apm/agent/awslambda/helper/S3TransactionHelper.java +++ b/apm-agent-plugins/apm-awslambda-plugin/src/main/java/co/elastic/apm/agent/awslambda/helper/S3TransactionHelper.java @@ -78,12 +78,20 @@ protected void setTransactionTriggerData(Transaction transaction, S3Event s3Even cloudOrigin.withRegion(s3NotificationRecord.getAwsRegion()); - if (null != s3NotificationRecord.getS3() && null != s3NotificationRecord.getS3().getBucket()) { + if (null != s3NotificationRecord.getS3()) { S3EventNotification.S3BucketEntity bucket = s3NotificationRecord.getS3().getBucket(); - ServiceOrigin serviceOrigin = transaction.getContext().getServiceOrigin(); - serviceOrigin.withId(bucket.getArn()); - serviceOrigin.withName(bucket.getName()); - serviceOrigin.withVersion(s3NotificationRecord.getEventVersion()); + if (null != bucket) { + ServiceOrigin serviceOrigin = transaction.getContext().getServiceOrigin(); + serviceOrigin.withId(bucket.getArn()); + serviceOrigin.withName(bucket.getName()); + serviceOrigin.withVersion(s3NotificationRecord.getEventVersion()); + + transaction.withOtelAttribute("aws.s3.bucket", bucket.getName()); + } + S3EventNotification.S3ObjectEntity object = s3NotificationRecord.getS3().getObject(); + if (null != object) { + transaction.withOtelAttribute("aws.s3.key", object.getKey()); + } } } } diff --git a/apm-agent-plugins/apm-awslambda-plugin/src/test/java/co/elastic/apm/agent/awslambda/S3EventLambdaTest.java b/apm-agent-plugins/apm-awslambda-plugin/src/test/java/co/elastic/apm/agent/awslambda/S3EventLambdaTest.java index d7d7d47e21..82c737523c 100644 --- a/apm-agent-plugins/apm-awslambda-plugin/src/test/java/co/elastic/apm/agent/awslambda/S3EventLambdaTest.java +++ b/apm-agent-plugins/apm-awslambda-plugin/src/test/java/co/elastic/apm/agent/awslambda/S3EventLambdaTest.java @@ -66,7 +66,8 @@ protected boolean supportsContextPropagation() { private S3EventNotification.S3EventNotificationRecord createS3NotificationRecord() { S3EventNotification.ResponseElementsEntity responseElements = new S3EventNotification.ResponseElementsEntity("xAmzId2", S3_REQUEST_ID); S3EventNotification.S3BucketEntity bucket = new S3EventNotification.S3BucketEntity(S3_BUCKET_NAME, null, S3_BUCKET_ARN); - S3EventNotification.S3Entity s3 = new S3EventNotification.S3Entity("configId", bucket, null, "3.3"); + S3EventNotification.S3ObjectEntity object = new S3EventNotification.S3ObjectEntity("b21b84d653bb07b05b1e6b33684dc11b", 1305107, "b21b84d653bb07b05b1e6b33684dc11b", "0C0F6F405D6ED209E1"); + S3EventNotification.S3Entity s3 = new S3EventNotification.S3Entity("configId", bucket, object, "3.3"); return new S3EventNotification.S3EventNotificationRecord(EVENT_SOURCE_REGION, S3_EVENT_NAME, S3_EVENT_SOURCE, null, S3_EVENT_VERSION, null, responseElements, s3, null); } @@ -102,6 +103,8 @@ public void testBasicCall() { assertThat(faas.getId()).isEqualTo(TestContext.FUNCTION_ARN); assertThat(faas.getTrigger().getType()).isEqualTo("datasource"); assertThat(faas.getTrigger().getRequestId()).isEqualTo(S3_REQUEST_ID); + + verifyOtelAttributes(transaction); } @Test @@ -174,4 +177,12 @@ private void validateResultsForUnspecifiedRecord() { assertThat(faas.getTrigger().getType()).isEqualTo("datasource"); assertThat(faas.getTrigger().getRequestId()).isNull(); } + + private void verifyOtelAttributes(Transaction transaction) { + Object s3keyAttribute = transaction.getOtelAttributes().get("aws.s3.key"); + assertThat(s3keyAttribute).isInstanceOf(String.class).isEqualTo("b21b84d653bb07b05b1e6b33684dc11b"); + + Object s3bucketAttribute = transaction.getOtelAttributes().get("aws.s3.bucket"); + assertThat(s3bucketAttribute).isInstanceOf(String.class).isEqualTo(S3_BUCKET_NAME); + } } From 2a3ff0bc1eac7f35bc69d3276e0fec8d71c6e440 Mon Sep 17 00:00:00 2001 From: Nugusbayev Kanagat <48118512+videnkz@users.noreply.github.com> Date: Wed, 25 Oct 2023 18:21:55 +0600 Subject: [PATCH 4/6] added entry to community plugins (#3383) --- docs/community-plugins.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/community-plugins.asciidoc b/docs/community-plugins.asciidoc index 47d565bffc..09c8568703 100644 --- a/docs/community-plugins.asciidoc +++ b/docs/community-plugins.asciidoc @@ -15,3 +15,4 @@ NOTE: Elastic provides no warranty or support for community-sourced plugins. [horizontal] https://github.com/elastic/apm-agent-java-plugin-example[example repo]:: an example plugin for a custom webserver https://github.com/videnkz/apm-jetty-httpclient-plugin[apm-jetty-httpclient-plugin]:: Jetty HttpClient plugin +https://github.com/videnkz/apm-grizzly-asynchttpclient-plugin[apm-grizzly-asynchttpclient-plugin]:: Grizzly AsyncHttpClient plugin From bab9ccebf488809a40014512576bbacd8c01fd66 Mon Sep 17 00:00:00 2001 From: Jonas Kunz Date: Wed, 25 Oct 2023 16:21:13 +0200 Subject: [PATCH 5/6] Update tests for Jedis 5 (#3382) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../apm-jedis-4-plugin/pom.xml | 14 +++++ .../agent/jedis/Jedis4InstrumentationIT.java | 6 ++- .../apm-jedis-5-tests/pom.xml | 54 +++++++++++++++++++ .../java/jedis/Jedis5InstrumentationIT.java | 34 ++++++++++++ .../agent/jedis/Jedis1InstrumentationIT.java | 4 ++ apm-agent-plugins/apm-redis-plugin/pom.xml | 1 + docs/supported-technologies.asciidoc | 4 +- 7 files changed, 114 insertions(+), 3 deletions(-) create mode 100644 apm-agent-plugins/apm-redis-plugin/apm-jedis-5-tests/pom.xml create mode 100644 apm-agent-plugins/apm-redis-plugin/apm-jedis-5-tests/src/test/java/jedis/Jedis5InstrumentationIT.java diff --git a/apm-agent-plugins/apm-redis-plugin/apm-jedis-4-plugin/pom.xml b/apm-agent-plugins/apm-redis-plugin/apm-jedis-4-plugin/pom.xml index b079d966ed..b9502d7152 100644 --- a/apm-agent-plugins/apm-redis-plugin/apm-jedis-4-plugin/pom.xml +++ b/apm-agent-plugins/apm-redis-plugin/apm-jedis-4-plugin/pom.xml @@ -44,4 +44,18 @@ + + + + maven-jar-plugin + + + + test-jar + + + + + + diff --git a/apm-agent-plugins/apm-redis-plugin/apm-jedis-4-plugin/src/test/java/co/elastic/apm/agent/jedis/Jedis4InstrumentationIT.java b/apm-agent-plugins/apm-redis-plugin/apm-jedis-4-plugin/src/test/java/co/elastic/apm/agent/jedis/Jedis4InstrumentationIT.java index db867f3636..7fb48eb2b4 100644 --- a/apm-agent-plugins/apm-redis-plugin/apm-jedis-4-plugin/src/test/java/co/elastic/apm/agent/jedis/Jedis4InstrumentationIT.java +++ b/apm-agent-plugins/apm-redis-plugin/apm-jedis-4-plugin/src/test/java/co/elastic/apm/agent/jedis/Jedis4InstrumentationIT.java @@ -29,7 +29,7 @@ import static org.assertj.core.api.Assertions.assertThat; -class Jedis4InstrumentationIT extends Jedis1InstrumentationIT { +public class Jedis4InstrumentationIT extends Jedis1InstrumentationIT { private JedisSharding shardedJedis; private UnifiedJedis binaryJedis; @@ -50,6 +50,10 @@ void testShardedJedis() { shardedJedis.set("foo", "bar"); assertThat(shardedJedis.get("foo".getBytes())).isEqualTo("bar".getBytes()); + verifyShardedJedisSpan(); + } + + protected void verifyShardedJedisSpan() { assertTransactionWithRedisSpans("CLIENT", "CLIENT", "SET", "GET"); } diff --git a/apm-agent-plugins/apm-redis-plugin/apm-jedis-5-tests/pom.xml b/apm-agent-plugins/apm-redis-plugin/apm-jedis-5-tests/pom.xml new file mode 100644 index 0000000000..80c85d75e9 --- /dev/null +++ b/apm-agent-plugins/apm-redis-plugin/apm-jedis-5-tests/pom.xml @@ -0,0 +1,54 @@ + + + 4.0.0 + + + apm-redis-plugin + co.elastic.apm + 1.43.1-SNAPSHOT + + + apm-jedis-5-tests + ${project.groupId}:${project.artifactId} + + + ${project.basedir}/../../.. + + + + + ${project.groupId} + apm-redis-common + test-jar + ${project.version} + test + + + ${project.groupId} + apm-jedis-plugin + test-jar + ${project.version} + test + + + ${project.groupId} + apm-jedis-4-plugin + ${project.version} + test + + + ${project.groupId} + apm-jedis-4-plugin + test-jar + ${project.version} + test + + + redis.clients + jedis + 5.0.2 + test + + + + diff --git a/apm-agent-plugins/apm-redis-plugin/apm-jedis-5-tests/src/test/java/jedis/Jedis5InstrumentationIT.java b/apm-agent-plugins/apm-redis-plugin/apm-jedis-5-tests/src/test/java/jedis/Jedis5InstrumentationIT.java new file mode 100644 index 0000000000..dc5ba27cb3 --- /dev/null +++ b/apm-agent-plugins/apm-redis-plugin/apm-jedis-5-tests/src/test/java/jedis/Jedis5InstrumentationIT.java @@ -0,0 +1,34 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package jedis; + +import co.elastic.apm.agent.jedis.Jedis4InstrumentationIT; + +public class Jedis5InstrumentationIT extends Jedis4InstrumentationIT { + + @Override + protected void verifyShardedJedisSpan() { + assertTransactionWithRedisSpans("CLIENT", "CLIENT", "CLIENT", "CLIENT", "SET", "GET"); + } + + @Override + protected void verifyBasicJedisSpans() { + assertTransactionWithRedisSpans( "CLIENT", "CLIENT", "SET", "GET"); + } +} diff --git a/apm-agent-plugins/apm-redis-plugin/apm-jedis-plugin/src/test/java/co/elastic/apm/agent/jedis/Jedis1InstrumentationIT.java b/apm-agent-plugins/apm-redis-plugin/apm-jedis-plugin/src/test/java/co/elastic/apm/agent/jedis/Jedis1InstrumentationIT.java index 14c7b03f3e..6b50bfb29b 100644 --- a/apm-agent-plugins/apm-redis-plugin/apm-jedis-plugin/src/test/java/co/elastic/apm/agent/jedis/Jedis1InstrumentationIT.java +++ b/apm-agent-plugins/apm-redis-plugin/apm-jedis-plugin/src/test/java/co/elastic/apm/agent/jedis/Jedis1InstrumentationIT.java @@ -55,6 +55,10 @@ void testJedis() { jedis.set("foo", "bar"); assertThat(jedis.get("foo".getBytes())).isEqualTo("bar".getBytes()); + verifyBasicJedisSpans(); + } + + protected void verifyBasicJedisSpans() { assertTransactionWithRedisSpans("SET", "GET"); } diff --git a/apm-agent-plugins/apm-redis-plugin/pom.xml b/apm-agent-plugins/apm-redis-plugin/pom.xml index cda530e82c..a61351f2f7 100644 --- a/apm-agent-plugins/apm-redis-plugin/pom.xml +++ b/apm-agent-plugins/apm-redis-plugin/pom.xml @@ -20,6 +20,7 @@ apm-jedis-2-tests apm-jedis-3-tests apm-jedis-4-plugin + apm-jedis-5-tests apm-jedis-plugin apm-lettuce-3-tests apm-lettuce-plugin diff --git a/docs/supported-technologies.asciidoc b/docs/supported-technologies.asciidoc index 06c35e2260..b95f660781 100644 --- a/docs/supported-technologies.asciidoc +++ b/docs/supported-technologies.asciidoc @@ -254,9 +254,9 @@ Other Servlet 3+ compliant servers will most likely work as well. |1.9.0 |Redis Jedis -|1.4.0-4.x +|1.4.0-5.x |The agent creates spans for interactions with the Jedis client. -|1.10.0, 4.x since 1.31.0 +|1.10.0, 4+ since 1.31.0 |Redis Lettuce |3.4+ From d49065d8701d18bfda95411c1044534da17121a2 Mon Sep 17 00:00:00 2001 From: jackshirazi Date: Thu, 26 Oct 2023 10:05:36 +0100 Subject: [PATCH 6/6] Process otel benchmark (#3384) * Add a processor class for the otel benchmark which outputs JSON for Elasticsearch * fix comment and make a couple of things final * fix javadoc break --- .../ProcessOtelBenchmarkResults.java | 236 ++++++++++++++++++ 1 file changed, 236 insertions(+) create mode 100644 apm-agent-benchmarks/src/main/java/co/elastic/apm/agent/benchmark/ProcessOtelBenchmarkResults.java diff --git a/apm-agent-benchmarks/src/main/java/co/elastic/apm/agent/benchmark/ProcessOtelBenchmarkResults.java b/apm-agent-benchmarks/src/main/java/co/elastic/apm/agent/benchmark/ProcessOtelBenchmarkResults.java new file mode 100644 index 0000000000..bfef48cbbc --- /dev/null +++ b/apm-agent-benchmarks/src/main/java/co/elastic/apm/agent/benchmark/ProcessOtelBenchmarkResults.java @@ -0,0 +1,236 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package co.elastic.apm.agent.benchmark; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import java.io.*; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.jar.JarFile; + +public class ProcessOtelBenchmarkResults { + + private final ArrayNode bechmarkResultJson; + private final String resultFilePath; + private final String elasticVersion; + private final String otelVersion; + private final ObjectMapper objectMapper; + + private ProcessOtelBenchmarkResults(OtelBenchParser parser, String resultFilePath, String elasticVersion, String otelVersion) throws IOException { + this.resultFilePath = resultFilePath; + this.elasticVersion = elasticVersion; + this.otelVersion = otelVersion; + objectMapper = new ObjectMapper(); + bechmarkResultJson = parser.createTree(objectMapper); + } + + /** + * Example file contents for args 0: + *

+ * 

+     * Performing startup warming phase for 60 seconds...
+     * Starting disposable JFR warmup recording...
+     * Stopping disposable JFR warmup recording...
+     * Warmup complete.
+     * ----------------------------------------------------------
+     *  Run at Tue Oct 24 16:11:38 UTC 2023
+     *  release : compares no agent, latest stable, and latest snapshot agents
+     *  5 users, 5000 iterations
+     * ----------------------------------------------------------
+     * Agent               :              none           latest         snapshot   elastic-latest    elastic-async elastic-snapshot
+     * Run duration        :          00:00:57         00:01:07         00:01:08         00:01:01         00:01:03         00:01:00
+     * Avg. CPU (user) %   :         0.3620707        0.4131534       0.40864184       0.39786905       0.42508292        0.3923088
+     * Max. CPU (user) %   :         0.5321782       0.57107234        0.5597015        0.5860349       0.58168316       0.54613465
+     * Avg. mch tot cpu %  :        0.94261116       0.95221955       0.93572134        0.9339143        0.9495207       0.93751615
+     * Startup time (ms)   :             11378            12870            12890            12861             8686            12868
+     * Total allocated MB  :          15069.28         20168.60         20155.76         16018.64         15854.65         15904.63
+     * Min heap used (MB)  :             74.39           123.06           115.00           139.10           114.14           108.85
+     * Max heap used (MB)  :            430.09           414.73           391.81           543.44           474.40           387.21
+     * Thread switch rate  :         54890.133        54588.465        54918.758        54230.484        53694.992        53908.035
+     * GC time (ms)        :               417              606              669              920              406              520
+     * GC pause time (ms)  :               423              606              669              920              406              520
+     * Req. mean (ms)      :              4.21             4.97             5.01             4.51             4.67             4.40
+     * Req. p95 (ms)       :             10.66            12.96            13.06            11.74            11.96            11.29
+     * Iter. mean (ms)     :             55.19            64.99            65.60            58.89            61.06            57.74
+     * Iter. p95 (ms)      :             85.51            99.71           100.96            89.89            95.43            88.39
+     * Net read avg (bps)  :       13733443.00      12234844.00      12082882.00      12154885.00      10863340.00      11873964.00
+     * Net write avg (bps) :       18365140.00      66959119.00      66142391.00      16239026.00      14507576.00      15864209.00
+     * Peak threads        :                40               54               53               48               48               48
+     *
+ * + * + * @param args 0 path to the test results (normally `build/reports/tests/test/classes/io.opentelemetry.OverheadTests.html`) + * 1 path to output file to be created by this class (eg `output.json`) + * 2 elastic version used in the test (eg `1.43.0`) + * 3 path to otel "latest" jar file used in the test (normally `opentelemetry-javaagent.jar`) + * @throws Exception io and parsing errors + */ + public static void main(String[] args) throws Exception { + OtelBenchParser parser = new OtelBenchParser(); + try (BufferedReader rdr = new BufferedReader(new FileReader(args[0]))) { + String line; + while ((line = rdr.readLine()) != null) { + parser.parse(line); + } + } + new ProcessOtelBenchmarkResults(parser, args[1], args[2], getVersionFromJar(args[3])).process(); + } + + private void process() throws IOException { + final JsonNode meta = bechmarkResultJson.objectNode() + .put("os_name", System.getProperty("os.name")) + .put("os_version", System.getProperty("os.version")) + .put("jdk_version", System.getProperty("java.version")) + .put("elastic_version", elasticVersion) + .put("otel_version", otelVersion) + .put("executed_at", Instant.now().toString()); + for (JsonNode benchmark : bechmarkResultJson) { + ((ObjectNode) benchmark).put("@timestamp", System.currentTimeMillis()); + ((ObjectNode) benchmark).set("meta", meta); + } + writeBulkFile(this.resultFilePath); + } + + private static String getVersionFromJar(String jarPath) throws IOException { + try (JarFile jar = new JarFile(jarPath)) { + return jar.getManifest().getMainAttributes().getValue("Implementation-Version"); + } + } + + private void writeBulkFile(String resultFilePath) throws IOException { + final File file = new File(resultFilePath); + final FileWriter fileWriter = new FileWriter(file); + for (JsonNode benchmark : bechmarkResultJson) { + fileWriter.append("{ \"index\" : { \"_index\" : \"microbenchmarks\" } }\n"); + fileWriter.append(objectMapper.writer().writeValueAsString(benchmark)); + fileWriter.append("\n"); + } + fileWriter.close(); + } + + public static class OtelBenchParser { + + private boolean inResults; + private String runAtComment; + private String releaseComment; + private String configComment; + private String[] agentNames; + private String[] runDurations; + private final List results = new ArrayList<>(); + + public ArrayNode createTree(ObjectMapper objectMapper) { + ArrayNode rootNode = objectMapper.createArrayNode(); + for (int i = 0; i < agentNames.length; i++) { + ObjectNode benchmarkNode = rootNode.objectNode(); + String agentName = agentNames[i].toLowerCase().contains("elastic") ? agentNames[i] : "otel-"+agentNames[i] ; + if (agentNames[i].equals("none")) { + agentName = "none"; + } + benchmarkNode.put("agentName", agentName); + benchmarkNode.put("benchmark", "benchmarkJavaApmWithOtel."+agentName); + for (ResultValues values: results) { + benchmarkNode.put(values.getJsonTitle(), values.get(i)); + } + + rootNode.add(benchmarkNode); + } + return rootNode; + } + + public void parse(String result) throws ParseException { + if (inResults) { + if (runAtComment == null) { + runAtComment = result.strip(); + } else if (releaseComment == null) { + releaseComment = result.strip(); + } else if (configComment == null) { + configComment = result.strip(); + } else if (result.matches("\\s*\\-+\\s*")) { + //skip + } else if (agentNames == null) { + String[] agentAndNames = result.split(":"); + if (agentAndNames[0].trim().equals("Agent")) { + agentNames = agentAndNames[1].trim().split("\\s+"); + } else { + throw new ParseException("Expecting 'Agent : none latest ....' but got: "+result); + } + } else if (runDurations == null) { + String[] durations = result.split("\\s+:\\s+"); + if (durations[0].trim().equals("Run duration")) { + runDurations = durations[1].trim().split("\\s+"); + } else { + throw new ParseException("Expecting 'Run duration : nn:nn:nn nn:nn:nn ....' but got: "+result); + } + } else if (result.matches("\\s*\\<.*")) { + inResults = false; + } else { + results.add(new ResultValues(result, agentNames.length)); + } + } else if (result.matches("\\s*\\-+\\s*")) { + inResults = true; + } else { + System.out.println("X "+result); + } + } + + public static class ResultValues { + String valuesTitle; + double[] values; + public ResultValues(String resultLine, int expectedValuesCount) throws ParseException { + String[] titleAndResults = resultLine.split(":"); + if (titleAndResults.length != 2) { + throw new ParseException("Expected ' : <value1 value2 ...>' but got: "+resultLine); + } + valuesTitle = titleAndResults[0].trim(); + String[] resultValues = titleAndResults[1].trim().split("\\s+"); + if (resultValues.length != expectedValuesCount) { + throw new ParseException("Expected "+expectedValuesCount+" values but got "+resultValues.length +" for line: "+resultLine); + } + values = new double[expectedValuesCount]; + for (int i = 0; i < resultValues.length; i++) { + try { + values[i] = Double.parseDouble(resultValues[i]); + } catch (NumberFormatException e) { + throw new ParseException("Expected but didn't get numeric float value at result index "+i+" in results: "+resultLine); + } + } + } + + public String getJsonTitle() { + return valuesTitle.toLowerCase().replaceAll("[^\\w ]","").trim().replace(' ','_'); + } + + public double get(int i) { + return values[i]; + } + } + + public static class ParseException extends Exception { + public ParseException(String string) { + super(string); + } + + } + } +}