From bbda9a319726cf78925d34a8ada011e160a1479c Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Wed, 24 Aug 2022 13:36:01 -0700 Subject: [PATCH 01/13] Rate limited sampling --- .../internal/configuration/Configuration.java | 55 +++++-- .../configuration/ConfigurationBuilder.java | 58 +++++-- .../internal/exporter/AgentLogExporter.java | 69 +++++++- .../internal/init/RpConfigurationPolling.java | 29 ++-- .../agent/internal/init/SecondEntryPoint.java | 21 ++- .../internal/sampling/AiOverrideSampler.java | 28 +++- .../agent/internal/sampling/AiSampler.java} | 46 +++--- .../RateLimitedSamplingPercentage.java | 112 +++++++++++++ .../agent/internal/sampling/Samplers.java | 34 +++- .../internal/sampling/SamplingOverrides.java | 54 +++++-- .../internal/sampling/SamplingPercentage.java | 43 +++++ .../agent/internal/SamplingTestUtil.java | 61 +++++++ .../ConfigurationBuilderTest.java | 6 +- .../configuration/SamplingPercentageTest.java | 8 +- .../init/RpConfigurationPollingTest.java | 44 ++--- .../sampling/SamplingOverridesTest.java | 142 ++++++---------- .../exporter/AzureMonitorLogExporter.java | 4 +- .../implementation/LogDataMapper.java | 24 +-- .../implementation/SpanDataMapper.java | 14 +- settings.gradle | 4 +- .../resources/applicationinsights.json | 7 + .../disabled_applicationinsights.json | 7 + .../resources/applicationinsights.json | 7 + ...ler_spans_enabled_applicationinsights.json | 7 + .../disabled_applicationinsights.json | 8 +- .../disabled_applicationinsights.json | 7 + .../resources/applicationinsights.json | 7 + .../resources/applicationinsights.json | 7 + .../resources/applicationinsights.json | 7 + .../resources/applicationinsights.json | 7 + .../httpserver4xx_applicationinsights.json | 7 + .../resources/applicationinsights.json | 7 + ...ler_spans_enabled_applicationinsights.json | 7 + .../disabled_applicationinsights.json | 7 + .../disabled_applicationinsights.json | 7 + .../unmasked_applicationinsights.json | 7 + .../disabled_applicationinsights.json | 7 + ...ler_spans_enabled_applicationinsights.json | 7 + .../disabled_applicationinsights.json | 7 + .../disabled_applicationinsights.json | 7 + .../resources/applicationinsights.json | 7 + .../disabled_applicationinsights.json | 7 + .../disabled_applicationinsights.json | 7 + ...ler_spans_enabled_applicationinsights.json | 7 + .../resources/applicationinsights.json | 7 + .../apps/RateLimitedSampling/build.gradle.kts | 8 + .../smoketestapp/HealthCheckServlet.java | 38 +++++ .../smoketestapp/ServletFuncs.java | 45 ++++++ .../smoketestapp/SimpleSamplingServlet.java | 53 ++++++ .../smoketest/RateLimitedSamplingTest.java | 85 ++++++++++ .../resources/applicationinsights.json | 9 ++ .../src/smokeTest/resources/logback-test.xml | 0 .../resources/applicationinsights.json | 4 + .../build.gradle.kts | 0 .../SamplingOverrideTestServlet.java} | 2 +- .../smoketest/SamplingOverride2Test.java} | 22 +-- .../smoketest/SamplingOverrideTest.java} | 24 +-- .../resources/applicationinsights.json} | 12 +- .../resources/applicationinsights2.json} | 4 + .../src/smokeTest/resources/logback-test.xml | 11 ++ .../apps/SamplingOverridesV2/build.gradle.kts | 7 + .../SamplingOverrideTestServlet.java | 151 ++++++++++++++++++ .../smoketest/SamplingOverride2Test.java | 75 +++++++++ .../smoketest/SamplingOverrideTest.java | 151 ++++++++++++++++++ .../resources/applicationinsights.json | 63 ++++++++ .../resources/applicationinsights2.json | 27 ++++ .../src/smokeTest/resources/logback-test.xml | 11 ++ .../main/resources/applicationinsights.json | 4 +- ...ler_spans_enabled_applicationinsights.json | 7 + ...ler_spans_enabled_applicationinsights.json | 7 + .../resources/applicationinsights.json | 9 ++ .../disabled_applicationinsights.json | 7 + .../resources/applicationinsights.json | 15 +- .../resources/applicationinsights.json | 7 + .../resources/applicationinsights.json | 7 + .../resources/applicationinsights.json | 7 + .../resources/applicationinsights.json | 7 + .../resources/applicationinsights.json | 7 + .../resources/applicationinsights.json | 7 + .../resources/applicationinsights.json | 7 + .../resources/applicationinsights.json | 7 + .../smoketest/SmokeTestExtension.java | 24 +-- .../default_applicationinsights.json | 9 ++ 83 files changed, 1680 insertions(+), 292 deletions(-) rename agent/{azure-monitor-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/AzureMonitorSampler.java => agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/AiSampler.java} (79%) create mode 100644 agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/RateLimitedSamplingPercentage.java create mode 100644 agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingPercentage.java create mode 100644 agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/SamplingTestUtil.java create mode 100644 smoke-tests/apps/RateLimitedSampling/build.gradle.kts create mode 100644 smoke-tests/apps/RateLimitedSampling/src/main/java/com/microsoft/applicationinsights/smoketestapp/HealthCheckServlet.java create mode 100644 smoke-tests/apps/RateLimitedSampling/src/main/java/com/microsoft/applicationinsights/smoketestapp/ServletFuncs.java create mode 100644 smoke-tests/apps/RateLimitedSampling/src/main/java/com/microsoft/applicationinsights/smoketestapp/SimpleSamplingServlet.java create mode 100644 smoke-tests/apps/RateLimitedSampling/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/RateLimitedSamplingTest.java create mode 100644 smoke-tests/apps/RateLimitedSampling/src/smokeTest/resources/applicationinsights.json rename smoke-tests/apps/{TelemetryFiltering => RateLimitedSampling}/src/smokeTest/resources/logback-test.xml (100%) rename smoke-tests/apps/{TelemetryFiltering => SamplingOverridesV1}/build.gradle.kts (100%) rename smoke-tests/apps/{TelemetryFiltering/src/main/java/com/microsoft/applicationinsights/smoketestapp/TelemetryFilteringTestServlet.java => SamplingOverridesV1/src/main/java/com/microsoft/applicationinsights/smoketestapp/SamplingOverrideTestServlet.java} (98%) rename smoke-tests/apps/{TelemetryFiltering/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/TelemetryFiltering2SmokeTest.java => SamplingOverridesV1/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/SamplingOverride2Test.java} (78%) rename smoke-tests/apps/{TelemetryFiltering/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/TelemetryFilteringSmokeTest.java => SamplingOverridesV1/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/SamplingOverrideTest.java} (88%) rename smoke-tests/apps/{TelemetryFiltering/src/smokeTest/resources/telemetryfiltering_applicationinsights.json => SamplingOverridesV1/src/smokeTest/resources/applicationinsights.json} (77%) rename smoke-tests/apps/{TelemetryFiltering/src/smokeTest/resources/telemetryfiltering2_applicationinsights.json => SamplingOverridesV1/src/smokeTest/resources/applicationinsights2.json} (82%) create mode 100644 smoke-tests/apps/SamplingOverridesV1/src/smokeTest/resources/logback-test.xml create mode 100644 smoke-tests/apps/SamplingOverridesV2/build.gradle.kts create mode 100644 smoke-tests/apps/SamplingOverridesV2/src/main/java/com/microsoft/applicationinsights/smoketestapp/SamplingOverrideTestServlet.java create mode 100644 smoke-tests/apps/SamplingOverridesV2/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/SamplingOverride2Test.java create mode 100644 smoke-tests/apps/SamplingOverridesV2/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/SamplingOverrideTest.java create mode 100644 smoke-tests/apps/SamplingOverridesV2/src/smokeTest/resources/applicationinsights.json create mode 100644 smoke-tests/apps/SamplingOverridesV2/src/smokeTest/resources/applicationinsights2.json create mode 100644 smoke-tests/apps/SamplingOverridesV2/src/smokeTest/resources/logback-test.xml create mode 100644 smoke-tests/apps/SpringScheduling/src/smokeTest/resources/applicationinsights.json create mode 100644 smoke-tests/framework/src/main/resources/default_applicationinsights.json diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java index c555e62705e..bc4be5ca35d 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java @@ -74,7 +74,7 @@ public void validate() { preview.validate(); } - // TODO (trask) investigate options for mapping lowercase values to otel enum directly + @Deprecated public enum SpanKind { @JsonProperty("server") SERVER(io.opentelemetry.api.trace.SpanKind.SERVER), @@ -94,6 +94,17 @@ public enum SpanKind { } } + public enum TelemetryKind { + @JsonProperty("request") + REQUEST, + @JsonProperty("dependency") + DEPENDENCY, + @JsonProperty("trace") + TRACE, + @JsonProperty("exception") + EXCEPTION + } + public enum MatchType { @JsonProperty("strict") STRICT, @@ -151,7 +162,12 @@ public static class Role { public static class Sampling { - public float percentage = 100; + // fixed percentage of requests + @Nullable public Double percentage; + + // default is 5 requests per second (set in ConfigurationBuilder if neither percentage nor + // limitPerSecond was configured) + @Nullable public Double limitPerSecond; } public static class SamplingPreview { @@ -175,6 +191,8 @@ public static class SamplingPreview { // Another (lesser) reason is because .NET SDK always propagates trace flags "00" (not // sampled) // + // future goal: make parentBased sampling the default if item count is received via tracestate + // // IMPORTANT if changing this default, we need to keep it at least on Azure Functions public boolean parentBased; @@ -352,7 +370,7 @@ public static class PreviewConfiguration { new HashSet<>(asList("b3", "b3multi")); public void validate() { - for (Configuration.SamplingOverride samplingOverride : sampling.overrides) { + for (SamplingOverride samplingOverride : sampling.overrides) { samplingOverride.validate(); } for (Configuration.InstrumentationKeyOverride instrumentationKeyOverride : @@ -578,22 +596,35 @@ private static boolean isRuntimeAttached() { } public static class SamplingOverride { - // TODO (trask) consider making this required when moving out of preview - @Nullable public SpanKind spanKind; + @Deprecated @Nullable public SpanKind spanKind; + + // TODO (trask) make this required when moving out of preview + // for now the default is both "request" and "dependency" for backwards compatibility + @Nullable public TelemetryKind telemetryKind; + + // TODO (trask) should this be named "standalone" (and meaning flipped) + // especially if we aren't going to change meaning of "request" to include unparented INTERNAL + // spans + public boolean inRequest = true; + // not using include/exclude, because you can still get exclude with this by adding a second // (exclude) override above it // (since only the first matching override is used) public List attributes = new ArrayList<>(); - public Float percentage; + public Double percentage; public String id; // optional, used for debugging purposes only + public boolean isForRequestTelemetry() { + return telemetryKind == TelemetryKind.REQUEST + || (telemetryKind == null && spanKind != SpanKind.CLIENT); + } + + public boolean isForDependencyTelemetry() { + return telemetryKind == TelemetryKind.DEPENDENCY + || (telemetryKind == null && spanKind != SpanKind.SERVER); + } + public void validate() { - if (spanKind == null && attributes.isEmpty()) { - // TODO add doc and go link, similar to telemetry processors - throw new FriendlyException( - "A sampling override configuration is missing \"spanKind\" and has no attributes.", - "Please provide at least one of \"spanKind\" or \"attributes\" for the sampling override configuration."); - } if (percentage == null) { // TODO add doc and go link, similar to telemetry processors throw new FriendlyException( diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/ConfigurationBuilder.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/ConfigurationBuilder.java index 605e6817b2c..ceb332815ca 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/ConfigurationBuilder.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/ConfigurationBuilder.java @@ -83,6 +83,9 @@ public class ConfigurationBuilder { private static final String APPLICATIONINSIGHTS_SAMPLING_PERCENTAGE = "APPLICATIONINSIGHTS_SAMPLING_PERCENTAGE"; + private static final String APPLICATIONINSIGHTS_SAMPLING_LIMIT_PER_SECOND = + "APPLICATIONINSIGHTS_SAMPLING_LIMIT_PER_SECOND"; + private static final String APPLICATIONINSIGHTS_INSTRUMENTATION_LOGGING_LEVEL = "APPLICATIONINSIGHTS_INSTRUMENTATION_LOGGING_LEVEL"; @@ -181,6 +184,15 @@ public static Configuration create(Path agentJarPath, @Nullable RpConfiguration + " and it is now enabled by default," + " so no need to enable it under preview configuration"); } + for (SamplingOverride override : config.preview.sampling.overrides) { + if (override.spanKind != null) { + configurationLogger.warn( + "Sampling overrides \"spanKind\" has been deprecated," + + " and support for it will be removed in a future release, please transition from" + + " \"spanKind\" to \"telemetryKind\"."); + } + } + logWarningIfUsingInternalAttributes(config); overlayFromEnv(config, agentJarPath.getParent()); @@ -194,6 +206,10 @@ public static Configuration create(Path agentJarPath, @Nullable RpConfiguration overlayFromEnv(rpConfiguration); overlayRpConfiguration(config, rpConfiguration); } + // only fall back to default sampling configuration after all overlays have been performed + if (config.sampling.limitPerSecond == null && config.sampling.percentage == null) { + config.sampling.limitPerSecond = 5.0; + } // only set role instance to host name as a last resort if (config.role.instance == null) { String hostname = HostName.get(); @@ -457,6 +473,10 @@ static void overlayFromEnv(Configuration config, Path baseDir) throws IOExceptio config.sampling.percentage = overlayWithEnvVar(APPLICATIONINSIGHTS_SAMPLING_PERCENTAGE, config.sampling.percentage); + config.sampling.limitPerSecond = + overlayWithEnvVar( + APPLICATIONINSIGHTS_SAMPLING_LIMIT_PER_SECOND, config.sampling.limitPerSecond); + config.proxy = overlayProxyFromEnv(config.proxy); config.selfDiagnostics.level = @@ -466,10 +486,9 @@ static void overlayFromEnv(Configuration config, Path baseDir) throws IOExceptio APPLICATIONINSIGHTS_SELF_DIAGNOSTICS_FILE_PATH, config.selfDiagnostics.file.path); config.preview.metricIntervalSeconds = - (int) - overlayWithEnvVar( - APPLICATIONINSIGHTS_PREVIEW_METRIC_INTERVAL_SECONDS, - config.preview.metricIntervalSeconds); + overlayWithEnvVar( + APPLICATIONINSIGHTS_PREVIEW_METRIC_INTERVAL_SECONDS, + config.preview.metricIntervalSeconds); config.preview.instrumentation.springIntegration.enabled = overlayWithEnvVar( @@ -576,6 +595,7 @@ static void overlayRpConfiguration(Configuration config, RpConfiguration rpConfi } if (rpConfiguration.sampling != null) { config.sampling.percentage = rpConfiguration.sampling.percentage; + config.sampling.limitPerSecond = rpConfiguration.sampling.limitPerSecond; } if (isTrimEmpty(config.role.name)) { // only use rp configuration role name as a fallback, similar to WEBSITE_SITE_NAME @@ -626,13 +646,25 @@ public static String overlayWithEnvVar(String name, String defaultValue) { return defaultValue; } - static float overlayWithEnvVar(String name, float defaultValue) { + @Nullable + static Double overlayWithEnvVar(String name, @Nullable Double defaultValue) { + String value = getEnvVar(name); + if (value != null) { + configurationLogger.debug("using environment variable: {}", name); + // intentionally allowing NumberFormatException to bubble up as invalid configuration and + // prevent agent from starting + return Double.parseDouble(value); + } + return defaultValue; + } + + static int overlayWithEnvVar(String name, int defaultValue) { String value = getEnvVar(name); if (value != null) { configurationLogger.debug("using environment variable: {}", name); // intentionally allowing NumberFormatException to bubble up as invalid configuration and // prevent agent from starting - return Float.parseFloat(value); + return Integer.parseInt(value); } return defaultValue; } @@ -798,17 +830,21 @@ public static Configuration loadJsonConfigFile(Path configPath) throws IOExcepti } // this is for external callers, where logging is ok - public static float roundToNearest(float samplingPercentage) { + public static double roundToNearest(double samplingPercentage) { return roundToNearest(samplingPercentage, false); } - // visible for testing - private static float roundToNearest(float samplingPercentage, boolean doNotLogWarnMessages) { + @Nullable + private static Double roundToNearest( + @Nullable Double samplingPercentage, boolean doNotLogWarnMessages) { + if (samplingPercentage == null) { + return null; + } if (samplingPercentage == 0) { - return 0; + return 0.0; } double itemCount = 100 / samplingPercentage; - float rounded = 100.0f / Math.round(itemCount); + double rounded = 100.0 / Math.round(itemCount); if (Math.abs(samplingPercentage - rounded) >= 1) { // TODO include link to docs in this warning message diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/AgentLogExporter.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/AgentLogExporter.java index d4d90d4511c..1372e4ea55a 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/AgentLogExporter.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/AgentLogExporter.java @@ -28,15 +28,22 @@ import com.azure.monitor.opentelemetry.exporter.implementation.logging.OperationLogger; import com.azure.monitor.opentelemetry.exporter.implementation.models.TelemetryItem; import com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.QuickPulse; +import com.microsoft.applicationinsights.agent.internal.configuration.Configuration.SamplingOverride; +import com.microsoft.applicationinsights.agent.internal.sampling.AiSampler; +import com.microsoft.applicationinsights.agent.internal.sampling.SamplingOverrides; import com.microsoft.applicationinsights.agent.internal.telemetry.BatchItemProcessor; import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryClient; import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryObservers; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import io.opentelemetry.api.trace.SpanContext; import io.opentelemetry.sdk.common.CompletableResultCode; import io.opentelemetry.sdk.logs.data.LogData; import io.opentelemetry.sdk.logs.data.Severity; import io.opentelemetry.sdk.logs.export.LogExporter; +import io.opentelemetry.semconv.trace.attributes.SemanticAttributes; import java.util.Collection; +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; import java.util.function.Consumer; import javax.annotation.Nullable; import org.slf4j.Logger; @@ -52,15 +59,21 @@ public class AgentLogExporter implements LogExporter { // TODO (trask) could implement this in a filtering LogExporter instead private volatile Severity threshold; + private final SamplingOverrides logSamplingOverrides; + private final SamplingOverrides exceptionSamplingOverrides; private final LogDataMapper mapper; private final Consumer telemetryItemConsumer; public AgentLogExporter( Severity threshold, + List logSamplingOverrides, + List exceptionSamplingOverrides, LogDataMapper mapper, @Nullable QuickPulse quickPulse, BatchItemProcessor batchItemProcessor) { this.threshold = threshold; + this.logSamplingOverrides = new SamplingOverrides(logSamplingOverrides); + this.exceptionSamplingOverrides = new SamplingOverrides(exceptionSamplingOverrides); this.mapper = mapper; telemetryItemConsumer = telemetryItem -> { @@ -86,10 +99,6 @@ public CompletableResultCode export(Collection logs) { return CompletableResultCode.ofFailure(); } for (LogData log : logs) { - SpanContext spanContext = log.getSpanContext(); - if (spanContext.isValid() && !spanContext.getTraceFlags().isSampled()) { - continue; - } logger.debug("exporting log: {}", log); try { int severity = log.getSeverity().getSeverityNumber(); @@ -97,7 +106,39 @@ public CompletableResultCode export(Collection logs) { if (severity < threshold) { continue; } - mapper.map(log, telemetryItemConsumer); + + String stack = log.getAttributes().get(SemanticAttributes.EXCEPTION_STACKTRACE); + + SamplingOverrides samplingOverrides = + stack != null ? exceptionSamplingOverrides : logSamplingOverrides; + + SpanContext spanContext = log.getSpanContext(); + + boolean inRequest = spanContext.isValid(); + Double samplingPercentage = + samplingOverrides.getOverridePercentage(inRequest, log.getAttributes()); + + if (samplingPercentage != null && !shouldSample(spanContext, samplingPercentage)) { + continue; + } + + if (samplingPercentage == null + && spanContext.isValid() + && !spanContext.getTraceFlags().isSampled()) { + // if there is no sampling override, and the log is part of an unsampled trace, then don't + // capture it + continue; + } + + Long itemCount = null; + if (samplingPercentage != null) { + // samplingPercentage cannot be 0 here + itemCount = Math.round(100.0 / samplingPercentage); + } + + TelemetryItem telemetryItem = mapper.map(log, stack, itemCount); + telemetryItemConsumer.accept(telemetryItem); + exportingLogLogger.recordSuccess(); } catch (Throwable t) { exportingLogLogger.recordFailure(t.getMessage(), t, EXPORTER_MAPPING_ERROR); @@ -116,4 +157,22 @@ public CompletableResultCode flush() { public CompletableResultCode shutdown() { return CompletableResultCode.ofSuccess(); } + + @SuppressFBWarnings( + value = "SECPR", // Predictable pseudorandom number generator + justification = "Predictable random is ok for sampling decision") + private static boolean shouldSample(SpanContext spanContext, double percentage) { + if (percentage == 100) { + // optimization, no need to calculate score + return true; + } + if (percentage == 0) { + // optimization, no need to calculate score + return false; + } + if (spanContext.isValid()) { + return AiSampler.shouldRecordAndSample(spanContext.getTraceId(), percentage); + } + return ThreadLocalRandom.current().nextDouble() < percentage / 100; + } } diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/RpConfigurationPolling.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/RpConfigurationPolling.java index 3ec58d3f684..10662113733 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/RpConfigurationPolling.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/RpConfigurationPolling.java @@ -36,6 +36,7 @@ import java.nio.file.Files; import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.FileTime; +import java.util.Objects; import java.util.concurrent.Executors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -100,24 +101,28 @@ public void run() { if (!newRpConfiguration.connectionString.equals(rpConfiguration.connectionString)) { logger.debug( "Connection string from the JSON config file is overriding the previously configured connection string."); + configuration.connectionString = newRpConfiguration.connectionString; telemetryClient.updateConnectionStrings( - newRpConfiguration.connectionString, + configuration.connectionString, configuration.internal.statsbeat.instrumentationKey, configuration.internal.statsbeat.endpoint); appIdSupplier.updateAppId(); } - if (newRpConfiguration.sampling.percentage != rpConfiguration.sampling.percentage) { - logger.debug( - "Updating sampling percentage from {} to {}", - rpConfiguration.sampling.percentage, - newRpConfiguration.sampling.percentage); - float roundedSamplingPercentage = - ConfigurationBuilder.roundToNearest(newRpConfiguration.sampling.percentage); - DelegatingSampler.getInstance() - .setDelegate(Samplers.getSampler(roundedSamplingPercentage, configuration)); - BytecodeUtilImpl.samplingPercentage = roundedSamplingPercentage; - rpConfiguration.sampling.percentage = newRpConfiguration.sampling.percentage; + if (!Objects.equals( + newRpConfiguration.sampling.percentage, rpConfiguration.sampling.percentage) + || !Objects.equals( + newRpConfiguration.sampling.limitPerSecond, + rpConfiguration.sampling.limitPerSecond)) { + logger.debug("Updating sampling percentage"); + configuration.sampling.percentage = newRpConfiguration.sampling.percentage; + configuration.sampling.limitPerSecond = newRpConfiguration.sampling.limitPerSecond; + DelegatingSampler.getInstance().setDelegate(Samplers.getSampler(configuration)); + if (configuration.sampling.percentage != null) { + BytecodeUtilImpl.samplingPercentage = configuration.sampling.percentage.floatValue(); + } else { + BytecodeUtilImpl.samplingPercentage = 100; + } } rpConfiguration = newRpConfiguration; } diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java index 39ddee83d23..23796e5cdb2 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java @@ -41,6 +41,7 @@ import com.microsoft.applicationinsights.agent.internal.common.FriendlyException; import com.microsoft.applicationinsights.agent.internal.configuration.Configuration; import com.microsoft.applicationinsights.agent.internal.configuration.Configuration.ProcessorConfig; +import com.microsoft.applicationinsights.agent.internal.configuration.Configuration.TelemetryKind; import com.microsoft.applicationinsights.agent.internal.configuration.RpConfiguration; import com.microsoft.applicationinsights.agent.internal.exporter.AgentLogExporter; import com.microsoft.applicationinsights.agent.internal.exporter.AgentMetricExporter; @@ -172,7 +173,11 @@ public void customize(AutoConfigurationCustomizer autoConfiguration) { TelemetryClient.setActive(telemetryClient); - BytecodeUtilImpl.samplingPercentage = configuration.sampling.percentage; + if (configuration.sampling.percentage != null) { + BytecodeUtilImpl.samplingPercentage = configuration.sampling.percentage.floatValue(); + } else { + BytecodeUtilImpl.samplingPercentage = 100; + } BytecodeUtilImpl.featureStatsbeat = statsbeatModule.getFeatureStatsbeat(); AppIdSupplier appIdSupplier = new AppIdSupplier(telemetryClient); @@ -309,8 +314,7 @@ private static SdkTracerProviderBuilder configureTracing( configuration.preview.additionalPropagators, configuration.preview.legacyRequestIdPropagation.enabled); } - DelegatingSampler.getInstance() - .setDelegate(Samplers.getSampler(configuration.sampling.percentage, configuration)); + DelegatingSampler.getInstance().setDelegate(Samplers.getSampler(configuration)); } else { // in Azure Functions, we configure later on, once we know user has opted in to tracing // (note: the default for DelegatingPropagator is to not propagate anything @@ -484,9 +488,20 @@ private static LogExporter createLogExporter( configuration.preview.captureLoggingLevelAsCustomDimension, telemetryClient::populateDefaults); + List logSamplingOverrides = + configuration.preview.sampling.overrides.stream() + .filter(override -> override.telemetryKind == TelemetryKind.TRACE) + .collect(Collectors.toList()); + List exceptionSamplingOverrides = + configuration.preview.sampling.overrides.stream() + .filter(override -> override.telemetryKind == TelemetryKind.EXCEPTION) + .collect(Collectors.toList()); + agentLogExporter = new AgentLogExporter( configuration.instrumentation.logging.getSeverity(), + logSamplingOverrides, + exceptionSamplingOverrides, mapper, quickPulse, telemetryClient.getGeneralBatchItemProcessor()); diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/AiOverrideSampler.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/AiOverrideSampler.java index 240bc6f4b70..7904ded84ec 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/AiOverrideSampler.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/AiOverrideSampler.java @@ -21,7 +21,11 @@ package com.microsoft.applicationinsights.agent.internal.sampling; +import com.azure.monitor.opentelemetry.exporter.implementation.SpanDataMapper; +import com.microsoft.applicationinsights.agent.internal.configuration.Configuration.SamplingOverride; import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.context.Context; import io.opentelemetry.sdk.trace.data.LinkData; @@ -31,11 +35,16 @@ class AiOverrideSampler implements Sampler { - private final SamplingOverrides samplingOverrides; + private final SamplingOverrides requestSamplingOverrides; + private final SamplingOverrides dependencySamplingOverrides; private final Sampler delegate; - AiOverrideSampler(SamplingOverrides samplingOverrides, Sampler delegate) { - this.samplingOverrides = samplingOverrides; + AiOverrideSampler( + List requestSamplingOverrides, + List dependencySamplingOverrides, + Sampler delegate) { + this.requestSamplingOverrides = new SamplingOverrides(requestSamplingOverrides); + this.dependencySamplingOverrides = new SamplingOverrides(dependencySamplingOverrides); this.delegate = delegate; } @@ -48,7 +57,18 @@ public SamplingResult shouldSample( Attributes attributes, List parentLinks) { - Sampler override = samplingOverrides.getOverride(spanKind, attributes); + SpanContext parentSpanContext = Span.fromContext(parentContext).getSpanContext(); + + // TODO isRequest won't be correct for spring-scheduling, quartz and customInstrumentation + // since we don't have scope available in the sampler (yet) + boolean isRequest = + SpanDataMapper.isRequest(spanKind, parentSpanContext, null, attributes::get); + + SamplingOverrides samplingOverrides = + isRequest ? requestSamplingOverrides : dependencySamplingOverrides; + + boolean inRequest = isRequest || parentSpanContext.isValid(); + Sampler override = samplingOverrides.getOverride(inRequest, attributes); if (override != null) { return override.shouldSample(parentContext, traceId, name, spanKind, attributes, parentLinks); } diff --git a/agent/azure-monitor-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/AzureMonitorSampler.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/AiSampler.java similarity index 79% rename from agent/azure-monitor-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/AzureMonitorSampler.java rename to agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/AiSampler.java index d50c50b8227..1614471ef40 100644 --- a/agent/azure-monitor-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/AzureMonitorSampler.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/AiSampler.java @@ -19,7 +19,7 @@ * DEALINGS IN THE SOFTWARE. */ -package com.azure.monitor.opentelemetry.exporter; +package com.microsoft.applicationinsights.agent.internal.sampling; import com.azure.monitor.opentelemetry.exporter.implementation.AiSemanticAttributes; import com.azure.monitor.opentelemetry.exporter.implementation.SamplingScoreGeneratorV2; @@ -28,6 +28,7 @@ import io.opentelemetry.api.trace.SpanContext; import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.context.Context; +import io.opentelemetry.instrumentation.api.internal.cache.Cache; import io.opentelemetry.sdk.trace.ReadableSpan; import io.opentelemetry.sdk.trace.data.LinkData; import io.opentelemetry.sdk.trace.samplers.Sampler; @@ -41,27 +42,19 @@ // nodes when some of those nodes are being monitored by other Application Insights SDKs (and 2.x // Java SDK) // * adds item count to span attribute if it is sampled -public class AzureMonitorSampler implements Sampler { +public class AiSampler implements Sampler { private final boolean localParentBased; - private final float samplingPercentage; - private final SamplingResult recordAndSampleWithItemCount; + private final SamplingPercentage samplingPercentage; + private final Cache recordAndSampleWithItemCountMap = Cache.bounded(100); - public AzureMonitorSampler(float samplingPercentage) { + public AiSampler(SamplingPercentage samplingPercentage) { this(samplingPercentage, true); } - public AzureMonitorSampler(float samplingPercentage, boolean localParentBased) { - int itemCount; - if (samplingPercentage != 0) { - itemCount = Math.round(100.0f / samplingPercentage); - this.samplingPercentage = 100.0f / itemCount; - } else { - itemCount = 1; - this.samplingPercentage = 0; - } + public AiSampler(SamplingPercentage samplingPercentage, boolean localParentBased) { + this.samplingPercentage = samplingPercentage; this.localParentBased = localParentBased; - recordAndSampleWithItemCount = new RecordAndSampleWithItemCount(itemCount); } @Override @@ -80,11 +73,24 @@ public SamplingResult shouldSample( } } - if (shouldRecordAndSample(traceId, samplingPercentage)) { - return recordAndSampleWithItemCount; - } else { + double sp = samplingPercentage.get(); + + if (sp == 0) { return SamplingResult.drop(); } + + if (!shouldRecordAndSample(traceId, sp)) { + return SamplingResult.drop(); + } + + // sp cannot be 0 here + long itemCount = Math.round(100.0 / sp); + SamplingResult samplingResult = recordAndSampleWithItemCountMap.get(itemCount); + if (samplingResult == null) { + samplingResult = new RecordAndSampleWithItemCount(itemCount); + recordAndSampleWithItemCountMap.put(itemCount, samplingResult); + } + return samplingResult; } @Nullable @@ -109,7 +115,7 @@ private static SamplingResult handleLocalParent(Context parentContext) { return null; } - private static boolean shouldRecordAndSample(String traceId, float percentage) { + public static boolean shouldRecordAndSample(String traceId, double percentage) { if (percentage == 100) { // optimization, no need to calculate score return true; @@ -123,7 +129,7 @@ private static boolean shouldRecordAndSample(String traceId, float percentage) { @Override public String getDescription() { - return String.format("AzureMonitorSampler{%.3f%%}", samplingPercentage); + return "AiSampler"; } private static class RecordAndSampleWithItemCount implements SamplingResult { diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/RateLimitedSamplingPercentage.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/RateLimitedSamplingPercentage.java new file mode 100644 index 00000000000..b15ee13f744 --- /dev/null +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/RateLimitedSamplingPercentage.java @@ -0,0 +1,112 @@ +/* + * ApplicationInsights-Java + * Copyright (c) Microsoft Corporation + * All rights reserved. + * + * MIT License + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the ""Software""), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the following conditions: + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE + * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +// Includes work from: +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.microsoft.applicationinsights.agent.internal.sampling; + +import static java.util.Objects.requireNonNull; + +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.LongSupplier; + +// uses adaptive algorithm from OpenTelemetry Java Contrib's ConsistentRateLimitingSampler +class RateLimitedSamplingPercentage implements SamplingPercentage { + + private static final class State { + private final double effectiveWindowCount; + private final double effectiveWindowNanos; + private final long lastNanoTime; + + public State(double effectiveWindowCount, double effectiveWindowNanos, long lastNanoTime) { + this.effectiveWindowCount = effectiveWindowCount; + this.effectiveWindowNanos = effectiveWindowNanos; + this.lastNanoTime = lastNanoTime; + } + } + + private final LongSupplier nanoTimeSupplier; + private final double inverseAdaptationTimeNanos; + private final double targetSpansPerNanosecondLimit; + private final AtomicReference state; + + RateLimitedSamplingPercentage(double targetSpansPerSecondLimit, double adaptationTimeSeconds) { + this(targetSpansPerSecondLimit, adaptationTimeSeconds, System::nanoTime); + } + + private RateLimitedSamplingPercentage( + double targetSpansPerSecondLimit, + double adaptationTimeSeconds, + LongSupplier nanoTimeSupplier) { + + if (targetSpansPerSecondLimit < 0.0) { + throw new IllegalArgumentException("Limit for sampled spans per second must be nonnegative!"); + } + if (adaptationTimeSeconds < 0.0) { + throw new IllegalArgumentException("Adaptation rate must be nonnegative!"); + } + this.nanoTimeSupplier = requireNonNull(nanoTimeSupplier); + + this.inverseAdaptationTimeNanos = 1e-9 / adaptationTimeSeconds; + this.targetSpansPerNanosecondLimit = 1e-9 * targetSpansPerSecondLimit; + + this.state = new AtomicReference<>(new State(0, 0, nanoTimeSupplier.getAsLong())); + } + + private State updateState(State oldState, long currentNanoTime) { + if (currentNanoTime <= oldState.lastNanoTime) { + return new State( + oldState.effectiveWindowCount + 1, oldState.effectiveWindowNanos, oldState.lastNanoTime); + } + long nanoTimeDelta = currentNanoTime - oldState.lastNanoTime; + double decayFactor = Math.exp(-nanoTimeDelta * inverseAdaptationTimeNanos); + double currentEffectiveWindowCount = oldState.effectiveWindowCount * decayFactor + 1; + double currentEffectiveWindowNanos = + oldState.effectiveWindowNanos * decayFactor + nanoTimeDelta; + return new State(currentEffectiveWindowCount, currentEffectiveWindowNanos, currentNanoTime); + } + + @Override + public double get() { + long currentNanoTime = nanoTimeSupplier.getAsLong(); + State currentState = state.updateAndGet(s -> updateState(s, currentNanoTime)); + + double samplingProbability = + (currentState.effectiveWindowNanos * targetSpansPerNanosecondLimit) + / currentState.effectiveWindowCount; + + double samplingPercentage = 100 * Math.min(samplingProbability, 1); + + return roundDownToNearest(samplingPercentage); + } + + private static double roundDownToNearest(double samplingPercentage) { + if (samplingPercentage == 0) { + return 0; + } + double itemCount = 100 / samplingPercentage; + return 100.0 / Math.ceil(itemCount); + } +} diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/Samplers.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/Samplers.java index cb9cd262d11..5dfd15c9d62 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/Samplers.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/Samplers.java @@ -21,21 +21,43 @@ package com.microsoft.applicationinsights.agent.internal.sampling; -import com.azure.monitor.opentelemetry.exporter.AzureMonitorSampler; import com.microsoft.applicationinsights.agent.internal.configuration.Configuration; +import com.microsoft.applicationinsights.agent.internal.configuration.Configuration.SamplingOverride; import io.opentelemetry.sdk.trace.samplers.Sampler; +import java.util.List; +import java.util.stream.Collectors; public class Samplers { - public static Sampler getSampler(float samplingPercentage, Configuration config) { - Sampler sampler = new AzureMonitorSampler(samplingPercentage); + public static Sampler getSampler(Configuration config) { + SamplingPercentage samplingPercentage; + if (config.sampling.limitPerSecond != null) { + samplingPercentage = SamplingPercentage.rateLimited(config.sampling.limitPerSecond); + } else if (config.sampling.percentage != null) { + samplingPercentage = SamplingPercentage.fixed(config.sampling.percentage); + } else { + throw new AssertionError("ConfigurationBuilder should have set the default sampling"); + } + + Sampler sampler = new AiSampler(samplingPercentage); + + Configuration.SamplingPreview sampling = config.preview.sampling; + + List requestSamplingOverrides = + config.preview.sampling.overrides.stream() + .filter(SamplingOverride::isForRequestTelemetry) + .collect(Collectors.toList()); + List dependencySamplingOverrides = + config.preview.sampling.overrides.stream() + .filter(SamplingOverride::isForDependencyTelemetry) + .collect(Collectors.toList()); - if (!config.preview.sampling.overrides.isEmpty()) { + if (!requestSamplingOverrides.isEmpty() || !dependencySamplingOverrides.isEmpty()) { sampler = - new AiOverrideSampler(new SamplingOverrides(config.preview.sampling.overrides), sampler); + new AiOverrideSampler(requestSamplingOverrides, dependencySamplingOverrides, sampler); } - if (!config.preview.sampling.parentBased) { + if (!sampling.parentBased) { return sampler; } diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverrides.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverrides.java index cb79ed29718..4966ab2de38 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverrides.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverrides.java @@ -21,14 +21,12 @@ package com.microsoft.applicationinsights.agent.internal.sampling; -import com.azure.monitor.opentelemetry.exporter.AzureMonitorSampler; import com.azure.monitor.opentelemetry.exporter.implementation.SpanDataMapper; import com.microsoft.applicationinsights.agent.internal.configuration.Configuration.MatchType; import com.microsoft.applicationinsights.agent.internal.configuration.Configuration.SamplingOverride; import com.microsoft.applicationinsights.agent.internal.configuration.Configuration.SamplingOverrideAttribute; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.sdk.trace.samplers.Sampler; import io.opentelemetry.semconv.trace.attributes.SemanticAttributes; import java.util.ArrayList; @@ -37,11 +35,11 @@ import javax.annotation.Nullable; // TODO find a better name for this class (and MatcherGroup too) -class SamplingOverrides { +public class SamplingOverrides { private final List matcherGroups; - SamplingOverrides(List overrides) { + public SamplingOverrides(List overrides) { matcherGroups = new ArrayList<>(); for (SamplingOverride override : overrides) { matcherGroups.add(new MatcherGroup(override)); @@ -49,36 +47,54 @@ class SamplingOverrides { } @Nullable - Sampler getOverride(SpanKind spanKind, Attributes attributes) { + public Sampler getOverride(boolean inRequest, Attributes attributes) { LazyHttpUrl lazyHttpUrl = new LazyHttpUrl(attributes); for (MatcherGroup matcherGroups : matcherGroups) { - if (matcherGroups.matches(spanKind, attributes, lazyHttpUrl)) { + if (matcherGroups.matches(inRequest, attributes, lazyHttpUrl)) { return matcherGroups.getSampler(); } } return null; } + // used to do sampling inside the log exporter + @Nullable + public Double getOverridePercentage(boolean inRequest, Attributes attributes) { + for (MatcherGroup matcherGroups : matcherGroups) { + if (matcherGroups.matches(inRequest, attributes, null)) { + return matcherGroups.getPercentage(); + } + } + return null; + } + private static class MatcherGroup { - @Nullable private final SpanKind spanKind; + private final boolean inRequest; private final List predicates; private final Sampler sampler; + private final SamplingPercentage samplingPercentage; private MatcherGroup(SamplingOverride override) { - spanKind = override.spanKind != null ? override.spanKind.otelSpanKind : null; + inRequest = override.inRequest; predicates = new ArrayList<>(); for (SamplingOverrideAttribute attribute : override.attributes) { predicates.add(toPredicate(attribute)); } - sampler = new AzureMonitorSampler(override.percentage, false); + samplingPercentage = SamplingPercentage.fixed(override.percentage); + sampler = new AiSampler(samplingPercentage, false); } Sampler getSampler() { return sampler; } - private boolean matches(SpanKind spanKind, Attributes attributes, LazyHttpUrl lazyHttpUrl) { - if (this.spanKind != null && !this.spanKind.equals(spanKind)) { + double getPercentage() { + return samplingPercentage.get(); + } + + private boolean matches( + boolean inRequest, Attributes attributes, @Nullable LazyHttpUrl lazyHttpUrl) { + if (this.inRequest != inRequest) { return false; } for (TempPredicate predicate : predicates) { @@ -161,9 +177,11 @@ private RegexpMatcher(String key, String value) { } @Override - public boolean test(Attributes attributes, LazyHttpUrl lazyHttpUrl) { + public boolean test(Attributes attributes, @Nullable LazyHttpUrl lazyHttpUrl) { String val = attributes.get(key); - if (val == null && key.getKey().equals(SemanticAttributes.HTTP_URL.getKey())) { + if (val == null + && key.getKey().equals(SemanticAttributes.HTTP_URL.getKey()) + && lazyHttpUrl != null) { val = lazyHttpUrl.get(); } return val != null && value.matcher(val).matches(); @@ -180,7 +198,7 @@ private RegexpArrayContainsMatcher(String key, String value) { } @Override - public boolean test(Attributes attributes, LazyHttpUrl lazyHttpUrl) { + public boolean test(Attributes attributes, @Nullable LazyHttpUrl lazyHttpUrl) { List val = attributes.get(key); if (val == null) { return false; @@ -202,9 +220,11 @@ private KeyOnlyMatcher(String key) { } @Override - public boolean test(Attributes attributes, LazyHttpUrl lazyHttpUrl) { + public boolean test(Attributes attributes, @Nullable LazyHttpUrl lazyHttpUrl) { String val = attributes.get(key); - if (val == null && key.getKey().equals(SemanticAttributes.HTTP_URL.getKey())) { + if (val == null + && key.getKey().equals(SemanticAttributes.HTTP_URL.getKey()) + && lazyHttpUrl != null) { val = lazyHttpUrl.get(); } return val != null; @@ -233,6 +253,6 @@ private String get() { // this is temporary until semantic attributes stabilize and we make breaking change // then can use java.util.functions.Predicate private interface TempPredicate { - boolean test(Attributes attributes, LazyHttpUrl lazyHttpUrl); + boolean test(Attributes attributes, @Nullable LazyHttpUrl lazyHttpUrl); } } diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingPercentage.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingPercentage.java new file mode 100644 index 00000000000..c65e0b5b939 --- /dev/null +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingPercentage.java @@ -0,0 +1,43 @@ +/* + * ApplicationInsights-Java + * Copyright (c) Microsoft Corporation + * All rights reserved. + * + * MIT License + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the ""Software""), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the following conditions: + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE + * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +package com.microsoft.applicationinsights.agent.internal.sampling; + +import com.microsoft.applicationinsights.agent.internal.configuration.ConfigurationBuilder; + +// all sampling percentage must be in a ratio of 100/N where N is a whole number (2, 3, 4, ...) +// e.g. 50 for 1/2 or 33.33 for 1/3 +// +// failure to follow this pattern can result in unexpected / incorrect computation of values in +// the portal +public interface SamplingPercentage { + + double get(); + + static SamplingPercentage fixed(double percentage) { + double roundedPercentage = ConfigurationBuilder.roundToNearest(percentage); + return () -> roundedPercentage; + } + + static SamplingPercentage rateLimited(double targetPerSecondLimit) { + return new RateLimitedSamplingPercentage(targetPerSecondLimit, 0.1); + } +} diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/SamplingTestUtil.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/SamplingTestUtil.java new file mode 100644 index 00000000000..10b3bb8b63d --- /dev/null +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/SamplingTestUtil.java @@ -0,0 +1,61 @@ +/* + * ApplicationInsights-Java + * Copyright (c) Microsoft Corporation + * All rights reserved. + * + * MIT License + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the ""Software""), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the following conditions: + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE + * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +package com.microsoft.applicationinsights.agent.internal; + +import com.azure.monitor.opentelemetry.exporter.implementation.AiSemanticAttributes; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.TraceFlags; +import io.opentelemetry.api.trace.TraceState; +import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import io.opentelemetry.sdk.trace.samplers.SamplingResult; +import java.util.Collections; + +public class SamplingTestUtil { + + public static double getCurrentSamplingPercentage(Sampler sampler) { + SpanContext spanContext = + SpanContext.createFromRemoteParent( + "12341234123412341234123412341234", + "1234123412341234", + TraceFlags.getSampled(), + TraceState.getDefault()); + Context parentContext = Context.root().with(Span.wrap(spanContext)); + SamplingResult samplingResult = + sampler.shouldSample( + parentContext, + // traceId=27272727272727272727272727272727 is known to produce a score of 0.66 (out + // of 100) so will be sampled as long as samplingPercentage > 1% + "27272727272727272727272727272727", + "my span name", + SpanKind.SERVER, + Attributes.empty(), + Collections.emptyList()); + Long itemCount = samplingResult.getAttributes().get(AiSemanticAttributes.ITEM_COUNT); + return 100.0 / itemCount; + } + + private SamplingTestUtil() {} +} diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/configuration/ConfigurationBuilderTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/configuration/ConfigurationBuilderTest.java index 7e40c27fa92..8f930e406b2 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/configuration/ConfigurationBuilderTest.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/configuration/ConfigurationBuilderTest.java @@ -129,7 +129,7 @@ void testGetJsonEncodingExceptionMessage() { @Test void testRpConfigurationOverlayWithEnvVarAndSysPropUnchanged() { String testConnectionString = "test-connection-string"; - float testSamplingPercentage = 10.0f; + double testSamplingPercentage = 10.0; RpConfiguration config = new RpConfiguration(); config.connectionString = testConnectionString; @@ -144,7 +144,7 @@ void testRpConfigurationOverlayWithEnvVarAndSysPropUnchanged() { @Test void testRpConfigurationOverlayWithEnvVarAndSysPropPopulated() throws Exception { String testConnectionString = "test-connection-string"; - float testSamplingPercentage = 10.0f; + double testSamplingPercentage = 10.0; withEnvironmentVariable("APPLICATIONINSIGHTS_CONNECTION_STRING", testConnectionString) .and("APPLICATIONINSIGHTS_SAMPLING_PERCENTAGE", String.valueOf(testSamplingPercentage)) @@ -153,7 +153,7 @@ void testRpConfigurationOverlayWithEnvVarAndSysPropPopulated() throws Exception RpConfiguration config = new RpConfiguration(); config.connectionString = String.format("original-%s", testConnectionString); - config.sampling.percentage = testSamplingPercentage + 1.0f; + config.sampling.percentage = testSamplingPercentage + 1.0; ConfigurationBuilder.overlayFromEnv(config); diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/configuration/SamplingPercentageTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/configuration/SamplingPercentageTest.java index 41513481041..7e7b7026fd0 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/configuration/SamplingPercentageTest.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/configuration/SamplingPercentageTest.java @@ -37,16 +37,16 @@ void testRoundToNearest() { assertThat(roundToNearest(50)).isEqualTo(50); assertThat(roundToNearest(10)).isEqualTo(10); assertThat(roundToNearest(2)).isEqualTo(2); - assertThat(roundToNearest(0.1f)).isEqualTo(0.1f); - assertThat(roundToNearest(0.001f)).isEqualTo(0.001f); + assertThat(roundToNearest(0.1)).isEqualTo(0.1); + assertThat(roundToNearest(0.001)).isEqualTo(0.001); assertThat(roundToNearest(0)).isEqualTo(0); // imperfect assertThat(roundToNearest(90)).isEqualTo(100); assertThat(roundToNearest(51)).isEqualTo(50); assertThat(roundToNearest(49)).isEqualTo(50); - assertThat(roundToNearest(34)).isCloseTo(33.333f, offset(0.001f)); - assertThat(roundToNearest(33)).isCloseTo(33.333f, offset(0.001f)); + assertThat(roundToNearest(34)).isCloseTo(33.333, offset(0.001)); + assertThat(roundToNearest(33)).isCloseTo(33.333, offset(0.001)); assertThat(roundToNearest(26)).isEqualTo(25); assertThat(roundToNearest(24)).isEqualTo(25); } diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/init/RpConfigurationPollingTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/init/RpConfigurationPollingTest.java index 8bf9e997397..d85d4cdf61c 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/init/RpConfigurationPollingTest.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/init/RpConfigurationPollingTest.java @@ -23,24 +23,15 @@ import static org.assertj.core.api.Assertions.assertThat; -import com.azure.monitor.opentelemetry.exporter.implementation.AiSemanticAttributes; +import com.microsoft.applicationinsights.agent.internal.SamplingTestUtil; import com.microsoft.applicationinsights.agent.internal.classicsdk.BytecodeUtilImpl; import com.microsoft.applicationinsights.agent.internal.configuration.Configuration; import com.microsoft.applicationinsights.agent.internal.configuration.RpConfiguration; import com.microsoft.applicationinsights.agent.internal.sampling.DelegatingSampler; import com.microsoft.applicationinsights.agent.internal.sampling.Samplers; import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryClient; -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.api.trace.SpanContext; -import io.opentelemetry.api.trace.SpanKind; -import io.opentelemetry.api.trace.TraceFlags; -import io.opentelemetry.api.trace.TraceState; -import io.opentelemetry.context.Context; -import io.opentelemetry.sdk.trace.samplers.SamplingResult; import java.net.URISyntaxException; import java.nio.file.Paths; -import java.util.Collections; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -57,14 +48,18 @@ class RpConfigurationPollingTest { @BeforeEach void beforeEach() { // default sampler at startup is "Sampler.alwaysOff()", and this test relies on real sampler - DelegatingSampler.getInstance().setDelegate(Samplers.getSampler(100, new Configuration())); + Configuration config = new Configuration(); + config.sampling.percentage = 100.0; + DelegatingSampler.getInstance().setDelegate(Samplers.getSampler(config)); } @AfterEach void afterEach() { // need to reset trace config back to default (with default sampler) // otherwise tests run after this can fail - DelegatingSampler.getInstance().setDelegate(Samplers.getSampler(100, new Configuration())); + Configuration config = new Configuration(); + config.sampling.percentage = 100.0; + DelegatingSampler.getInstance().setDelegate(Samplers.getSampler(config)); } @Test @@ -72,7 +67,7 @@ void shouldUpdate() throws URISyntaxException { // given RpConfiguration rpConfiguration = new RpConfiguration(); rpConfiguration.connectionString = "InstrumentationKey=11111111-1111-1111-1111-111111111111"; - rpConfiguration.sampling.percentage = 90; + rpConfiguration.sampling.percentage = 90.0; rpConfiguration.configPath = Paths.get( RpConfigurationPollingTest.class.getResource("/applicationinsights-rp.json").toURI()); @@ -107,7 +102,7 @@ void shouldBePopulatedByEnvVars() throws URISyntaxException { // given RpConfiguration rpConfiguration = new RpConfiguration(); rpConfiguration.connectionString = "InstrumentationKey=11111111-1111-1111-1111-111111111111"; - rpConfiguration.sampling.percentage = 90; + rpConfiguration.sampling.percentage = 90.0; rpConfiguration.configPath = Paths.get( RpConfigurationPollingTest.class.getResource("/applicationinsights-rp.json").toURI()); @@ -143,25 +138,6 @@ void shouldBePopulatedByEnvVars() throws URISyntaxException { } private static double getCurrentSamplingPercentage() { - SpanContext spanContext = - SpanContext.createFromRemoteParent( - "12341234123412341234123412341234", - "1234123412341234", - TraceFlags.getSampled(), - TraceState.getDefault()); - Context parentContext = Context.root().with(Span.wrap(spanContext)); - SamplingResult samplingResult = - DelegatingSampler.getInstance() - .shouldSample( - parentContext, - // traceId=27272727272727272727272727272727 is known to produce a score of 0.66 (out - // of 100) so will be sampled as long as samplingPercentage > 1% - "27272727272727272727272727272727", - "my span name", - SpanKind.SERVER, - Attributes.empty(), - Collections.emptyList()); - Long itemCount = samplingResult.getAttributes().get(AiSemanticAttributes.ITEM_COUNT); - return 100.0 / itemCount; + return SamplingTestUtil.getCurrentSamplingPercentage(DelegatingSampler.getInstance()); } } diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverridesTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverridesTest.java index db174c1fca4..066fc8a7d0d 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverridesTest.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverridesTest.java @@ -24,13 +24,12 @@ import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; -import com.microsoft.applicationinsights.agent.internal.configuration.Configuration; +import com.microsoft.applicationinsights.agent.internal.SamplingTestUtil; import com.microsoft.applicationinsights.agent.internal.configuration.Configuration.MatchType; import com.microsoft.applicationinsights.agent.internal.configuration.Configuration.SamplingOverride; import com.microsoft.applicationinsights.agent.internal.configuration.Configuration.SamplingOverrideAttribute; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.sdk.trace.samplers.Sampler; import java.util.ArrayList; import java.util.Arrays; @@ -47,38 +46,36 @@ void shouldSampleByDefault() { Attributes attributes = Attributes.empty(); // when - Sampler sampler = samplingOverrides.getOverride(SpanKind.SERVER, attributes); + Sampler sampler = samplingOverrides.getOverride(true, attributes); // expect assertThat(sampler).isNull(); } @Test - void shouldFilterBySpanKind() { + void shouldFilterInRequest() { // given - List overrides = - singletonList(newOverride(Configuration.SpanKind.SERVER, 25)); + List overrides = singletonList(newOverride(25)); SamplingOverrides samplingOverrides = new SamplingOverrides(overrides); Attributes attributes = Attributes.empty(); // when - Sampler sampler = samplingOverrides.getOverride(SpanKind.SERVER, attributes); + Sampler sampler = samplingOverrides.getOverride(true, attributes); // expect assertThat(sampler).isNotNull(); - assertThat(sampler.getDescription()).isEqualTo("AzureMonitorSampler{25.000%}"); + assertThat(SamplingTestUtil.getCurrentSamplingPercentage(sampler)).isEqualTo(25); } @Test - void shouldNotFilterBySpanKind() { + void shouldNotFilterInRequest() { // given - List overrides = - singletonList(newOverride(Configuration.SpanKind.SERVER, 25)); + List overrides = singletonList(newOverride(25)); SamplingOverrides samplingOverrides = new SamplingOverrides(overrides); Attributes attributes = Attributes.empty(); // when - Sampler sampler = samplingOverrides.getOverride(SpanKind.CLIENT, attributes); + Sampler sampler = samplingOverrides.getOverride(false, attributes); // expect assertThat(sampler).isNull(); @@ -88,62 +85,28 @@ void shouldNotFilterBySpanKind() { void shouldFilterStrictMatch() { // given List overrides = - singletonList( - newOverride(Configuration.SpanKind.SERVER, 25, newStrictAttribute("one", "1"))); - SamplingOverrides samplingOverrides = new SamplingOverrides(overrides); - Attributes attributes = Attributes.of(AttributeKey.stringKey("one"), "1"); - - // when - Sampler sampler = samplingOverrides.getOverride(SpanKind.SERVER, attributes); - - // expect - assertThat(sampler).isNotNull(); - assertThat(sampler.getDescription()).isEqualTo("AzureMonitorSampler{25.000%}"); - } - - @Test - void shouldFilterStrictMatchWithNullSpanKind() { - // given - List overrides = - singletonList(newOverride(null, 25, newStrictAttribute("one", "1"))); + singletonList(newOverride(25, newStrictAttribute("one", "1"))); SamplingOverrides samplingOverrides = new SamplingOverrides(overrides); Attributes attributes = Attributes.of(AttributeKey.stringKey("one"), "1"); // when - Sampler sampler = samplingOverrides.getOverride(SpanKind.SERVER, attributes); + Sampler sampler = samplingOverrides.getOverride(true, attributes); // expect assertThat(sampler).isNotNull(); - assertThat(sampler.getDescription()).isEqualTo("AzureMonitorSampler{25.000%}"); - } - - @Test - void shouldNotFilterStrictMatchWithWrongSpanKind() { - // given - List overrides = - singletonList( - newOverride(Configuration.SpanKind.SERVER, 25, newStrictAttribute("one", "1"))); - SamplingOverrides samplingOverrides = new SamplingOverrides(overrides); - Attributes attributes = Attributes.of(AttributeKey.stringKey("one"), "1"); - - // when - Sampler sampler = samplingOverrides.getOverride(SpanKind.CLIENT, attributes); - - // expect - assertThat(sampler).isNull(); + assertThat(SamplingTestUtil.getCurrentSamplingPercentage(sampler)).isEqualTo(25); } @Test void shouldNotFilterStrictMatch() { // given List overrides = - singletonList( - newOverride(Configuration.SpanKind.SERVER, 25, newStrictAttribute("one", "1"))); + singletonList(newOverride(25, newStrictAttribute("one", "1"))); SamplingOverrides samplingOverrides = new SamplingOverrides(overrides); Attributes attributes = Attributes.of(AttributeKey.stringKey("one"), "2"); // when - Sampler sampler = samplingOverrides.getOverride(SpanKind.SERVER, attributes); + Sampler sampler = samplingOverrides.getOverride(true, attributes); // expect assertThat(sampler).isNull(); @@ -153,13 +116,12 @@ void shouldNotFilterStrictMatch() { void shouldNotFilterMissingStrictMatch() { // given List overrides = - singletonList( - newOverride(Configuration.SpanKind.SERVER, 25, newStrictAttribute("one", "1"))); + singletonList(newOverride(25, newStrictAttribute("one", "1"))); SamplingOverrides samplingOverrides = new SamplingOverrides(overrides); Attributes attributes = Attributes.of(AttributeKey.stringKey("two"), "1"); // when - Sampler sampler = samplingOverrides.getOverride(SpanKind.SERVER, attributes); + Sampler sampler = samplingOverrides.getOverride(true, attributes); // expect assertThat(sampler).isNull(); @@ -169,30 +131,28 @@ void shouldNotFilterMissingStrictMatch() { void shouldFilterRegexpMatch() { // given List overrides = - singletonList( - newOverride(Configuration.SpanKind.SERVER, 25, newRegexpAttribute("one", "1.*"))); + singletonList(newOverride(25, newRegexpAttribute("one", "1.*"))); SamplingOverrides samplingOverrides = new SamplingOverrides(overrides); Attributes attributes = Attributes.of(AttributeKey.stringKey("one"), "11"); // when - Sampler sampler = samplingOverrides.getOverride(SpanKind.SERVER, attributes); + Sampler sampler = samplingOverrides.getOverride(true, attributes); // expect assertThat(sampler).isNotNull(); - assertThat(sampler.getDescription()).isEqualTo("AzureMonitorSampler{25.000%}"); + assertThat(SamplingTestUtil.getCurrentSamplingPercentage(sampler)).isEqualTo(25); } @Test void shouldNotFilterRegexpMatch() { // given List overrides = - singletonList( - newOverride(Configuration.SpanKind.SERVER, 25, newRegexpAttribute("one", "1.*"))); + singletonList(newOverride(25, newRegexpAttribute("one", "1.*"))); SamplingOverrides samplingOverrides = new SamplingOverrides(overrides); Attributes attributes = Attributes.of(AttributeKey.stringKey("one"), "22"); // when - Sampler sampler = samplingOverrides.getOverride(SpanKind.SERVER, attributes); + Sampler sampler = samplingOverrides.getOverride(true, attributes); // expect assertThat(sampler).isNull(); @@ -202,13 +162,12 @@ void shouldNotFilterRegexpMatch() { void shouldNotFilterMissingRegexpMatch() { // given List overrides = - singletonList( - newOverride(Configuration.SpanKind.SERVER, 25, newRegexpAttribute("one", "1.*"))); + singletonList(newOverride(25, newRegexpAttribute("one", "1.*"))); SamplingOverrides samplingOverrides = new SamplingOverrides(overrides); Attributes attributes = Attributes.of(AttributeKey.stringKey("two"), "11"); // when - Sampler sampler = samplingOverrides.getOverride(SpanKind.SERVER, attributes); + Sampler sampler = samplingOverrides.getOverride(true, attributes); // expect assertThat(sampler).isNull(); @@ -217,29 +176,27 @@ void shouldNotFilterMissingRegexpMatch() { @Test void shouldFilterKeyOnlyMatch() { // given - List overrides = - singletonList(newOverride(Configuration.SpanKind.SERVER, 25, newKeyOnlyAttribute("one"))); + List overrides = singletonList(newOverride(25, newKeyOnlyAttribute("one"))); SamplingOverrides samplingOverrides = new SamplingOverrides(overrides); Attributes attributes = Attributes.of(AttributeKey.stringKey("one"), "11"); // when - Sampler sampler = samplingOverrides.getOverride(SpanKind.SERVER, attributes); + Sampler sampler = samplingOverrides.getOverride(true, attributes); // expect assertThat(sampler).isNotNull(); - assertThat(sampler.getDescription()).isEqualTo("AzureMonitorSampler{25.000%}"); + assertThat(SamplingTestUtil.getCurrentSamplingPercentage(sampler)).isEqualTo(25); } @Test void shouldNotFilterKeyOnlyMatch() { // given - List overrides = - singletonList(newOverride(Configuration.SpanKind.SERVER, 25, newKeyOnlyAttribute("one"))); + List overrides = singletonList(newOverride(25, newKeyOnlyAttribute("one"))); SamplingOverrides samplingOverrides = new SamplingOverrides(overrides); Attributes attributes = Attributes.of(AttributeKey.stringKey("two"), "22"); // when - Sampler sampler = samplingOverrides.getOverride(SpanKind.SERVER, attributes); + Sampler sampler = samplingOverrides.getOverride(true, attributes); // expect assertThat(sampler).isNull(); @@ -250,21 +207,17 @@ void shouldFilterMultiAttributes() { // given List overrides = singletonList( - newOverride( - Configuration.SpanKind.SERVER, - 25, - newStrictAttribute("one", "1"), - newRegexpAttribute("two", "2.*"))); + newOverride(25, newStrictAttribute("one", "1"), newRegexpAttribute("two", "2.*"))); SamplingOverrides samplerOverride = new SamplingOverrides(overrides); Attributes attributes = Attributes.of(AttributeKey.stringKey("one"), "1", AttributeKey.stringKey("two"), "22"); // when - Sampler sampler = samplerOverride.getOverride(SpanKind.SERVER, attributes); + Sampler sampler = samplerOverride.getOverride(true, attributes); // expect assertThat(sampler).isNotNull(); - assertThat(sampler.getDescription()).isEqualTo("AzureMonitorSampler{25.000%}"); + assertThat(SamplingTestUtil.getCurrentSamplingPercentage(sampler)).isEqualTo(25); } @Test @@ -272,17 +225,13 @@ void shouldNotFilterMultiAttributes() { // given List overrides = singletonList( - newOverride( - Configuration.SpanKind.SERVER, - 25, - newStrictAttribute("one", "1"), - newRegexpAttribute("two", "2.*"))); + newOverride(25, newStrictAttribute("one", "1"), newRegexpAttribute("two", "2.*"))); SamplingOverrides samplingOverrides = new SamplingOverrides(overrides); Attributes attributes = Attributes.of(AttributeKey.stringKey("one"), "2", AttributeKey.stringKey("two"), "22"); // when - Sampler sampler = samplingOverrides.getOverride(SpanKind.SERVER, attributes); + Sampler sampler = samplingOverrides.getOverride(true, attributes); // expect assertThat(sampler).isNull(); @@ -293,18 +242,18 @@ void shouldFilterMultiConfigsBothMatch() { // given List overrides = Arrays.asList( - newOverride(Configuration.SpanKind.SERVER, 25, newStrictAttribute("one", "1")), - newOverride(Configuration.SpanKind.SERVER, 0, newRegexpAttribute("two", "2.*"))); + newOverride(25, newStrictAttribute("one", "1")), + newOverride(0, newRegexpAttribute("two", "2.*"))); SamplingOverrides samplingOverrides = new SamplingOverrides(overrides); Attributes attributes = Attributes.of(AttributeKey.stringKey("one"), "1", AttributeKey.stringKey("two"), "22"); // when - Sampler sampler = samplingOverrides.getOverride(SpanKind.SERVER, attributes); + Sampler sampler = samplingOverrides.getOverride(true, attributes); // expect assertThat(sampler).isNotNull(); - assertThat(sampler.getDescription()).isEqualTo("AzureMonitorSampler{25.000%}"); + assertThat(SamplingTestUtil.getCurrentSamplingPercentage(sampler)).isEqualTo(25); } @Test @@ -312,18 +261,18 @@ void shouldFilterMultiConfigsOneMatch() { // given List overrides = Arrays.asList( - newOverride(Configuration.SpanKind.SERVER, 50, newStrictAttribute("one", "1")), - newOverride(Configuration.SpanKind.SERVER, 25, newRegexpAttribute("two", "2.*"))); + newOverride(50, newStrictAttribute("one", "1")), + newOverride(25, newRegexpAttribute("two", "2.*"))); SamplingOverrides samplingOverrides = new SamplingOverrides(overrides); Attributes attributes = Attributes.of(AttributeKey.stringKey("one"), "2", AttributeKey.stringKey("two"), "22"); // when - Sampler sampler = samplingOverrides.getOverride(SpanKind.SERVER, attributes); + Sampler sampler = samplingOverrides.getOverride(true, attributes); // expect assertThat(sampler).isNotNull(); - assertThat(sampler.getDescription()).isEqualTo("AzureMonitorSampler{25.000%}"); + assertThat(SamplingTestUtil.getCurrentSamplingPercentage(sampler)).isEqualTo(25); } @Test @@ -331,23 +280,22 @@ void shouldNotFilterMultiConfigsNoMatch() { // given List overrides = Arrays.asList( - newOverride(Configuration.SpanKind.SERVER, 50, newStrictAttribute("one", "1")), - newOverride(Configuration.SpanKind.SERVER, 25, newRegexpAttribute("two", "2.*"))); + newOverride(50, newStrictAttribute("one", "1")), + newOverride(25, newRegexpAttribute("two", "2.*"))); SamplingOverrides samplingOverrides = new SamplingOverrides(overrides); Attributes attributes = Attributes.of(AttributeKey.stringKey("one"), "2", AttributeKey.stringKey("two"), "33"); // when - Sampler sampler = samplingOverrides.getOverride(SpanKind.SERVER, attributes); + Sampler sampler = samplingOverrides.getOverride(true, attributes); // expect assertThat(sampler).isNull(); } private static SamplingOverride newOverride( - Configuration.SpanKind spanKind, float percentage, SamplingOverrideAttribute... attribute) { + double percentage, SamplingOverrideAttribute... attribute) { SamplingOverride override = new SamplingOverride(); - override.spanKind = spanKind; override.attributes = Arrays.asList(attribute); override.percentage = percentage; return override; diff --git a/agent/azure-monitor-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/AzureMonitorLogExporter.java b/agent/azure-monitor-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/AzureMonitorLogExporter.java index 9026b687d2d..a1bdf1f5ae6 100644 --- a/agent/azure-monitor-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/AzureMonitorLogExporter.java +++ b/agent/azure-monitor-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/AzureMonitorLogExporter.java @@ -31,6 +31,7 @@ import io.opentelemetry.sdk.common.CompletableResultCode; import io.opentelemetry.sdk.logs.data.LogData; import io.opentelemetry.sdk.logs.export.LogExporter; +import io.opentelemetry.semconv.trace.attributes.SemanticAttributes; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -69,7 +70,8 @@ public CompletableResultCode export(Collection logs) { for (LogData log : logs) { LOGGER.verbose("exporting log: {}", log); try { - mapper.map(log, telemetryItems::add); + String stack = log.getAttributes().get(SemanticAttributes.EXCEPTION_STACKTRACE); + telemetryItems.add(mapper.map(log, stack, null)); exportingLogLogger.recordSuccess(); } catch (Throwable t) { exportingLogLogger.recordFailure(t.getMessage(), t, EXPORTER_MAPPING_ERROR); diff --git a/agent/azure-monitor-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/LogDataMapper.java b/agent/azure-monitor-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/LogDataMapper.java index 7410c14f901..d1420ac1584 100644 --- a/agent/azure-monitor-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/LogDataMapper.java +++ b/agent/azure-monitor-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/LogDataMapper.java @@ -36,7 +36,6 @@ import io.opentelemetry.sdk.resources.Resource; import io.opentelemetry.semconv.trace.attributes.SemanticAttributes; import java.util.function.BiConsumer; -import java.util.function.Consumer; import javax.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,23 +54,22 @@ public LogDataMapper( this.telemetryInitializer = telemetryInitializer; } - public void map(LogData log, Consumer consumer) { - String stack = log.getAttributes().get(SemanticAttributes.EXCEPTION_STACKTRACE); + public TelemetryItem map(LogData log, @Nullable String stack, @Nullable Long itemCount) { if (stack == null) { - consumer.accept(createMessageTelemetryItem(log)); + return createMessageTelemetryItem(log, itemCount); } else { - consumer.accept(createExceptionTelemetryItem(log, stack)); + return createExceptionTelemetryItem(log, itemCount, stack); } } - private TelemetryItem createMessageTelemetryItem(LogData log) { + private TelemetryItem createMessageTelemetryItem(LogData log, @Nullable Long itemCount) { MessageTelemetryBuilder telemetryBuilder = MessageTelemetryBuilder.create(); telemetryInitializer.accept(telemetryBuilder, log.getResource()); // set standard properties setOperationTags(telemetryBuilder, log); setTime(telemetryBuilder, log.getEpochNanos()); - setItemCount(telemetryBuilder, log); + setItemCount(telemetryBuilder, log, itemCount); // update tags Attributes attributes = log.getAttributes(); @@ -90,14 +88,15 @@ private TelemetryItem createMessageTelemetryItem(LogData log) { return telemetryBuilder.build(); } - private TelemetryItem createExceptionTelemetryItem(LogData log, String stack) { + private TelemetryItem createExceptionTelemetryItem( + LogData log, @Nullable Long itemCount, String stack) { ExceptionTelemetryBuilder telemetryBuilder = ExceptionTelemetryBuilder.create(); telemetryInitializer.accept(telemetryBuilder, log.getResource()); // set standard properties setOperationTags(telemetryBuilder, log); setTime(telemetryBuilder, log.getEpochNanos()); - setItemCount(telemetryBuilder, log); + setItemCount(telemetryBuilder, log, itemCount); // update tags Attributes attributes = log.getAttributes(); @@ -142,8 +141,11 @@ private static void setTime(AbstractTelemetryBuilder telemetryBuilder, long epoc telemetryBuilder.setTime(FormattedTime.offSetDateTimeFromEpochNanos(epochNanos)); } - private static void setItemCount(AbstractTelemetryBuilder telemetryBuilder, LogData log) { - Long itemCount = log.getAttributes().get(AiSemanticAttributes.ITEM_COUNT); + private static void setItemCount( + AbstractTelemetryBuilder telemetryBuilder, LogData log, @Nullable Long itemCount) { + if (itemCount == null) { + itemCount = log.getAttributes().get(AiSemanticAttributes.ITEM_COUNT); + } if (itemCount != null && itemCount != 1) { telemetryBuilder.setSampleRate(100.0f / itemCount); } diff --git a/agent/azure-monitor-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/SpanDataMapper.java b/agent/azure-monitor-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/SpanDataMapper.java index 7dee2a8f299..eedde69d5cc 100644 --- a/agent/azure-monitor-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/SpanDataMapper.java +++ b/agent/azure-monitor-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/SpanDataMapper.java @@ -200,18 +200,20 @@ public static boolean isRequest(ReadableSpan span) { span::getAttribute); } - private static boolean isRequest( + public static boolean isRequest( SpanKind kind, SpanContext parentSpanContext, - InstrumentationScopeInfo scopeInfo, + @Nullable InstrumentationScopeInfo scopeInfo, Function, String> attrFn) { - String instrumentationName = scopeInfo.getName(); + String instrumentationName = scopeInfo == null ? null : scopeInfo.getName(); if (kind == SpanKind.INTERNAL) { // TODO (trask) AI mapping: need semantic convention for determining whether to map INTERNAL // to request or dependency (or need clarification to use SERVER for this) - return (instrumentationName.startsWith("io.opentelemetry.spring-scheduling-") - || instrumentationName.equals("io.opentelemetry.methods")) - && !parentSpanContext.isValid(); + return !parentSpanContext.isValid() + && instrumentationName != null + && (instrumentationName.startsWith("io.opentelemetry.spring-scheduling-") + || instrumentationName.startsWith("io.opentelemetry.quartz-") + || instrumentationName.equals("io.opentelemetry.methods")); } else if (kind == SpanKind.CLIENT || kind == SpanKind.PRODUCER) { return false; } else if (kind == SpanKind.CONSUMER diff --git a/settings.gradle b/settings.gradle index faab5f5a19f..c2e13e54e36 100644 --- a/settings.gradle +++ b/settings.gradle @@ -103,8 +103,11 @@ include ':smoke-tests:apps:MongoDB' include ':smoke-tests:apps:NonDaemonThreads' include ':smoke-tests:apps:OpenTelemetryApiSupport' include ':smoke-tests:apps:OpenTelemetryMetric' +include ':smoke-tests:apps:RateLimitedSampling' include ':smoke-tests:apps:ReadOnly' include ':smoke-tests:apps:Sampling' +include ':smoke-tests:apps:SamplingOverridesV1' +include ':smoke-tests:apps:SamplingOverridesV2' include ':smoke-tests:apps:SpringBoot1_3Auto' include ':smoke-tests:apps:SpringBootAttachInMain' include ':smoke-tests:apps:SpringBootAuto' @@ -113,7 +116,6 @@ include ':smoke-tests:apps:SpringCloudStream' include ':smoke-tests:apps:SpringScheduling' include ':smoke-tests:apps:Statsbeat' include ':smoke-tests:apps:SystemExit' -include ':smoke-tests:apps:TelemetryFiltering' include ':smoke-tests:apps:TelemetryProcessors' include ':smoke-tests:apps:TraceJavaUtilLoggingUsingAgent' include ':smoke-tests:apps:TraceLog4j1_2' diff --git a/smoke-tests/apps/ActuatorMetrics/src/smokeTest/resources/applicationinsights.json b/smoke-tests/apps/ActuatorMetrics/src/smokeTest/resources/applicationinsights.json index 7918abf3ad7..566f862fe76 100644 --- a/smoke-tests/apps/ActuatorMetrics/src/smokeTest/resources/applicationinsights.json +++ b/smoke-tests/apps/ActuatorMetrics/src/smokeTest/resources/applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "preview": { "metricIntervalSeconds": 5, "processors": [ diff --git a/smoke-tests/apps/ActuatorMetrics/src/smokeTest/resources/disabled_applicationinsights.json b/smoke-tests/apps/ActuatorMetrics/src/smokeTest/resources/disabled_applicationinsights.json index 5b2b48caff9..fae0afd40f0 100644 --- a/smoke-tests/apps/ActuatorMetrics/src/smokeTest/resources/disabled_applicationinsights.json +++ b/smoke-tests/apps/ActuatorMetrics/src/smokeTest/resources/disabled_applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "instrumentation": { "micrometer": { "enabled": false diff --git a/smoke-tests/apps/AutoPerfCounters/src/smokeTest/resources/applicationinsights.json b/smoke-tests/apps/AutoPerfCounters/src/smokeTest/resources/applicationinsights.json index 196b235f109..5b24d9d03ab 100644 --- a/smoke-tests/apps/AutoPerfCounters/src/smokeTest/resources/applicationinsights.json +++ b/smoke-tests/apps/AutoPerfCounters/src/smokeTest/resources/applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "preview": { "metricIntervalSeconds": 5 } diff --git a/smoke-tests/apps/AzureSdk/src/smokeTest/resources/controller_spans_enabled_applicationinsights.json b/smoke-tests/apps/AzureSdk/src/smokeTest/resources/controller_spans_enabled_applicationinsights.json index ce7adc74627..03e94410b60 100644 --- a/smoke-tests/apps/AzureSdk/src/smokeTest/resources/controller_spans_enabled_applicationinsights.json +++ b/smoke-tests/apps/AzureSdk/src/smokeTest/resources/controller_spans_enabled_applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "preview": { "captureControllerSpans": true } diff --git a/smoke-tests/apps/AzureSdk/src/smokeTest/resources/disabled_applicationinsights.json b/smoke-tests/apps/AzureSdk/src/smokeTest/resources/disabled_applicationinsights.json index 953b0a2f401..5e59d545b80 100644 --- a/smoke-tests/apps/AzureSdk/src/smokeTest/resources/disabled_applicationinsights.json +++ b/smoke-tests/apps/AzureSdk/src/smokeTest/resources/disabled_applicationinsights.json @@ -1,5 +1,11 @@ { - "connectionString": "InstrumentationKey=00000000-0000-0000-0000-0FEEDDADBEEF;IngestionEndpoint=http://host.docker.internal:6060/", + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "instrumentation": { "azureSdk": { "enabled": false diff --git a/smoke-tests/apps/Cassandra/src/smokeTest/resources/disabled_applicationinsights.json b/smoke-tests/apps/Cassandra/src/smokeTest/resources/disabled_applicationinsights.json index d156545f60f..1cfb34d0229 100644 --- a/smoke-tests/apps/Cassandra/src/smokeTest/resources/disabled_applicationinsights.json +++ b/smoke-tests/apps/Cassandra/src/smokeTest/resources/disabled_applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "instrumentation": { "cassandra": { "enabled": false diff --git a/smoke-tests/apps/CustomDimensions/src/smokeTest/resources/applicationinsights.json b/smoke-tests/apps/CustomDimensions/src/smokeTest/resources/applicationinsights.json index 298829dabb7..594d6b35618 100644 --- a/smoke-tests/apps/CustomDimensions/src/smokeTest/resources/applicationinsights.json +++ b/smoke-tests/apps/CustomDimensions/src/smokeTest/resources/applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "customDimensions": { "test": "value", "home": "${HOME}", diff --git a/smoke-tests/apps/CustomInstrumentation/src/smokeTest/resources/applicationinsights.json b/smoke-tests/apps/CustomInstrumentation/src/smokeTest/resources/applicationinsights.json index 8ac2a331504..3769f028be4 100644 --- a/smoke-tests/apps/CustomInstrumentation/src/smokeTest/resources/applicationinsights.json +++ b/smoke-tests/apps/CustomInstrumentation/src/smokeTest/resources/applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "preview": { "customInstrumentation": [ { diff --git a/smoke-tests/apps/HeartBeat/src/smokeTest/resources/applicationinsights.json b/smoke-tests/apps/HeartBeat/src/smokeTest/resources/applicationinsights.json index 98fb9682caa..cf6fd297840 100644 --- a/smoke-tests/apps/HeartBeat/src/smokeTest/resources/applicationinsights.json +++ b/smoke-tests/apps/HeartBeat/src/smokeTest/resources/applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "heartbeat": { "intervalSeconds": 5 } diff --git a/smoke-tests/apps/HttpHeaders/src/smokeTest/resources/applicationinsights.json b/smoke-tests/apps/HttpHeaders/src/smokeTest/resources/applicationinsights.json index 8f9e6eedc17..060d425c5ff 100644 --- a/smoke-tests/apps/HttpHeaders/src/smokeTest/resources/applicationinsights.json +++ b/smoke-tests/apps/HttpHeaders/src/smokeTest/resources/applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "preview": { "captureHttpServerHeaders": { "requestHeaders": [ diff --git a/smoke-tests/apps/HttpServer4xx/src/smokeTest/resources/httpserver4xx_applicationinsights.json b/smoke-tests/apps/HttpServer4xx/src/smokeTest/resources/httpserver4xx_applicationinsights.json index 8b5a07f2392..d5cbdf0eee3 100644 --- a/smoke-tests/apps/HttpServer4xx/src/smokeTest/resources/httpserver4xx_applicationinsights.json +++ b/smoke-tests/apps/HttpServer4xx/src/smokeTest/resources/httpserver4xx_applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "preview": { "captureHttpServer4xxAsError": false } diff --git a/smoke-tests/apps/InheritedAttributes/src/smokeTest/resources/applicationinsights.json b/smoke-tests/apps/InheritedAttributes/src/smokeTest/resources/applicationinsights.json index f6249243b08..b34399c8924 100644 --- a/smoke-tests/apps/InheritedAttributes/src/smokeTest/resources/applicationinsights.json +++ b/smoke-tests/apps/InheritedAttributes/src/smokeTest/resources/applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "preview": { "inheritedAttributes": [ { diff --git a/smoke-tests/apps/JMS/src/smokeTest/resources/controller_spans_enabled_applicationinsights.json b/smoke-tests/apps/JMS/src/smokeTest/resources/controller_spans_enabled_applicationinsights.json index ce7adc74627..03e94410b60 100644 --- a/smoke-tests/apps/JMS/src/smokeTest/resources/controller_spans_enabled_applicationinsights.json +++ b/smoke-tests/apps/JMS/src/smokeTest/resources/controller_spans_enabled_applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "preview": { "captureControllerSpans": true } diff --git a/smoke-tests/apps/JMS/src/smokeTest/resources/disabled_applicationinsights.json b/smoke-tests/apps/JMS/src/smokeTest/resources/disabled_applicationinsights.json index 6dd1ae9913f..0b065b482bb 100644 --- a/smoke-tests/apps/JMS/src/smokeTest/resources/disabled_applicationinsights.json +++ b/smoke-tests/apps/JMS/src/smokeTest/resources/disabled_applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "instrumentation": { "jms": { "enabled": false diff --git a/smoke-tests/apps/Jdbc/src/smokeTest/resources/disabled_applicationinsights.json b/smoke-tests/apps/Jdbc/src/smokeTest/resources/disabled_applicationinsights.json index e90407242bf..ce8ef52ee60 100644 --- a/smoke-tests/apps/Jdbc/src/smokeTest/resources/disabled_applicationinsights.json +++ b/smoke-tests/apps/Jdbc/src/smokeTest/resources/disabled_applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "instrumentation": { "jdbc": { "enabled": false diff --git a/smoke-tests/apps/Jdbc/src/smokeTest/resources/unmasked_applicationinsights.json b/smoke-tests/apps/Jdbc/src/smokeTest/resources/unmasked_applicationinsights.json index 1aebcdcff8f..b05a38c9420 100644 --- a/smoke-tests/apps/Jdbc/src/smokeTest/resources/unmasked_applicationinsights.json +++ b/smoke-tests/apps/Jdbc/src/smokeTest/resources/unmasked_applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "instrumentation": { "jdbc": { "masking": { diff --git a/smoke-tests/apps/Jedis/src/smokeTest/resources/disabled_applicationinsights.json b/smoke-tests/apps/Jedis/src/smokeTest/resources/disabled_applicationinsights.json index 5a8ea771bdc..47440a74c4c 100644 --- a/smoke-tests/apps/Jedis/src/smokeTest/resources/disabled_applicationinsights.json +++ b/smoke-tests/apps/Jedis/src/smokeTest/resources/disabled_applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "instrumentation": { "redis": { "enabled": false diff --git a/smoke-tests/apps/Kafka/src/smokeTest/resources/controller_spans_enabled_applicationinsights.json b/smoke-tests/apps/Kafka/src/smokeTest/resources/controller_spans_enabled_applicationinsights.json index ce7adc74627..03e94410b60 100644 --- a/smoke-tests/apps/Kafka/src/smokeTest/resources/controller_spans_enabled_applicationinsights.json +++ b/smoke-tests/apps/Kafka/src/smokeTest/resources/controller_spans_enabled_applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "preview": { "captureControllerSpans": true } diff --git a/smoke-tests/apps/Kafka/src/smokeTest/resources/disabled_applicationinsights.json b/smoke-tests/apps/Kafka/src/smokeTest/resources/disabled_applicationinsights.json index 2388baf1750..6495998ade9 100644 --- a/smoke-tests/apps/Kafka/src/smokeTest/resources/disabled_applicationinsights.json +++ b/smoke-tests/apps/Kafka/src/smokeTest/resources/disabled_applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "instrumentation": { "kafka": { "enabled": false diff --git a/smoke-tests/apps/Lettuce/src/smokeTest/resources/disabled_applicationinsights.json b/smoke-tests/apps/Lettuce/src/smokeTest/resources/disabled_applicationinsights.json index 5a8ea771bdc..47440a74c4c 100644 --- a/smoke-tests/apps/Lettuce/src/smokeTest/resources/disabled_applicationinsights.json +++ b/smoke-tests/apps/Lettuce/src/smokeTest/resources/disabled_applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "instrumentation": { "redis": { "enabled": false diff --git a/smoke-tests/apps/Micrometer/src/smokeTest/resources/applicationinsights.json b/smoke-tests/apps/Micrometer/src/smokeTest/resources/applicationinsights.json index 7918abf3ad7..566f862fe76 100644 --- a/smoke-tests/apps/Micrometer/src/smokeTest/resources/applicationinsights.json +++ b/smoke-tests/apps/Micrometer/src/smokeTest/resources/applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "preview": { "metricIntervalSeconds": 5, "processors": [ diff --git a/smoke-tests/apps/Micrometer/src/smokeTest/resources/disabled_applicationinsights.json b/smoke-tests/apps/Micrometer/src/smokeTest/resources/disabled_applicationinsights.json index 5b2b48caff9..fae0afd40f0 100644 --- a/smoke-tests/apps/Micrometer/src/smokeTest/resources/disabled_applicationinsights.json +++ b/smoke-tests/apps/Micrometer/src/smokeTest/resources/disabled_applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "instrumentation": { "micrometer": { "enabled": false diff --git a/smoke-tests/apps/MongoDB/src/smokeTest/resources/disabled_applicationinsights.json b/smoke-tests/apps/MongoDB/src/smokeTest/resources/disabled_applicationinsights.json index 29de978cf76..350af825743 100644 --- a/smoke-tests/apps/MongoDB/src/smokeTest/resources/disabled_applicationinsights.json +++ b/smoke-tests/apps/MongoDB/src/smokeTest/resources/disabled_applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "instrumentation": { "mongo": { "enabled": false diff --git a/smoke-tests/apps/OpenTelemetryApiSupport/src/smokeTest/resources/controller_spans_enabled_applicationinsights.json b/smoke-tests/apps/OpenTelemetryApiSupport/src/smokeTest/resources/controller_spans_enabled_applicationinsights.json index ce7adc74627..03e94410b60 100644 --- a/smoke-tests/apps/OpenTelemetryApiSupport/src/smokeTest/resources/controller_spans_enabled_applicationinsights.json +++ b/smoke-tests/apps/OpenTelemetryApiSupport/src/smokeTest/resources/controller_spans_enabled_applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "preview": { "captureControllerSpans": true } diff --git a/smoke-tests/apps/OpenTelemetryMetric/src/smokeTest/resources/applicationinsights.json b/smoke-tests/apps/OpenTelemetryMetric/src/smokeTest/resources/applicationinsights.json index 20127e96483..b2771da7d9c 100644 --- a/smoke-tests/apps/OpenTelemetryMetric/src/smokeTest/resources/applicationinsights.json +++ b/smoke-tests/apps/OpenTelemetryMetric/src/smokeTest/resources/applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "preview": { "metricIntervalSeconds": 5 }, diff --git a/smoke-tests/apps/RateLimitedSampling/build.gradle.kts b/smoke-tests/apps/RateLimitedSampling/build.gradle.kts new file mode 100644 index 00000000000..36db70e0f92 --- /dev/null +++ b/smoke-tests/apps/RateLimitedSampling/build.gradle.kts @@ -0,0 +1,8 @@ +plugins { + id("ai.smoke-test-war") +} + +dependencies { + implementation("com.microsoft.azure:applicationinsights-core") + implementation("org.apache.httpcomponents:httpclient:4.5.7") +} diff --git a/smoke-tests/apps/RateLimitedSampling/src/main/java/com/microsoft/applicationinsights/smoketestapp/HealthCheckServlet.java b/smoke-tests/apps/RateLimitedSampling/src/main/java/com/microsoft/applicationinsights/smoketestapp/HealthCheckServlet.java new file mode 100644 index 00000000000..5df2d6dece3 --- /dev/null +++ b/smoke-tests/apps/RateLimitedSampling/src/main/java/com/microsoft/applicationinsights/smoketestapp/HealthCheckServlet.java @@ -0,0 +1,38 @@ +/* + * ApplicationInsights-Java + * Copyright (c) Microsoft Corporation + * All rights reserved. + * + * MIT License + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the ""Software""), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the following conditions: + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE + * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +package com.microsoft.applicationinsights.smoketestapp; + +import java.io.IOException; +import javax.servlet.annotation.WebServlet; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +@WebServlet( + description = "smoke test health check", + urlPatterns = {"/"}) +public class HealthCheckServlet extends HttpServlet { + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { + resp.getWriter().println("OK"); + } +} diff --git a/smoke-tests/apps/RateLimitedSampling/src/main/java/com/microsoft/applicationinsights/smoketestapp/ServletFuncs.java b/smoke-tests/apps/RateLimitedSampling/src/main/java/com/microsoft/applicationinsights/smoketestapp/ServletFuncs.java new file mode 100644 index 00000000000..c8b4a98e8d3 --- /dev/null +++ b/smoke-tests/apps/RateLimitedSampling/src/main/java/com/microsoft/applicationinsights/smoketestapp/ServletFuncs.java @@ -0,0 +1,45 @@ +/* + * ApplicationInsights-Java + * Copyright (c) Microsoft Corporation + * All rights reserved. + * + * MIT License + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the ""Software""), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the following conditions: + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE + * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +package com.microsoft.applicationinsights.smoketestapp; + +import java.io.IOException; +import java.io.PrintWriter; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +public class ServletFuncs { + + protected static void getRenderedHtml(HttpServletRequest request, HttpServletResponse response) + throws IOException { + response.setContentType("text/html;charset=UTF-8"); + renderHtml(request, response.getWriter()); + } + + private static void renderHtml(HttpServletRequest req, PrintWriter writer) { + writer.println(""); + writer.println("Calculation Result"); + writer.println(""); + writer.printf("

%s

", req.getRequestURI()); + writer.println("

OK!

"); + writer.println(""); + } +} diff --git a/smoke-tests/apps/RateLimitedSampling/src/main/java/com/microsoft/applicationinsights/smoketestapp/SimpleSamplingServlet.java b/smoke-tests/apps/RateLimitedSampling/src/main/java/com/microsoft/applicationinsights/smoketestapp/SimpleSamplingServlet.java new file mode 100644 index 00000000000..b3d8f9a9171 --- /dev/null +++ b/smoke-tests/apps/RateLimitedSampling/src/main/java/com/microsoft/applicationinsights/smoketestapp/SimpleSamplingServlet.java @@ -0,0 +1,53 @@ +/* + * ApplicationInsights-Java + * Copyright (c) Microsoft Corporation + * All rights reserved. + * + * MIT License + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the ""Software""), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the following conditions: + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE + * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +package com.microsoft.applicationinsights.smoketestapp; + +import java.io.IOException; +import java.util.concurrent.ThreadLocalRandom; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.servlet.annotation.WebServlet; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +@WebServlet( + description = "simple", + urlPatterns = {"/sampling"}) +public class SimpleSamplingServlet extends HttpServlet { + + private static final Logger logger = Logger.getLogger(SimpleSamplingServlet.class.getName()); + + protected void doGet(HttpServletRequest request, HttpServletResponse response) + throws IOException { + ServletFuncs.getRenderedHtml(request, response); + + logger.log(Level.WARNING, "test"); + + try { + Thread.sleep(ThreadLocalRandom.current().nextInt(20)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } +} diff --git a/smoke-tests/apps/RateLimitedSampling/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/RateLimitedSamplingTest.java b/smoke-tests/apps/RateLimitedSampling/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/RateLimitedSamplingTest.java new file mode 100644 index 00000000000..5e46559e5bd --- /dev/null +++ b/smoke-tests/apps/RateLimitedSampling/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/RateLimitedSamplingTest.java @@ -0,0 +1,85 @@ +/* + * ApplicationInsights-Java + * Copyright (c) Microsoft Corporation + * All rights reserved. + * + * MIT License + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the ""Software""), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the following conditions: + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE + * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +package com.microsoft.applicationinsights.smoketest; + +import static com.microsoft.applicationinsights.smoketest.WarEnvironmentValue.TOMCAT_8_JAVA_11; +import static com.microsoft.applicationinsights.smoketest.WarEnvironmentValue.TOMCAT_8_JAVA_11_OPENJ9; +import static com.microsoft.applicationinsights.smoketest.WarEnvironmentValue.TOMCAT_8_JAVA_17; +import static com.microsoft.applicationinsights.smoketest.WarEnvironmentValue.TOMCAT_8_JAVA_18; +import static com.microsoft.applicationinsights.smoketest.WarEnvironmentValue.TOMCAT_8_JAVA_19; +import static com.microsoft.applicationinsights.smoketest.WarEnvironmentValue.TOMCAT_8_JAVA_8; +import static com.microsoft.applicationinsights.smoketest.WarEnvironmentValue.TOMCAT_8_JAVA_8_OPENJ9; +import static com.microsoft.applicationinsights.smoketest.WarEnvironmentValue.WILDFLY_13_JAVA_8; +import static com.microsoft.applicationinsights.smoketest.WarEnvironmentValue.WILDFLY_13_JAVA_8_OPENJ9; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.assertj.core.api.Assertions.assertThat; + +import com.microsoft.applicationinsights.smoketest.schemav2.Envelope; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +@UseAgent +abstract class RateLimitedSamplingTest { + + @RegisterExtension static final SmokeTestExtension testing = SmokeTestExtension.create(); + + @Test + @TargetUri(value = "/sampling", callCount = 1000) + void testSampling() throws Exception { + // give some time to collect + Thread.sleep(SECONDS.toMillis(5)); + + List rdEnvelopes = testing.mockedIngestion.getItemsEnvelopeDataType("RequestData"); + + // average response time of 10 ms, times 1000 requests, equals 10 seconds + // so ideally with rate of 0.5 requests per second we would get 5 requests + assertThat(rdEnvelopes.size()).isLessThan(20); + } + + @Environment(TOMCAT_8_JAVA_8) + static class Tomcat8Java8Test extends RateLimitedSamplingTest {} + + @Environment(TOMCAT_8_JAVA_8_OPENJ9) + static class Tomcat8Java8OpenJ9Test extends RateLimitedSamplingTest {} + + @Environment(TOMCAT_8_JAVA_11) + static class Tomcat8Java11Test extends RateLimitedSamplingTest {} + + @Environment(TOMCAT_8_JAVA_11_OPENJ9) + static class Tomcat8Java11OpenJ9Test extends RateLimitedSamplingTest {} + + @Environment(TOMCAT_8_JAVA_17) + static class Tomcat8Java17Test extends RateLimitedSamplingTest {} + + @Environment(TOMCAT_8_JAVA_18) + static class Tomcat8Java18Test extends RateLimitedSamplingTest {} + + @Environment(TOMCAT_8_JAVA_19) + static class Tomcat8Java19Test extends RateLimitedSamplingTest {} + + @Environment(WILDFLY_13_JAVA_8) + static class Wildfly13Java8Test extends RateLimitedSamplingTest {} + + @Environment(WILDFLY_13_JAVA_8_OPENJ9) + static class Wildfly13Java8OpenJ9Test extends RateLimitedSamplingTest {} +} diff --git a/smoke-tests/apps/RateLimitedSampling/src/smokeTest/resources/applicationinsights.json b/smoke-tests/apps/RateLimitedSampling/src/smokeTest/resources/applicationinsights.json new file mode 100644 index 00000000000..44d7167bc2f --- /dev/null +++ b/smoke-tests/apps/RateLimitedSampling/src/smokeTest/resources/applicationinsights.json @@ -0,0 +1,9 @@ +{ + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling" : { + "limitPerSecond": 0.5 + } +} diff --git a/smoke-tests/apps/TelemetryFiltering/src/smokeTest/resources/logback-test.xml b/smoke-tests/apps/RateLimitedSampling/src/smokeTest/resources/logback-test.xml similarity index 100% rename from smoke-tests/apps/TelemetryFiltering/src/smokeTest/resources/logback-test.xml rename to smoke-tests/apps/RateLimitedSampling/src/smokeTest/resources/logback-test.xml diff --git a/smoke-tests/apps/Sampling/src/smokeTest/resources/applicationinsights.json b/smoke-tests/apps/Sampling/src/smokeTest/resources/applicationinsights.json index b9ad76764e4..25d2ee4f959 100644 --- a/smoke-tests/apps/Sampling/src/smokeTest/resources/applicationinsights.json +++ b/smoke-tests/apps/Sampling/src/smokeTest/resources/applicationinsights.json @@ -1,4 +1,8 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, "sampling": { "percentage": 50 } diff --git a/smoke-tests/apps/TelemetryFiltering/build.gradle.kts b/smoke-tests/apps/SamplingOverridesV1/build.gradle.kts similarity index 100% rename from smoke-tests/apps/TelemetryFiltering/build.gradle.kts rename to smoke-tests/apps/SamplingOverridesV1/build.gradle.kts diff --git a/smoke-tests/apps/TelemetryFiltering/src/main/java/com/microsoft/applicationinsights/smoketestapp/TelemetryFilteringTestServlet.java b/smoke-tests/apps/SamplingOverridesV1/src/main/java/com/microsoft/applicationinsights/smoketestapp/SamplingOverrideTestServlet.java similarity index 98% rename from smoke-tests/apps/TelemetryFiltering/src/main/java/com/microsoft/applicationinsights/smoketestapp/TelemetryFilteringTestServlet.java rename to smoke-tests/apps/SamplingOverridesV1/src/main/java/com/microsoft/applicationinsights/smoketestapp/SamplingOverrideTestServlet.java index 85dc9449987..41a2584c45f 100644 --- a/smoke-tests/apps/TelemetryFiltering/src/main/java/com/microsoft/applicationinsights/smoketestapp/TelemetryFilteringTestServlet.java +++ b/smoke-tests/apps/SamplingOverridesV1/src/main/java/com/microsoft/applicationinsights/smoketestapp/SamplingOverrideTestServlet.java @@ -34,7 +34,7 @@ import org.hsqldb.jdbc.JDBCDriver; @WebServlet("/*") -public class TelemetryFilteringTestServlet extends HttpServlet { +public class SamplingOverrideTestServlet extends HttpServlet { public void init() throws ServletException { try { diff --git a/smoke-tests/apps/TelemetryFiltering/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/TelemetryFiltering2SmokeTest.java b/smoke-tests/apps/SamplingOverridesV1/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/SamplingOverride2Test.java similarity index 78% rename from smoke-tests/apps/TelemetryFiltering/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/TelemetryFiltering2SmokeTest.java rename to smoke-tests/apps/SamplingOverridesV1/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/SamplingOverride2Test.java index dc0c44434ea..26d08f9dd98 100644 --- a/smoke-tests/apps/TelemetryFiltering/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/TelemetryFiltering2SmokeTest.java +++ b/smoke-tests/apps/SamplingOverridesV1/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/SamplingOverride2Test.java @@ -34,8 +34,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; -@UseAgent("telemetryfiltering2_applicationinsights.json") -abstract class TelemetryFiltering2SmokeTest { +@UseAgent("applicationinsights2.json") +abstract class SamplingOverride2Test { @RegisterExtension static final SmokeTestExtension testing = SmokeTestExtension.create(); @@ -47,29 +47,29 @@ void testSampling() throws Exception { } @Environment(TOMCAT_8_JAVA_8) - static class Tomcat8Java8Test extends TelemetryFiltering2SmokeTest {} + static class Tomcat8Java8Test extends SamplingOverride2Test {} @Environment(TOMCAT_8_JAVA_8_OPENJ9) - static class Tomcat8Java8OpenJ9Test extends TelemetryFiltering2SmokeTest {} + static class Tomcat8Java8OpenJ9Test extends SamplingOverride2Test {} @Environment(TOMCAT_8_JAVA_11) - static class Tomcat8Java11Test extends TelemetryFiltering2SmokeTest {} + static class Tomcat8Java11Test extends SamplingOverride2Test {} @Environment(TOMCAT_8_JAVA_11_OPENJ9) - static class Tomcat8Java11OpenJ9Test extends TelemetryFiltering2SmokeTest {} + static class Tomcat8Java11OpenJ9Test extends SamplingOverride2Test {} @Environment(TOMCAT_8_JAVA_17) - static class Tomcat8Java17Test extends TelemetryFiltering2SmokeTest {} + static class Tomcat8Java17Test extends SamplingOverride2Test {} @Environment(TOMCAT_8_JAVA_18) - static class Tomcat8Java18Test extends TelemetryFiltering2SmokeTest {} + static class Tomcat8Java18Test extends SamplingOverride2Test {} @Environment(TOMCAT_8_JAVA_19) - static class Tomcat8Java19Test extends TelemetryFiltering2SmokeTest {} + static class Tomcat8Java19Test extends SamplingOverride2Test {} @Environment(WILDFLY_13_JAVA_8) - static class Wildfly13Java8Test extends TelemetryFiltering2SmokeTest {} + static class Wildfly13Java8Test extends SamplingOverride2Test {} @Environment(WILDFLY_13_JAVA_8_OPENJ9) - static class Wildfly13Java8OpenJ9Test extends TelemetryFiltering2SmokeTest {} + static class Wildfly13Java8OpenJ9Test extends SamplingOverride2Test {} } diff --git a/smoke-tests/apps/TelemetryFiltering/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/TelemetryFilteringSmokeTest.java b/smoke-tests/apps/SamplingOverridesV1/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/SamplingOverrideTest.java similarity index 88% rename from smoke-tests/apps/TelemetryFiltering/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/TelemetryFilteringSmokeTest.java rename to smoke-tests/apps/SamplingOverridesV1/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/SamplingOverrideTest.java index 49af2702c31..440517680c0 100644 --- a/smoke-tests/apps/TelemetryFiltering/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/TelemetryFilteringSmokeTest.java +++ b/smoke-tests/apps/SamplingOverridesV1/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/SamplingOverrideTest.java @@ -42,8 +42,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; -@UseAgent("telemetryfiltering_applicationinsights.json") -abstract class TelemetryFilteringSmokeTest { +@UseAgent("applicationinsights.json") +abstract class SamplingOverrideTest { @RegisterExtension static final SmokeTestExtension testing = SmokeTestExtension.create(); @@ -117,33 +117,33 @@ void testRegularJdbc() throws Exception { assertThat(rddEnvelope.getTags()).containsEntry("ai.cloud.role", "app3"); assertThat(rdd.getSuccess()).isTrue(); - SmokeTestExtension.assertParentChild(rd, rdEnvelope, rddEnvelope, "GET /TelemetryFiltering/*"); + SmokeTestExtension.assertParentChild(rd, rdEnvelope, rddEnvelope, "GET /SamplingOverridesV1/*"); } @Environment(TOMCAT_8_JAVA_8) - static class Tomcat8Java8Test extends TelemetryFilteringSmokeTest {} + static class Tomcat8Java8Test extends SamplingOverrideTest {} @Environment(TOMCAT_8_JAVA_8_OPENJ9) - static class Tomcat8Java8OpenJ9Test extends TelemetryFilteringSmokeTest {} + static class Tomcat8Java8OpenJ9Test extends SamplingOverrideTest {} @Environment(TOMCAT_8_JAVA_11) - static class Tomcat8Java11Test extends TelemetryFilteringSmokeTest {} + static class Tomcat8Java11Test extends SamplingOverrideTest {} @Environment(TOMCAT_8_JAVA_11_OPENJ9) - static class Tomcat8Java11OpenJ9Test extends TelemetryFilteringSmokeTest {} + static class Tomcat8Java11OpenJ9Test extends SamplingOverrideTest {} @Environment(TOMCAT_8_JAVA_17) - static class Tomcat8Java17Test extends TelemetryFilteringSmokeTest {} + static class Tomcat8Java17Test extends SamplingOverrideTest {} @Environment(TOMCAT_8_JAVA_18) - static class Tomcat8Java18Test extends TelemetryFilteringSmokeTest {} + static class Tomcat8Java18Test extends SamplingOverrideTest {} @Environment(TOMCAT_8_JAVA_19) - static class Tomcat8Java19Test extends TelemetryFilteringSmokeTest {} + static class Tomcat8Java19Test extends SamplingOverrideTest {} @Environment(WILDFLY_13_JAVA_8) - static class Wildfly13Java8Test extends TelemetryFilteringSmokeTest {} + static class Wildfly13Java8Test extends SamplingOverrideTest {} @Environment(WILDFLY_13_JAVA_8_OPENJ9) - static class Wildfly13Java8OpenJ9Test extends TelemetryFilteringSmokeTest {} + static class Wildfly13Java8OpenJ9Test extends SamplingOverrideTest {} } diff --git a/smoke-tests/apps/TelemetryFiltering/src/smokeTest/resources/telemetryfiltering_applicationinsights.json b/smoke-tests/apps/SamplingOverridesV1/src/smokeTest/resources/applicationinsights.json similarity index 77% rename from smoke-tests/apps/TelemetryFiltering/src/smokeTest/resources/telemetryfiltering_applicationinsights.json rename to smoke-tests/apps/SamplingOverridesV1/src/smokeTest/resources/applicationinsights.json index df0f38e13b0..b5a40dcbec3 100644 --- a/smoke-tests/apps/TelemetryFiltering/src/smokeTest/resources/telemetryfiltering_applicationinsights.json +++ b/smoke-tests/apps/SamplingOverridesV1/src/smokeTest/resources/applicationinsights.json @@ -1,4 +1,8 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, "sampling": { "percentage": 100 }, @@ -33,21 +37,21 @@ }, "instrumentationKeyOverrides": [ { - "httpPathPrefix": "/TelemetryFiltering/login", + "httpPathPrefix": "/SamplingOverridesV1/login", "instrumentationKey": "12345678-0000-0000-0000-0FEEDDADBEEF" }, { - "httpPathPrefix": "/TelemetryFiltering/regular-jdbc", + "httpPathPrefix": "/SamplingOverridesV1/regular-jdbc", "instrumentationKey": "87654321-0000-0000-0000-0FEEDDADBEEF" } ], "roleNameOverrides": [ { - "httpPathPrefix": "/TelemetryFiltering/login", + "httpPathPrefix": "/SamplingOverridesV1/login", "roleName": "app2" }, { - "httpPathPrefix": "/TelemetryFiltering/regular-jdbc", + "httpPathPrefix": "/SamplingOverridesV1/regular-jdbc", "roleName": "app3" } ] diff --git a/smoke-tests/apps/TelemetryFiltering/src/smokeTest/resources/telemetryfiltering2_applicationinsights.json b/smoke-tests/apps/SamplingOverridesV1/src/smokeTest/resources/applicationinsights2.json similarity index 82% rename from smoke-tests/apps/TelemetryFiltering/src/smokeTest/resources/telemetryfiltering2_applicationinsights.json rename to smoke-tests/apps/SamplingOverridesV1/src/smokeTest/resources/applicationinsights2.json index ed342f15ff9..d113136a128 100644 --- a/smoke-tests/apps/TelemetryFiltering/src/smokeTest/resources/telemetryfiltering2_applicationinsights.json +++ b/smoke-tests/apps/SamplingOverridesV1/src/smokeTest/resources/applicationinsights2.json @@ -1,4 +1,8 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, "sampling": { "percentage": 50 }, diff --git a/smoke-tests/apps/SamplingOverridesV1/src/smokeTest/resources/logback-test.xml b/smoke-tests/apps/SamplingOverridesV1/src/smokeTest/resources/logback-test.xml new file mode 100644 index 00000000000..0cbbecd57ce --- /dev/null +++ b/smoke-tests/apps/SamplingOverridesV1/src/smokeTest/resources/logback-test.xml @@ -0,0 +1,11 @@ + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} %-5level %logger{36} - %msg%n + + + + + + diff --git a/smoke-tests/apps/SamplingOverridesV2/build.gradle.kts b/smoke-tests/apps/SamplingOverridesV2/build.gradle.kts new file mode 100644 index 00000000000..1b1cfb106e9 --- /dev/null +++ b/smoke-tests/apps/SamplingOverridesV2/build.gradle.kts @@ -0,0 +1,7 @@ +plugins { + id("ai.smoke-test-war") +} + +dependencies { + implementation("org.hsqldb:hsqldb:2.5.1") +} diff --git a/smoke-tests/apps/SamplingOverridesV2/src/main/java/com/microsoft/applicationinsights/smoketestapp/SamplingOverrideTestServlet.java b/smoke-tests/apps/SamplingOverridesV2/src/main/java/com/microsoft/applicationinsights/smoketestapp/SamplingOverrideTestServlet.java new file mode 100644 index 00000000000..7097524bfa1 --- /dev/null +++ b/smoke-tests/apps/SamplingOverridesV2/src/main/java/com/microsoft/applicationinsights/smoketestapp/SamplingOverrideTestServlet.java @@ -0,0 +1,151 @@ +/* + * ApplicationInsights-Java + * Copyright (c) Microsoft Corporation + * All rights reserved. + * + * MIT License + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the ""Software""), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the following conditions: + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE + * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +package com.microsoft.applicationinsights.smoketestapp; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.concurrent.Callable; +import java.util.logging.Logger; +import javax.servlet.ServletException; +import javax.servlet.annotation.WebServlet; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import org.hsqldb.jdbc.JDBCDriver; + +@WebServlet("/*") +public class SamplingOverrideTestServlet extends HttpServlet { + + private static final Logger logger = Logger.getLogger("smoketestapp"); + + public void init() throws ServletException { + try { + setupHsqldb(); + } catch (Exception e) { + // surprisingly not all application servers seem to log init exceptions to stdout + e.printStackTrace(); + throw new ServletException(e); + } + } + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException { + try { + int statusCode = doGetInternal(req); + resp.getWriter().println(statusCode); + } catch (ServletException e) { + throw e; + } catch (Exception e) { + throw new ServletException(e); + } + } + + private int doGetInternal(HttpServletRequest req) throws Exception { + String pathInfo = req.getPathInfo(); + if (pathInfo.equals("/")) { + return 200; + } else if (pathInfo.equals("/health-check")) { + Connection connection = getHsqldbConnection(); + executeStatement(connection); + connection.close(); + logger.info("always capture me"); + return 200; + } else if (pathInfo.equals("/login")) { + Connection connection = getHsqldbConnection(); + executeStatement(connection); + connection.close(); + return 200; + } else if (pathInfo.equals("/noisy-jdbc")) { + Connection connection = getHsqldbConnection(); + executeNoisyStatement(connection); + connection.close(); + return 200; + } else if (pathInfo.equals("/regular-jdbc")) { + Connection connection = getHsqldbConnection(); + executeStatement(connection); + connection.close(); + return 200; + } else { + throw new ServletException("Unexpected url: " + pathInfo); + } + } + + private static Connection getHsqldbConnection() throws Exception { + return JDBCDriver.getConnection("jdbc:hsqldb:mem:testdb", null); + } + + private void executeStatement(Connection connection) throws SQLException { + Statement statement = connection.createStatement(); + ResultSet rs = statement.executeQuery("select * from abc"); + while (rs.next()) {} + rs.close(); + statement.close(); + } + + private void executeNoisyStatement(Connection connection) throws SQLException { + Statement statement = connection.createStatement(); + ResultSet rs = statement.executeQuery("select count(*) from abc"); + while (rs.next()) {} + rs.close(); + statement.close(); + } + + private static void setupHsqldb() throws Exception { + Connection connection = + getConnection( + new Callable() { + @Override + public Connection call() throws Exception { + return getHsqldbConnection(); + } + }); + setup(connection); + connection.close(); + } + + private static Connection getConnection(Callable callable) throws Exception { + Exception exception; + long startTimeMillis = System.currentTimeMillis(); + do { + try { + return callable.call(); + } catch (Exception e) { + exception = e; + } + } while (System.currentTimeMillis() - startTimeMillis < 30000); + throw exception; + } + + private static void setup(Connection connection) throws SQLException { + Statement statement = connection.createStatement(); + try { + statement.execute("create table abc (xyz varchar(10))"); + statement.execute("insert into abc (xyz) values ('x')"); + statement.execute("insert into abc (xyz) values ('y')"); + statement.execute("insert into abc (xyz) values ('z')"); + } finally { + statement.close(); + } + } +} diff --git a/smoke-tests/apps/SamplingOverridesV2/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/SamplingOverride2Test.java b/smoke-tests/apps/SamplingOverridesV2/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/SamplingOverride2Test.java new file mode 100644 index 00000000000..26d08f9dd98 --- /dev/null +++ b/smoke-tests/apps/SamplingOverridesV2/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/SamplingOverride2Test.java @@ -0,0 +1,75 @@ +/* + * ApplicationInsights-Java + * Copyright (c) Microsoft Corporation + * All rights reserved. + * + * MIT License + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the ""Software""), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the following conditions: + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE + * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +package com.microsoft.applicationinsights.smoketest; + +import static com.microsoft.applicationinsights.smoketest.WarEnvironmentValue.TOMCAT_8_JAVA_11; +import static com.microsoft.applicationinsights.smoketest.WarEnvironmentValue.TOMCAT_8_JAVA_11_OPENJ9; +import static com.microsoft.applicationinsights.smoketest.WarEnvironmentValue.TOMCAT_8_JAVA_17; +import static com.microsoft.applicationinsights.smoketest.WarEnvironmentValue.TOMCAT_8_JAVA_18; +import static com.microsoft.applicationinsights.smoketest.WarEnvironmentValue.TOMCAT_8_JAVA_19; +import static com.microsoft.applicationinsights.smoketest.WarEnvironmentValue.TOMCAT_8_JAVA_8; +import static com.microsoft.applicationinsights.smoketest.WarEnvironmentValue.TOMCAT_8_JAVA_8_OPENJ9; +import static com.microsoft.applicationinsights.smoketest.WarEnvironmentValue.WILDFLY_13_JAVA_8; +import static com.microsoft.applicationinsights.smoketest.WarEnvironmentValue.WILDFLY_13_JAVA_8_OPENJ9; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +@UseAgent("applicationinsights2.json") +abstract class SamplingOverride2Test { + + @RegisterExtension static final SmokeTestExtension testing = SmokeTestExtension.create(); + + @Test + @TargetUri(value = "/login", callCount = 100) + void testSampling() throws Exception { + testing.mockedIngestion.waitForItems("RequestData", 100); + testing.mockedIngestion.waitForItems("RemoteDependencyData", 100); + } + + @Environment(TOMCAT_8_JAVA_8) + static class Tomcat8Java8Test extends SamplingOverride2Test {} + + @Environment(TOMCAT_8_JAVA_8_OPENJ9) + static class Tomcat8Java8OpenJ9Test extends SamplingOverride2Test {} + + @Environment(TOMCAT_8_JAVA_11) + static class Tomcat8Java11Test extends SamplingOverride2Test {} + + @Environment(TOMCAT_8_JAVA_11_OPENJ9) + static class Tomcat8Java11OpenJ9Test extends SamplingOverride2Test {} + + @Environment(TOMCAT_8_JAVA_17) + static class Tomcat8Java17Test extends SamplingOverride2Test {} + + @Environment(TOMCAT_8_JAVA_18) + static class Tomcat8Java18Test extends SamplingOverride2Test {} + + @Environment(TOMCAT_8_JAVA_19) + static class Tomcat8Java19Test extends SamplingOverride2Test {} + + @Environment(WILDFLY_13_JAVA_8) + static class Wildfly13Java8Test extends SamplingOverride2Test {} + + @Environment(WILDFLY_13_JAVA_8_OPENJ9) + static class Wildfly13Java8OpenJ9Test extends SamplingOverride2Test {} +} diff --git a/smoke-tests/apps/SamplingOverridesV2/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/SamplingOverrideTest.java b/smoke-tests/apps/SamplingOverridesV2/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/SamplingOverrideTest.java new file mode 100644 index 00000000000..346d6235abb --- /dev/null +++ b/smoke-tests/apps/SamplingOverridesV2/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/SamplingOverrideTest.java @@ -0,0 +1,151 @@ +/* + * ApplicationInsights-Java + * Copyright (c) Microsoft Corporation + * All rights reserved. + * + * MIT License + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the ""Software""), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the following conditions: + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE + * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +package com.microsoft.applicationinsights.smoketest; + +import static com.microsoft.applicationinsights.smoketest.WarEnvironmentValue.TOMCAT_8_JAVA_11; +import static com.microsoft.applicationinsights.smoketest.WarEnvironmentValue.TOMCAT_8_JAVA_11_OPENJ9; +import static com.microsoft.applicationinsights.smoketest.WarEnvironmentValue.TOMCAT_8_JAVA_17; +import static com.microsoft.applicationinsights.smoketest.WarEnvironmentValue.TOMCAT_8_JAVA_18; +import static com.microsoft.applicationinsights.smoketest.WarEnvironmentValue.TOMCAT_8_JAVA_19; +import static com.microsoft.applicationinsights.smoketest.WarEnvironmentValue.TOMCAT_8_JAVA_8; +import static com.microsoft.applicationinsights.smoketest.WarEnvironmentValue.TOMCAT_8_JAVA_8_OPENJ9; +import static com.microsoft.applicationinsights.smoketest.WarEnvironmentValue.WILDFLY_13_JAVA_8; +import static com.microsoft.applicationinsights.smoketest.WarEnvironmentValue.WILDFLY_13_JAVA_8_OPENJ9; +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.assertj.core.api.Assertions.assertThat; + +import com.microsoft.applicationinsights.smoketest.schemav2.Data; +import com.microsoft.applicationinsights.smoketest.schemav2.Envelope; +import com.microsoft.applicationinsights.smoketest.schemav2.RemoteDependencyData; +import com.microsoft.applicationinsights.smoketest.schemav2.RequestData; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +@UseAgent("applicationinsights.json") +abstract class SamplingOverrideTest { + + @RegisterExtension static final SmokeTestExtension testing = SmokeTestExtension.create(); + + @Test + @TargetUri(value = "/health-check", callCount = 100) + void testSampling() throws Exception { + // super super low chance that number of sampled requests is less than 25 + long start = System.nanoTime(); + while (testing.mockedIngestion.getCountForType("RequestData") < 25 + && NANOSECONDS.toSeconds(System.nanoTime() - start) < 10) {} + // wait ten more seconds to before checking that we didn't receive too many + Thread.sleep(SECONDS.toMillis(10)); + int requestCount = testing.mockedIngestion.getCountForType("RequestData"); + int dependencyCount = testing.mockedIngestion.getCountForType("RemoteDependencyData"); + int logCount = testing.mockedIngestion.getCountForType("MessageData"); + // super super low chance that number of sampled requests/dependencies + // is less than 25 or greater than 75 + assertThat(requestCount).isGreaterThanOrEqualTo(25); + assertThat(dependencyCount).isGreaterThanOrEqualTo(25); + assertThat(requestCount).isLessThanOrEqualTo(75); + assertThat(dependencyCount).isLessThanOrEqualTo(75); + assertThat(logCount).isEqualTo(100); + } + + @Test + @TargetUri("/noisy-jdbc") + void testNoisyJdbc() throws Exception { + List rdList = testing.mockedIngestion.waitForItems("RequestData", 1); + Thread.sleep(10000); + assertThat(testing.mockedIngestion.getCountForType("RemoteDependencyData")).isZero(); + + Envelope rdEnvelope = rdList.get(0); + + assertThat(rdEnvelope.getSampleRate()).isNull(); + + RequestData rd = (RequestData) ((Data) rdEnvelope.getData()).getBaseData(); + + assertThat(rdEnvelope.getIKey()).isEqualTo("00000000-0000-0000-0000-0FEEDDADBEEF"); + assertThat(rdEnvelope.getTags()).containsEntry("ai.cloud.role", "testrolename"); + assertThat(rd.getSuccess()).isTrue(); + } + + @Test + @TargetUri("/regular-jdbc") + void testRegularJdbc() throws Exception { + List rdList = testing.mockedIngestion.waitForItems("RequestData", 1); + + Envelope rdEnvelope = rdList.get(0); + String operationId = rdEnvelope.getTags().get("ai.operation.id"); + + List rddList = + testing.mockedIngestion.waitForItemsInOperation("RemoteDependencyData", 1, operationId); + assertThat(testing.mockedIngestion.getCountForType("EventData")).isZero(); + + Envelope rddEnvelope = rddList.get(0); + + assertThat(rdEnvelope.getSampleRate()).isNull(); + assertThat(rddEnvelope.getSampleRate()).isNull(); + + RequestData rd = (RequestData) ((Data) rdEnvelope.getData()).getBaseData(); + RemoteDependencyData rdd = + (RemoteDependencyData) ((Data) rddEnvelope.getData()).getBaseData(); + + assertThat(rdEnvelope.getIKey()).isEqualTo("87654321-0000-0000-0000-0FEEDDADBEEF"); + assertThat(rdEnvelope.getTags()).containsEntry("ai.cloud.role", "app3"); + assertThat(rd.getSuccess()).isTrue(); + + assertThat(rdd.getType()).isEqualTo("SQL"); + assertThat(rdd.getTarget()).isEqualTo("testdb"); + assertThat(rdd.getName()).isEqualTo("SELECT testdb.abc"); + assertThat(rdd.getData()).isEqualTo("select * from abc"); + assertThat(rddEnvelope.getIKey()).isEqualTo("87654321-0000-0000-0000-0FEEDDADBEEF"); + assertThat(rddEnvelope.getTags()).containsEntry("ai.cloud.role", "app3"); + assertThat(rdd.getSuccess()).isTrue(); + + SmokeTestExtension.assertParentChild(rd, rdEnvelope, rddEnvelope, "GET /SamplingOverridesV2/*"); + } + + @Environment(TOMCAT_8_JAVA_8) + static class Tomcat8Java8Test extends SamplingOverrideTest {} + + @Environment(TOMCAT_8_JAVA_8_OPENJ9) + static class Tomcat8Java8OpenJ9Test extends SamplingOverrideTest {} + + @Environment(TOMCAT_8_JAVA_11) + static class Tomcat8Java11Test extends SamplingOverrideTest {} + + @Environment(TOMCAT_8_JAVA_11_OPENJ9) + static class Tomcat8Java11OpenJ9Test extends SamplingOverrideTest {} + + @Environment(TOMCAT_8_JAVA_17) + static class Tomcat8Java17Test extends SamplingOverrideTest {} + + @Environment(TOMCAT_8_JAVA_18) + static class Tomcat8Java18Test extends SamplingOverrideTest {} + + @Environment(TOMCAT_8_JAVA_19) + static class Tomcat8Java19Test extends SamplingOverrideTest {} + + @Environment(WILDFLY_13_JAVA_8) + static class Wildfly13Java8Test extends SamplingOverrideTest {} + + @Environment(WILDFLY_13_JAVA_8_OPENJ9) + static class Wildfly13Java8OpenJ9Test extends SamplingOverrideTest {} +} diff --git a/smoke-tests/apps/SamplingOverridesV2/src/smokeTest/resources/applicationinsights.json b/smoke-tests/apps/SamplingOverridesV2/src/smokeTest/resources/applicationinsights.json new file mode 100644 index 00000000000..15b4374fcb2 --- /dev/null +++ b/smoke-tests/apps/SamplingOverridesV2/src/smokeTest/resources/applicationinsights.json @@ -0,0 +1,63 @@ +{ + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, + "preview": { + "sampling": { + "overrides": [ + { + "telemetryKind": "request", + "attributes": [ + { + "key": "http.url", + "value": ".*/health-check", + "matchType": "regexp" + } + ], + "percentage": 50, + "id": "filter out health check" + }, + { + "telemetryKind": "dependency", + "attributes": [ + { + "key": "db.statement", + "value": "select count(*) from abc", + "matchType": "strict" + } + ], + "percentage": 0, + "id": "filter out noisy jdbc" + }, + { + "telemetryKind": "trace", + "percentage": 100 + } + ] + }, + "instrumentationKeyOverrides": [ + { + "httpPathPrefix": "/SamplingOverridesV2/login", + "instrumentationKey": "12345678-0000-0000-0000-0FEEDDADBEEF" + }, + { + "httpPathPrefix": "/SamplingOverridesV2/regular-jdbc", + "instrumentationKey": "87654321-0000-0000-0000-0FEEDDADBEEF" + } + ], + "roleNameOverrides": [ + { + "httpPathPrefix": "/SamplingOverridesV2/login", + "roleName": "app2" + }, + { + "httpPathPrefix": "/SamplingOverridesV2/regular-jdbc", + "roleName": "app3" + } + ] + } +} diff --git a/smoke-tests/apps/SamplingOverridesV2/src/smokeTest/resources/applicationinsights2.json b/smoke-tests/apps/SamplingOverridesV2/src/smokeTest/resources/applicationinsights2.json new file mode 100644 index 00000000000..9027191d1e5 --- /dev/null +++ b/smoke-tests/apps/SamplingOverridesV2/src/smokeTest/resources/applicationinsights2.json @@ -0,0 +1,27 @@ +{ + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 50 + }, + "preview": { + "sampling": { + "overrides": [ + { + "telemetryKind": "request", + "attributes": [ + { + "key": "http.url", + "value": ".*/login", + "matchType": "regexp" + } + ], + "percentage": 100, + "id": "capture all login telemetry" + } + ] + } + } +} diff --git a/smoke-tests/apps/SamplingOverridesV2/src/smokeTest/resources/logback-test.xml b/smoke-tests/apps/SamplingOverridesV2/src/smokeTest/resources/logback-test.xml new file mode 100644 index 00000000000..0cbbecd57ce --- /dev/null +++ b/smoke-tests/apps/SamplingOverridesV2/src/smokeTest/resources/logback-test.xml @@ -0,0 +1,11 @@ + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} %-5level %logger{36} - %msg%n + + + + + + diff --git a/smoke-tests/apps/SpringBootAttachInMain/src/main/resources/applicationinsights.json b/smoke-tests/apps/SpringBootAttachInMain/src/main/resources/applicationinsights.json index b456737efd7..4c85dc10be2 100644 --- a/smoke-tests/apps/SpringBootAttachInMain/src/main/resources/applicationinsights.json +++ b/smoke-tests/apps/SpringBootAttachInMain/src/main/resources/applicationinsights.json @@ -1,7 +1,9 @@ { - "connectionString": "InstrumentationKey=00000000-0000-0000-0000-0FEEDDADBEEF;IngestionEndpoint=http://host.testcontainers.internal:6060/", "role": { "name": "testrolename", "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 } } diff --git a/smoke-tests/apps/SpringBootTest/src/smokeTest/resources/controller_spans_enabled_applicationinsights.json b/smoke-tests/apps/SpringBootTest/src/smokeTest/resources/controller_spans_enabled_applicationinsights.json index ce7adc74627..03e94410b60 100644 --- a/smoke-tests/apps/SpringBootTest/src/smokeTest/resources/controller_spans_enabled_applicationinsights.json +++ b/smoke-tests/apps/SpringBootTest/src/smokeTest/resources/controller_spans_enabled_applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "preview": { "captureControllerSpans": true } diff --git a/smoke-tests/apps/SpringCloudStream/src/smokeTest/resources/controller_spans_enabled_applicationinsights.json b/smoke-tests/apps/SpringCloudStream/src/smokeTest/resources/controller_spans_enabled_applicationinsights.json index ce7adc74627..03e94410b60 100644 --- a/smoke-tests/apps/SpringCloudStream/src/smokeTest/resources/controller_spans_enabled_applicationinsights.json +++ b/smoke-tests/apps/SpringCloudStream/src/smokeTest/resources/controller_spans_enabled_applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "preview": { "captureControllerSpans": true } diff --git a/smoke-tests/apps/SpringScheduling/src/smokeTest/resources/applicationinsights.json b/smoke-tests/apps/SpringScheduling/src/smokeTest/resources/applicationinsights.json new file mode 100644 index 00000000000..4c85dc10be2 --- /dev/null +++ b/smoke-tests/apps/SpringScheduling/src/smokeTest/resources/applicationinsights.json @@ -0,0 +1,9 @@ +{ + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + } +} diff --git a/smoke-tests/apps/SpringScheduling/src/smokeTest/resources/disabled_applicationinsights.json b/smoke-tests/apps/SpringScheduling/src/smokeTest/resources/disabled_applicationinsights.json index 97ee6eefe78..c9932fdbf01 100644 --- a/smoke-tests/apps/SpringScheduling/src/smokeTest/resources/disabled_applicationinsights.json +++ b/smoke-tests/apps/SpringScheduling/src/smokeTest/resources/disabled_applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "instrumentation": { "springScheduling": { "enabled": false diff --git a/smoke-tests/apps/Statsbeat/src/smokeTest/resources/applicationinsights.json b/smoke-tests/apps/Statsbeat/src/smokeTest/resources/applicationinsights.json index 91f1b4464ee..0a19c9afd7a 100644 --- a/smoke-tests/apps/Statsbeat/src/smokeTest/resources/applicationinsights.json +++ b/smoke-tests/apps/Statsbeat/src/smokeTest/resources/applicationinsights.json @@ -1,14 +1,21 @@ { - "preview": { - "statsbeat": { - "disabled": false - } + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 }, "customDimensions": { "tag1": "abc", "tag2": "def", "service.version": "123" }, + "preview": { + "statsbeat": { + "disabled": false + } + }, "internal": { "statsbeat": { "disabledAll": false, diff --git a/smoke-tests/apps/TelemetryProcessors/src/smokeTest/resources/applicationinsights.json b/smoke-tests/apps/TelemetryProcessors/src/smokeTest/resources/applicationinsights.json index 55e56076dbf..187398d9327 100644 --- a/smoke-tests/apps/TelemetryProcessors/src/smokeTest/resources/applicationinsights.json +++ b/smoke-tests/apps/TelemetryProcessors/src/smokeTest/resources/applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "preview": { "processors": [ { diff --git a/smoke-tests/apps/TraceJavaUtilLoggingUsingAgent/src/smokeTest/resources/applicationinsights.json b/smoke-tests/apps/TraceJavaUtilLoggingUsingAgent/src/smokeTest/resources/applicationinsights.json index 655b35a2365..96680a4fd60 100644 --- a/smoke-tests/apps/TraceJavaUtilLoggingUsingAgent/src/smokeTest/resources/applicationinsights.json +++ b/smoke-tests/apps/TraceJavaUtilLoggingUsingAgent/src/smokeTest/resources/applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "instrumentation": { "logging": { "level": "warn" diff --git a/smoke-tests/apps/TraceLog4j1_2/src/smokeTest/resources/applicationinsights.json b/smoke-tests/apps/TraceLog4j1_2/src/smokeTest/resources/applicationinsights.json index 655b35a2365..96680a4fd60 100644 --- a/smoke-tests/apps/TraceLog4j1_2/src/smokeTest/resources/applicationinsights.json +++ b/smoke-tests/apps/TraceLog4j1_2/src/smokeTest/resources/applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "instrumentation": { "logging": { "level": "warn" diff --git a/smoke-tests/apps/TraceLog4j1_2UsingAgent/src/smokeTest/resources/applicationinsights.json b/smoke-tests/apps/TraceLog4j1_2UsingAgent/src/smokeTest/resources/applicationinsights.json index 655b35a2365..96680a4fd60 100644 --- a/smoke-tests/apps/TraceLog4j1_2UsingAgent/src/smokeTest/resources/applicationinsights.json +++ b/smoke-tests/apps/TraceLog4j1_2UsingAgent/src/smokeTest/resources/applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "instrumentation": { "logging": { "level": "warn" diff --git a/smoke-tests/apps/TraceLog4j2/src/smokeTest/resources/applicationinsights.json b/smoke-tests/apps/TraceLog4j2/src/smokeTest/resources/applicationinsights.json index 655b35a2365..96680a4fd60 100644 --- a/smoke-tests/apps/TraceLog4j2/src/smokeTest/resources/applicationinsights.json +++ b/smoke-tests/apps/TraceLog4j2/src/smokeTest/resources/applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "instrumentation": { "logging": { "level": "warn" diff --git a/smoke-tests/apps/TraceLog4j2UsingAgent/src/smokeTest/resources/applicationinsights.json b/smoke-tests/apps/TraceLog4j2UsingAgent/src/smokeTest/resources/applicationinsights.json index 655b35a2365..96680a4fd60 100644 --- a/smoke-tests/apps/TraceLog4j2UsingAgent/src/smokeTest/resources/applicationinsights.json +++ b/smoke-tests/apps/TraceLog4j2UsingAgent/src/smokeTest/resources/applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "instrumentation": { "logging": { "level": "warn" diff --git a/smoke-tests/apps/TraceLogBack/src/smokeTest/resources/applicationinsights.json b/smoke-tests/apps/TraceLogBack/src/smokeTest/resources/applicationinsights.json index 655b35a2365..96680a4fd60 100644 --- a/smoke-tests/apps/TraceLogBack/src/smokeTest/resources/applicationinsights.json +++ b/smoke-tests/apps/TraceLogBack/src/smokeTest/resources/applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "instrumentation": { "logging": { "level": "warn" diff --git a/smoke-tests/apps/TraceLogBackUsingAgent/src/smokeTest/resources/applicationinsights.json b/smoke-tests/apps/TraceLogBackUsingAgent/src/smokeTest/resources/applicationinsights.json index 655b35a2365..96680a4fd60 100644 --- a/smoke-tests/apps/TraceLogBackUsingAgent/src/smokeTest/resources/applicationinsights.json +++ b/smoke-tests/apps/TraceLogBackUsingAgent/src/smokeTest/resources/applicationinsights.json @@ -1,4 +1,11 @@ { + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + }, "instrumentation": { "logging": { "level": "warn" diff --git a/smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/SmokeTestExtension.java b/smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/SmokeTestExtension.java index 7e791b674a9..f32ecd6331a 100644 --- a/smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/SmokeTestExtension.java +++ b/smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/SmokeTestExtension.java @@ -364,8 +364,6 @@ protected Set getLivenessCheckPorts() { "APPLICATIONINSIGHTS_CONNECTION_STRING", "InstrumentationKey=00000000-0000-0000-0000-0FEEDDADBEEF;" + "IngestionEndpoint=http://host.testcontainers.internal:6060/") - .withEnv("APPLICATIONINSIGHTS_ROLE_NAME", "testrolename") - .withEnv("APPLICATIONINSIGHTS_ROLE_INSTANCE", "testroleinstance") .withNetwork(network) .withExposedPorts(8080) .withFileSystemBind( @@ -390,16 +388,20 @@ protected Set getLivenessCheckPorts() { "/applicationinsights-agent.jar", BindMode.READ_ONLY); URL resource = SmokeTestExtension.class.getClassLoader().getResource(agentConfigurationPath); - if (resource != null) { - File json = File.createTempFile("applicationinsights", ".json"); - Path jsonPath = json.toPath(); - try (InputStream in = resource.openStream()) { - Files.copy(in, jsonPath, StandardCopyOption.REPLACE_EXISTING); - } - container = - container.withFileSystemBind( - json.getAbsolutePath(), "/applicationinsights.json", BindMode.READ_ONLY); + if (resource == null) { + resource = + SmokeTestExtension.class + .getClassLoader() + .getResource("default_applicationinsights.json"); } + File json = File.createTempFile("applicationinsights", ".json"); + Path jsonPath = json.toPath(); + try (InputStream in = resource.openStream()) { + Files.copy(in, jsonPath, StandardCopyOption.REPLACE_EXISTING); + } + container = + container.withFileSystemBind( + json.getAbsolutePath(), "/applicationinsights.json", BindMode.READ_ONLY); } if (appFile.getName().endsWith(".jar")) { diff --git a/smoke-tests/framework/src/main/resources/default_applicationinsights.json b/smoke-tests/framework/src/main/resources/default_applicationinsights.json new file mode 100644 index 00000000000..4c85dc10be2 --- /dev/null +++ b/smoke-tests/framework/src/main/resources/default_applicationinsights.json @@ -0,0 +1,9 @@ +{ + "role": { + "name": "testrolename", + "instance": "testroleinstance" + }, + "sampling": { + "percentage": 100 + } +} From 4a247368203e0269c9e34f9e277f3a1dc632f2eb Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Thu, 25 Aug 2022 11:02:15 -0700 Subject: [PATCH 02/13] Comment --- .../agent/internal/configuration/Configuration.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java index bc4be5ca35d..547fbdf3437 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java @@ -616,11 +616,13 @@ public static class SamplingOverride { public boolean isForRequestTelemetry() { return telemetryKind == TelemetryKind.REQUEST + // this part is for backwards compatibility: || (telemetryKind == null && spanKind != SpanKind.CLIENT); } public boolean isForDependencyTelemetry() { return telemetryKind == TelemetryKind.DEPENDENCY + // this part is for backwards compatibility: || (telemetryKind == null && spanKind != SpanKind.SERVER); } From 7e2986ade207637529b4f59b09561873e21ffc9d Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Thu, 25 Aug 2022 11:16:07 -0700 Subject: [PATCH 03/13] Comment --- .../agent/internal/exporter/AgentLogExporter.java | 1 + 1 file changed, 1 insertion(+) diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/AgentLogExporter.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/AgentLogExporter.java index 1372e4ea55a..e92d7fb3572 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/AgentLogExporter.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/AgentLogExporter.java @@ -173,6 +173,7 @@ private static boolean shouldSample(SpanContext spanContext, double percentage) if (spanContext.isValid()) { return AiSampler.shouldRecordAndSample(spanContext.getTraceId(), percentage); } + // this is a standalone log (not part of a trace), so randomly sample at the given percentage return ThreadLocalRandom.current().nextDouble() < percentage / 100; } } From 2a97137aeb2eb73f64d0bdbf9c105dad503ac655 Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Thu, 25 Aug 2022 11:18:36 -0700 Subject: [PATCH 04/13] Feedback --- .../internal/init/RpConfigurationPolling.java | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/RpConfigurationPolling.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/RpConfigurationPolling.java index 10662113733..079cd636f0a 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/RpConfigurationPolling.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/RpConfigurationPolling.java @@ -109,12 +109,24 @@ public void run() { appIdSupplier.updateAppId(); } + boolean changed = false; if (!Objects.equals( - newRpConfiguration.sampling.percentage, rpConfiguration.sampling.percentage) - || !Objects.equals( - newRpConfiguration.sampling.limitPerSecond, - rpConfiguration.sampling.limitPerSecond)) { - logger.debug("Updating sampling percentage"); + rpConfiguration.sampling.percentage, newRpConfiguration.sampling.percentage)) { + logger.debug( + "Updating sampling percentage from {} to {}", + rpConfiguration.sampling.percentage, + newRpConfiguration.sampling.percentage); + changed = true; + } + if (!Objects.equals( + rpConfiguration.sampling.limitPerSecond, newRpConfiguration.sampling.limitPerSecond)) { + logger.debug( + "Updating limit per second from {} to {}", + rpConfiguration.sampling.limitPerSecond, + newRpConfiguration.sampling.limitPerSecond); + changed = true; + } + if (changed) { configuration.sampling.percentage = newRpConfiguration.sampling.percentage; configuration.sampling.limitPerSecond = newRpConfiguration.sampling.limitPerSecond; DelegatingSampler.getInstance().setDelegate(Samplers.getSampler(configuration)); From e3b8fd8db88f42c83d1ef9c4a4e6115d72d62764 Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Thu, 25 Aug 2022 11:19:32 -0700 Subject: [PATCH 05/13] Private --- .../agent/internal/sampling/RateLimitedSamplingPercentage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/RateLimitedSamplingPercentage.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/RateLimitedSamplingPercentage.java index b15ee13f744..027a77c05a4 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/RateLimitedSamplingPercentage.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/RateLimitedSamplingPercentage.java @@ -40,7 +40,7 @@ private static final class State { private final double effectiveWindowNanos; private final long lastNanoTime; - public State(double effectiveWindowCount, double effectiveWindowNanos, long lastNanoTime) { + private State(double effectiveWindowCount, double effectiveWindowNanos, long lastNanoTime) { this.effectiveWindowCount = effectiveWindowCount; this.effectiveWindowNanos = effectiveWindowNanos; this.lastNanoTime = lastNanoTime; From be1f6905867feda8ce7d919b0cd2317ede1f1870 Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Thu, 25 Aug 2022 13:17:30 -0700 Subject: [PATCH 06/13] Link --- .../agent/internal/sampling/RateLimitedSamplingPercentage.java | 1 + 1 file changed, 1 insertion(+) diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/RateLimitedSamplingPercentage.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/RateLimitedSamplingPercentage.java index 027a77c05a4..427c3a8aa5b 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/RateLimitedSamplingPercentage.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/RateLimitedSamplingPercentage.java @@ -33,6 +33,7 @@ import java.util.function.LongSupplier; // uses adaptive algorithm from OpenTelemetry Java Contrib's ConsistentRateLimitingSampler +// (https://github.com/open-telemetry/opentelemetry-java-contrib/blob/main/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java) class RateLimitedSamplingPercentage implements SamplingPercentage { private static final class State { From d543f285234937fbb0b6bfda35f8fa8a6194dd63 Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Thu, 25 Aug 2022 20:36:18 -0700 Subject: [PATCH 07/13] Fix merge --- .../gRPC/src/smokeTest/resources/applicationinsights.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/smoke-tests/apps/gRPC/src/smokeTest/resources/applicationinsights.json b/smoke-tests/apps/gRPC/src/smokeTest/resources/applicationinsights.json index fabbaf6c881..df270eac2f8 100644 --- a/smoke-tests/apps/gRPC/src/smokeTest/resources/applicationinsights.json +++ b/smoke-tests/apps/gRPC/src/smokeTest/resources/applicationinsights.json @@ -1,10 +1,12 @@ { - "connectionString": "InstrumentationKey=00000000-0000-0000-0000-0FEEDDADBEEF;IngestionEndpoint=http://host.docker.internal:6060/", "role": { "name": "testrolename", "instance": "testroleinstance" }, + "sampling": { + "percentage": 100 + }, "preview": { "metricIntervalSeconds": 10 } -} \ No newline at end of file +} From ad0e017f19f70c6f04aefe42164fea00acc4bdeb Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Fri, 26 Aug 2022 12:19:38 -0700 Subject: [PATCH 08/13] Feedback --- .../internal/configuration/Configuration.java | 16 ++++++++-------- .../internal/exporter/AgentLogExporter.java | 3 +-- .../agent/internal/init/SecondEntryPoint.java | 6 +++--- .../internal/sampling/AiOverrideSampler.java | 4 ++-- .../sampling/RateLimitedSamplingPercentage.java | 2 ++ .../internal/sampling/SamplingOverrides.java | 16 ++++++++-------- 6 files changed, 24 insertions(+), 23 deletions(-) diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java index 547fbdf3437..99b14283416 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java @@ -94,7 +94,8 @@ public enum SpanKind { } } - public enum TelemetryKind { + public enum SamplingTelemetryKind { + // restricted to telemetry kinds that are supported by SamplingOverrides @JsonProperty("request") REQUEST, @JsonProperty("dependency") @@ -600,12 +601,11 @@ public static class SamplingOverride { // TODO (trask) make this required when moving out of preview // for now the default is both "request" and "dependency" for backwards compatibility - @Nullable public TelemetryKind telemetryKind; + @Nullable public SamplingTelemetryKind telemetryKind; - // TODO (trask) should this be named "standalone" (and meaning flipped) - // especially if we aren't going to change meaning of "request" to include unparented INTERNAL - // spans - public boolean inRequest = true; + // TODO (trask) add test for this + // this is primarily useful for batch jobs + public boolean includeStandaloneTelemetry; // not using include/exclude, because you can still get exclude with this by adding a second // (exclude) override above it @@ -615,13 +615,13 @@ public static class SamplingOverride { public String id; // optional, used for debugging purposes only public boolean isForRequestTelemetry() { - return telemetryKind == TelemetryKind.REQUEST + return telemetryKind == SamplingTelemetryKind.REQUEST // this part is for backwards compatibility: || (telemetryKind == null && spanKind != SpanKind.CLIENT); } public boolean isForDependencyTelemetry() { - return telemetryKind == TelemetryKind.DEPENDENCY + return telemetryKind == SamplingTelemetryKind.DEPENDENCY // this part is for backwards compatibility: || (telemetryKind == null && spanKind != SpanKind.SERVER); } diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/AgentLogExporter.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/AgentLogExporter.java index e92d7fb3572..e4c34a03568 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/AgentLogExporter.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/AgentLogExporter.java @@ -114,9 +114,8 @@ public CompletableResultCode export(Collection logs) { SpanContext spanContext = log.getSpanContext(); - boolean inRequest = spanContext.isValid(); Double samplingPercentage = - samplingOverrides.getOverridePercentage(inRequest, log.getAttributes()); + samplingOverrides.getOverridePercentage(spanContext.isValid(), log.getAttributes()); if (samplingPercentage != null && !shouldSample(spanContext, samplingPercentage)) { continue; diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java index fafb8c16b43..4c78bd0f35d 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java @@ -41,7 +41,7 @@ import com.microsoft.applicationinsights.agent.internal.common.FriendlyException; import com.microsoft.applicationinsights.agent.internal.configuration.Configuration; import com.microsoft.applicationinsights.agent.internal.configuration.Configuration.ProcessorConfig; -import com.microsoft.applicationinsights.agent.internal.configuration.Configuration.TelemetryKind; +import com.microsoft.applicationinsights.agent.internal.configuration.Configuration.SamplingTelemetryKind; import com.microsoft.applicationinsights.agent.internal.configuration.RpConfiguration; import com.microsoft.applicationinsights.agent.internal.exporter.AgentLogExporter; import com.microsoft.applicationinsights.agent.internal.exporter.AgentMetricExporter; @@ -490,11 +490,11 @@ private static LogExporter createLogExporter( List logSamplingOverrides = configuration.preview.sampling.overrides.stream() - .filter(override -> override.telemetryKind == TelemetryKind.TRACE) + .filter(override -> override.telemetryKind == SamplingTelemetryKind.TRACE) .collect(Collectors.toList()); List exceptionSamplingOverrides = configuration.preview.sampling.overrides.stream() - .filter(override -> override.telemetryKind == TelemetryKind.EXCEPTION) + .filter(override -> override.telemetryKind == SamplingTelemetryKind.EXCEPTION) .collect(Collectors.toList()); agentLogExporter = diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/AiOverrideSampler.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/AiOverrideSampler.java index 7904ded84ec..bc07b6e1891 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/AiOverrideSampler.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/AiOverrideSampler.java @@ -67,8 +67,8 @@ public SamplingResult shouldSample( SamplingOverrides samplingOverrides = isRequest ? requestSamplingOverrides : dependencySamplingOverrides; - boolean inRequest = isRequest || parentSpanContext.isValid(); - Sampler override = samplingOverrides.getOverride(inRequest, attributes); + boolean standaloneTelemetry = !isRequest && !parentSpanContext.isValid(); + Sampler override = samplingOverrides.getOverride(standaloneTelemetry, attributes); if (override != null) { return override.shouldSample(parentContext, traceId, name, spanKind, attributes, parentLinks); } diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/RateLimitedSamplingPercentage.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/RateLimitedSamplingPercentage.java index 427c3a8aa5b..d6a68a041fc 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/RateLimitedSamplingPercentage.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/RateLimitedSamplingPercentage.java @@ -34,6 +34,8 @@ // uses adaptive algorithm from OpenTelemetry Java Contrib's ConsistentRateLimitingSampler // (https://github.com/open-telemetry/opentelemetry-java-contrib/blob/main/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java) +// +// TODO see if we can port test over also class RateLimitedSamplingPercentage implements SamplingPercentage { private static final class State { diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverrides.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverrides.java index 4966ab2de38..274ab0a5c61 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverrides.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverrides.java @@ -47,10 +47,10 @@ public SamplingOverrides(List overrides) { } @Nullable - public Sampler getOverride(boolean inRequest, Attributes attributes) { + public Sampler getOverride(boolean standaloneTelemetry, Attributes attributes) { LazyHttpUrl lazyHttpUrl = new LazyHttpUrl(attributes); for (MatcherGroup matcherGroups : matcherGroups) { - if (matcherGroups.matches(inRequest, attributes, lazyHttpUrl)) { + if (matcherGroups.matches(standaloneTelemetry, attributes, lazyHttpUrl)) { return matcherGroups.getSampler(); } } @@ -59,9 +59,9 @@ public Sampler getOverride(boolean inRequest, Attributes attributes) { // used to do sampling inside the log exporter @Nullable - public Double getOverridePercentage(boolean inRequest, Attributes attributes) { + public Double getOverridePercentage(boolean standaloneTelemetry, Attributes attributes) { for (MatcherGroup matcherGroups : matcherGroups) { - if (matcherGroups.matches(inRequest, attributes, null)) { + if (matcherGroups.matches(standaloneTelemetry, attributes, null)) { return matcherGroups.getPercentage(); } } @@ -69,13 +69,13 @@ public Double getOverridePercentage(boolean inRequest, Attributes attributes) { } private static class MatcherGroup { - private final boolean inRequest; + private final boolean includeStandaloneTelemetry; private final List predicates; private final Sampler sampler; private final SamplingPercentage samplingPercentage; private MatcherGroup(SamplingOverride override) { - inRequest = override.inRequest; + includeStandaloneTelemetry = override.includeStandaloneTelemetry; predicates = new ArrayList<>(); for (SamplingOverrideAttribute attribute : override.attributes) { predicates.add(toPredicate(attribute)); @@ -93,8 +93,8 @@ Sampler getSampler() { } private boolean matches( - boolean inRequest, Attributes attributes, @Nullable LazyHttpUrl lazyHttpUrl) { - if (this.inRequest != inRequest) { + boolean standaloneTelemetry, Attributes attributes, @Nullable LazyHttpUrl lazyHttpUrl) { + if (standaloneTelemetry && !this.includeStandaloneTelemetry) { return false; } for (TempPredicate predicate : predicates) { From 4758211cf2d712f9f272aff1bf0e426020185e93 Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Fri, 26 Aug 2022 14:14:11 -0700 Subject: [PATCH 09/13] Fix --- .../sampling/SamplingOverridesTest.java | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverridesTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverridesTest.java index 066fc8a7d0d..a5ed5dee817 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverridesTest.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverridesTest.java @@ -46,7 +46,7 @@ void shouldSampleByDefault() { Attributes attributes = Attributes.empty(); // when - Sampler sampler = samplingOverrides.getOverride(true, attributes); + Sampler sampler = samplingOverrides.getOverride(false, attributes); // expect assertThat(sampler).isNull(); @@ -60,7 +60,7 @@ void shouldFilterInRequest() { Attributes attributes = Attributes.empty(); // when - Sampler sampler = samplingOverrides.getOverride(true, attributes); + Sampler sampler = samplingOverrides.getOverride(false, attributes); // expect assertThat(sampler).isNotNull(); @@ -68,14 +68,14 @@ void shouldFilterInRequest() { } @Test - void shouldNotFilterInRequest() { + void shouldNotFilterStandaloneTelemetry() { // given List overrides = singletonList(newOverride(25)); SamplingOverrides samplingOverrides = new SamplingOverrides(overrides); Attributes attributes = Attributes.empty(); // when - Sampler sampler = samplingOverrides.getOverride(false, attributes); + Sampler sampler = samplingOverrides.getOverride(true, attributes); // expect assertThat(sampler).isNull(); @@ -90,7 +90,7 @@ void shouldFilterStrictMatch() { Attributes attributes = Attributes.of(AttributeKey.stringKey("one"), "1"); // when - Sampler sampler = samplingOverrides.getOverride(true, attributes); + Sampler sampler = samplingOverrides.getOverride(false, attributes); // expect assertThat(sampler).isNotNull(); @@ -106,7 +106,7 @@ void shouldNotFilterStrictMatch() { Attributes attributes = Attributes.of(AttributeKey.stringKey("one"), "2"); // when - Sampler sampler = samplingOverrides.getOverride(true, attributes); + Sampler sampler = samplingOverrides.getOverride(false, attributes); // expect assertThat(sampler).isNull(); @@ -121,7 +121,7 @@ void shouldNotFilterMissingStrictMatch() { Attributes attributes = Attributes.of(AttributeKey.stringKey("two"), "1"); // when - Sampler sampler = samplingOverrides.getOverride(true, attributes); + Sampler sampler = samplingOverrides.getOverride(false, attributes); // expect assertThat(sampler).isNull(); @@ -136,7 +136,7 @@ void shouldFilterRegexpMatch() { Attributes attributes = Attributes.of(AttributeKey.stringKey("one"), "11"); // when - Sampler sampler = samplingOverrides.getOverride(true, attributes); + Sampler sampler = samplingOverrides.getOverride(false, attributes); // expect assertThat(sampler).isNotNull(); @@ -152,7 +152,7 @@ void shouldNotFilterRegexpMatch() { Attributes attributes = Attributes.of(AttributeKey.stringKey("one"), "22"); // when - Sampler sampler = samplingOverrides.getOverride(true, attributes); + Sampler sampler = samplingOverrides.getOverride(false, attributes); // expect assertThat(sampler).isNull(); @@ -167,7 +167,7 @@ void shouldNotFilterMissingRegexpMatch() { Attributes attributes = Attributes.of(AttributeKey.stringKey("two"), "11"); // when - Sampler sampler = samplingOverrides.getOverride(true, attributes); + Sampler sampler = samplingOverrides.getOverride(false, attributes); // expect assertThat(sampler).isNull(); @@ -181,7 +181,7 @@ void shouldFilterKeyOnlyMatch() { Attributes attributes = Attributes.of(AttributeKey.stringKey("one"), "11"); // when - Sampler sampler = samplingOverrides.getOverride(true, attributes); + Sampler sampler = samplingOverrides.getOverride(false, attributes); // expect assertThat(sampler).isNotNull(); @@ -196,7 +196,7 @@ void shouldNotFilterKeyOnlyMatch() { Attributes attributes = Attributes.of(AttributeKey.stringKey("two"), "22"); // when - Sampler sampler = samplingOverrides.getOverride(true, attributes); + Sampler sampler = samplingOverrides.getOverride(false, attributes); // expect assertThat(sampler).isNull(); @@ -213,7 +213,7 @@ void shouldFilterMultiAttributes() { Attributes.of(AttributeKey.stringKey("one"), "1", AttributeKey.stringKey("two"), "22"); // when - Sampler sampler = samplerOverride.getOverride(true, attributes); + Sampler sampler = samplerOverride.getOverride(false, attributes); // expect assertThat(sampler).isNotNull(); @@ -231,7 +231,7 @@ void shouldNotFilterMultiAttributes() { Attributes.of(AttributeKey.stringKey("one"), "2", AttributeKey.stringKey("two"), "22"); // when - Sampler sampler = samplingOverrides.getOverride(true, attributes); + Sampler sampler = samplingOverrides.getOverride(false, attributes); // expect assertThat(sampler).isNull(); @@ -249,7 +249,7 @@ void shouldFilterMultiConfigsBothMatch() { Attributes.of(AttributeKey.stringKey("one"), "1", AttributeKey.stringKey("two"), "22"); // when - Sampler sampler = samplingOverrides.getOverride(true, attributes); + Sampler sampler = samplingOverrides.getOverride(false, attributes); // expect assertThat(sampler).isNotNull(); @@ -268,7 +268,7 @@ void shouldFilterMultiConfigsOneMatch() { Attributes.of(AttributeKey.stringKey("one"), "2", AttributeKey.stringKey("two"), "22"); // when - Sampler sampler = samplingOverrides.getOverride(true, attributes); + Sampler sampler = samplingOverrides.getOverride(false, attributes); // expect assertThat(sampler).isNotNull(); @@ -287,7 +287,7 @@ void shouldNotFilterMultiConfigsNoMatch() { Attributes.of(AttributeKey.stringKey("one"), "2", AttributeKey.stringKey("two"), "33"); // when - Sampler sampler = samplingOverrides.getOverride(true, attributes); + Sampler sampler = samplingOverrides.getOverride(false, attributes); // expect assertThat(sampler).isNull(); From 3641cbe53deb0a2fe801338ac1b7b872e4476844 Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Fri, 26 Aug 2022 15:04:10 -0700 Subject: [PATCH 10/13] Fix --- .../agent/internal/exporter/AgentLogExporter.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/AgentLogExporter.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/AgentLogExporter.java index e4c34a03568..d8924016758 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/AgentLogExporter.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/AgentLogExporter.java @@ -114,15 +114,16 @@ public CompletableResultCode export(Collection logs) { SpanContext spanContext = log.getSpanContext(); + boolean standaloneLog = !spanContext.isValid(); Double samplingPercentage = - samplingOverrides.getOverridePercentage(spanContext.isValid(), log.getAttributes()); + samplingOverrides.getOverridePercentage(standaloneLog, log.getAttributes()); if (samplingPercentage != null && !shouldSample(spanContext, samplingPercentage)) { continue; } if (samplingPercentage == null - && spanContext.isValid() + && !standaloneLog && !spanContext.getTraceFlags().isSampled()) { // if there is no sampling override, and the log is part of an unsampled trace, then don't // capture it From 0b2afed13ca2403dfcf55bf6158db1b744fe7642 Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Fri, 26 Aug 2022 16:43:12 -0700 Subject: [PATCH 11/13] Fix --- .../applicationinsights/smoketest/SamplingOverrideTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smoke-tests/apps/SamplingOverridesV2/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/SamplingOverrideTest.java b/smoke-tests/apps/SamplingOverridesV2/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/SamplingOverrideTest.java index 346d6235abb..3d279f650aa 100644 --- a/smoke-tests/apps/SamplingOverridesV2/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/SamplingOverrideTest.java +++ b/smoke-tests/apps/SamplingOverridesV2/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/SamplingOverrideTest.java @@ -112,7 +112,7 @@ void testRegularJdbc() throws Exception { assertThat(rd.getSuccess()).isTrue(); assertThat(rdd.getType()).isEqualTo("SQL"); - assertThat(rdd.getTarget()).isEqualTo("testdb"); + assertThat(rdd.getTarget()).isEqualTo("hsqldb | testdb"); assertThat(rdd.getName()).isEqualTo("SELECT testdb.abc"); assertThat(rdd.getData()).isEqualTo("select * from abc"); assertThat(rddEnvelope.getIKey()).isEqualTo("87654321-0000-0000-0000-0FEEDDADBEEF"); From 48b3d8eb99058b9a0411dc3a8e1fc40717ca5e00 Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Sat, 27 Aug 2022 19:30:51 -0700 Subject: [PATCH 12/13] Add test --- .../RateLimitedSamplingPercentage.java | 18 +- .../RateLimitedSamplingPercentageTest.java | 195 ++++++++++++++++++ 2 files changed, 207 insertions(+), 6 deletions(-) create mode 100644 agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/sampling/RateLimitedSamplingPercentageTest.java diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/RateLimitedSamplingPercentage.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/RateLimitedSamplingPercentage.java index d6a68a041fc..151d6447808 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/RateLimitedSamplingPercentage.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/RateLimitedSamplingPercentage.java @@ -34,8 +34,6 @@ // uses adaptive algorithm from OpenTelemetry Java Contrib's ConsistentRateLimitingSampler // (https://github.com/open-telemetry/opentelemetry-java-contrib/blob/main/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java) -// -// TODO see if we can port test over also class RateLimitedSamplingPercentage implements SamplingPercentage { private static final class State { @@ -54,15 +52,18 @@ private State(double effectiveWindowCount, double effectiveWindowNanos, long las private final double inverseAdaptationTimeNanos; private final double targetSpansPerNanosecondLimit; private final AtomicReference state; + private final boolean roundToNearest; RateLimitedSamplingPercentage(double targetSpansPerSecondLimit, double adaptationTimeSeconds) { - this(targetSpansPerSecondLimit, adaptationTimeSeconds, System::nanoTime); + this(targetSpansPerSecondLimit, adaptationTimeSeconds, System::nanoTime, true); } - private RateLimitedSamplingPercentage( + // visible for testing + RateLimitedSamplingPercentage( double targetSpansPerSecondLimit, double adaptationTimeSeconds, - LongSupplier nanoTimeSupplier) { + LongSupplier nanoTimeSupplier, + boolean roundToNearest) { if (targetSpansPerSecondLimit < 0.0) { throw new IllegalArgumentException("Limit for sampled spans per second must be nonnegative!"); @@ -76,6 +77,8 @@ private RateLimitedSamplingPercentage( this.targetSpansPerNanosecondLimit = 1e-9 * targetSpansPerSecondLimit; this.state = new AtomicReference<>(new State(0, 0, nanoTimeSupplier.getAsLong())); + + this.roundToNearest = roundToNearest; } private State updateState(State oldState, long currentNanoTime) { @@ -102,7 +105,10 @@ public double get() { double samplingPercentage = 100 * Math.min(samplingProbability, 1); - return roundDownToNearest(samplingPercentage); + if (roundToNearest) { + samplingPercentage = roundDownToNearest(samplingPercentage); + } + return samplingPercentage; } private static double roundDownToNearest(double samplingPercentage) { diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/sampling/RateLimitedSamplingPercentageTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/sampling/RateLimitedSamplingPercentageTest.java new file mode 100644 index 00000000000..ba4eac24181 --- /dev/null +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/sampling/RateLimitedSamplingPercentageTest.java @@ -0,0 +1,195 @@ +/* + * ApplicationInsights-Java + * Copyright (c) Microsoft Corporation + * All rights reserved. + * + * MIT License + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the ""Software""), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the following conditions: + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE + * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +// Includes work from: +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.microsoft.applicationinsights.agent.internal.sampling; + +import org.assertj.core.data.Percentage; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.function.LongSupplier; + +import static org.assertj.core.api.Assertions.assertThat; + +// uses tests from OpenTelemetry Java Contrib's ConsistentRateLimitingSampler +// (https://github.com/open-telemetry/opentelemetry-java-contrib/blob/main/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSamplerTest.java) +class RateLimitedSamplingPercentageTest { + + private long[] nanoTime; + private LongSupplier nanoTimeSupplier; + + @BeforeEach + void init() { + nanoTime = new long[] {0L}; + nanoTimeSupplier = () -> nanoTime[0]; + } + + private void advanceTime(long nanosIncrement) { + nanoTime[0] += nanosIncrement; + } + + private long getCurrentTimeNanos() { + return nanoTime[0]; + } + + @Test + void testConstantRate() { + + double targetSpansPerSecondLimit = 1000; + double adaptationTimeSeconds = 5; + + RateLimitedSamplingPercentage samplingPercentage = + new RateLimitedSamplingPercentage( + targetSpansPerSecondLimit, adaptationTimeSeconds, nanoTimeSupplier, false); + + long nanosBetweenSpans = TimeUnit.MICROSECONDS.toNanos(100); + int numSpans = 1000000; + + List spanSampledNanos = new ArrayList<>(); + + for (int i = 0; i < numSpans; ++i) { + advanceTime(nanosBetweenSpans); + if (ThreadLocalRandom.current().nextDouble() < samplingPercentage.get() / 100) { + spanSampledNanos.add(getCurrentTimeNanos()); + } + } + + long numSampledSpansInLast5Seconds = + spanSampledNanos.stream() + .filter(x -> x > TimeUnit.SECONDS.toNanos(95) && x <= TimeUnit.SECONDS.toNanos(100)) + .count(); + + assertThat(numSampledSpansInLast5Seconds / 5.) + .isCloseTo(targetSpansPerSecondLimit, Percentage.withPercentage(5)); + } + + @Test + void testRateIncrease() { + + double targetSpansPerSecondLimit = 1000; + double adaptationTimeSeconds = 5; + + RateLimitedSamplingPercentage samplingPercentage = + new RateLimitedSamplingPercentage( + targetSpansPerSecondLimit, adaptationTimeSeconds, nanoTimeSupplier, false); + + long nanosBetweenSpans1 = TimeUnit.MICROSECONDS.toNanos(100); + long nanosBetweenSpans2 = TimeUnit.MICROSECONDS.toNanos(10); + int numSpans1 = 500000; + int numSpans2 = 5000000; + + List spanSampledNanos = new ArrayList<>(); + + for (int i = 0; i < numSpans1; ++i) { + advanceTime(nanosBetweenSpans1); + if (ThreadLocalRandom.current().nextDouble() < samplingPercentage.get() / 100) { + spanSampledNanos.add(getCurrentTimeNanos()); + } + } + for (int i = 0; i < numSpans2; ++i) { + advanceTime(nanosBetweenSpans2); + if (ThreadLocalRandom.current().nextDouble() < samplingPercentage.get() / 100) { + spanSampledNanos.add(getCurrentTimeNanos()); + } + } + + long numSampledSpansWithin5SecondsBeforeChange = + spanSampledNanos.stream() + .filter(x -> x > TimeUnit.SECONDS.toNanos(45) && x <= TimeUnit.SECONDS.toNanos(50)) + .count(); + long numSampledSpansWithin5SecondsAfterChange = + spanSampledNanos.stream() + .filter(x -> x > TimeUnit.SECONDS.toNanos(50) && x <= TimeUnit.SECONDS.toNanos(55)) + .count(); + long numSampledSpansInLast5Seconds = + spanSampledNanos.stream() + .filter(x -> x > TimeUnit.SECONDS.toNanos(95) && x <= TimeUnit.SECONDS.toNanos(100)) + .count(); + + assertThat(numSampledSpansWithin5SecondsBeforeChange / 5.) + .isCloseTo(targetSpansPerSecondLimit, Percentage.withPercentage(5)); + assertThat(numSampledSpansWithin5SecondsAfterChange / 5.) + .isGreaterThan(2. * targetSpansPerSecondLimit); + assertThat(numSampledSpansInLast5Seconds / 5.) + .isCloseTo(targetSpansPerSecondLimit, Percentage.withPercentage(5)); + } + + @Test + void testRateDecrease() { + + double targetSpansPerSecondLimit = 1000; + double adaptationTimeSeconds = 5; + + RateLimitedSamplingPercentage samplingPercentage = + new RateLimitedSamplingPercentage( + targetSpansPerSecondLimit, adaptationTimeSeconds, nanoTimeSupplier, false); + + long nanosBetweenSpans1 = TimeUnit.MICROSECONDS.toNanos(10); + long nanosBetweenSpans2 = TimeUnit.MICROSECONDS.toNanos(100); + int numSpans1 = 5000000; + int numSpans2 = 500000; + + List spanSampledNanos = new ArrayList<>(); + + for (int i = 0; i < numSpans1; ++i) { + advanceTime(nanosBetweenSpans1); + if (ThreadLocalRandom.current().nextDouble() < samplingPercentage.get() / 100) { + spanSampledNanos.add(getCurrentTimeNanos()); + } + } + for (int i = 0; i < numSpans2; ++i) { + advanceTime(nanosBetweenSpans2); + if (ThreadLocalRandom.current().nextDouble() < samplingPercentage.get() / 100) { + spanSampledNanos.add(getCurrentTimeNanos()); + } + } + + long numSampledSpansWithin5SecondsBeforeChange = + spanSampledNanos.stream() + .filter(x -> x > TimeUnit.SECONDS.toNanos(45) && x <= TimeUnit.SECONDS.toNanos(50)) + .count(); + long numSampledSpansWithin5SecondsAfterChange = + spanSampledNanos.stream() + .filter(x -> x > TimeUnit.SECONDS.toNanos(50) && x <= TimeUnit.SECONDS.toNanos(55)) + .count(); + long numSampledSpansInLast5Seconds = + spanSampledNanos.stream() + .filter(x -> x > TimeUnit.SECONDS.toNanos(95) && x <= TimeUnit.SECONDS.toNanos(100)) + .count(); + + assertThat(numSampledSpansWithin5SecondsBeforeChange / 5.) + .isCloseTo(targetSpansPerSecondLimit, Percentage.withPercentage(5)); + assertThat(numSampledSpansWithin5SecondsAfterChange / 5.) + .isLessThan(0.5 * targetSpansPerSecondLimit); + assertThat(numSampledSpansInLast5Seconds / 5.) + .isCloseTo(targetSpansPerSecondLimit, Percentage.withPercentage(5)); + } +} From 1c24cd3667c246182e053684bb0c955ed0cdc0c3 Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Sat, 27 Aug 2022 20:38:06 -0700 Subject: [PATCH 13/13] Spotless --- .../sampling/RateLimitedSamplingPercentageTest.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/sampling/RateLimitedSamplingPercentageTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/sampling/RateLimitedSamplingPercentageTest.java index ba4eac24181..122d3ec7721 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/sampling/RateLimitedSamplingPercentageTest.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/sampling/RateLimitedSamplingPercentageTest.java @@ -27,17 +27,16 @@ package com.microsoft.applicationinsights.agent.internal.sampling; -import org.assertj.core.data.Percentage; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; import java.util.ArrayList; import java.util.List; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.function.LongSupplier; - -import static org.assertj.core.api.Assertions.assertThat; +import org.assertj.core.data.Percentage; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; // uses tests from OpenTelemetry Java Contrib's ConsistentRateLimitingSampler // (https://github.com/open-telemetry/opentelemetry-java-contrib/blob/main/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSamplerTest.java)