From 4e3ff8713ba819fc3e8eb34567cc660e601db530 Mon Sep 17 00:00:00 2001 From: David Venable Date: Thu, 2 Mar 2023 09:53:00 -0600 Subject: [PATCH 1/5] Replace the java.util.* allowed pattern in the ObjectInputFilter with specific classes which are commonly used in Data Prepper. (#2351) Signed-off-by: David Venable --- .../codec/PeerForwarderCodecAppConfig.java | 10 ++++++++- ...rCodecAppConfig_SerializationFilterIT.java | 22 ++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/data-prepper-core/src/main/java/org/opensearch/dataprepper/peerforwarder/codec/PeerForwarderCodecAppConfig.java b/data-prepper-core/src/main/java/org/opensearch/dataprepper/peerforwarder/codec/PeerForwarderCodecAppConfig.java index f114ad8d5a..e4ad12e7de 100644 --- a/data-prepper-core/src/main/java/org/opensearch/dataprepper/peerforwarder/codec/PeerForwarderCodecAppConfig.java +++ b/data-prepper-core/src/main/java/org/opensearch/dataprepper/peerforwarder/codec/PeerForwarderCodecAppConfig.java @@ -44,7 +44,15 @@ public ObjectInputFilter objectInputFilter() { final String pattern = "java.lang.Object;" + - "java.util.*;" + + "java.util.Collections*;" + + "java.util.ArrayList*;" + + "java.util.LinkedList*;" + + "java.util.Map*;" + + "java.util.HashMap*;" + + "java.util.LinkedHashMap*;" + + "java.util.HashSet*;" + + "java.util.LinkedHashSet*;" + + "java.util.Date*;" + "java.time.*;" + "com.fasterxml.jackson.databind.node.*;" + "org.opensearch.dataprepper.peerforwarder.model.*;" + diff --git a/data-prepper-core/src/test/java/org/opensearch/dataprepper/peerforwarder/codec/PeerForwarderCodecAppConfig_SerializationFilterIT.java b/data-prepper-core/src/test/java/org/opensearch/dataprepper/peerforwarder/codec/PeerForwarderCodecAppConfig_SerializationFilterIT.java index 81b2c52c7e..602ac8fbf0 100644 --- a/data-prepper-core/src/test/java/org/opensearch/dataprepper/peerforwarder/codec/PeerForwarderCodecAppConfig_SerializationFilterIT.java +++ b/data-prepper-core/src/test/java/org/opensearch/dataprepper/peerforwarder/codec/PeerForwarderCodecAppConfig_SerializationFilterIT.java @@ -40,7 +40,15 @@ import java.lang.reflect.Modifier; import java.time.Duration; import java.time.Instant; +import java.util.ArrayList; +import java.util.Calendar; import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; @@ -241,7 +249,8 @@ public Stream provideArguments(final ExtensionContext conte return Stream.of( arguments(new LinkedBlockingQueue<>()), arguments(new ArrayBlockingQueue<>(1)), - arguments(Pattern.compile("[1-9]")) + arguments(Pattern.compile("[1-9]")), + arguments(Calendar.getInstance()) ); } } @@ -252,7 +261,18 @@ public Stream provideArguments(final ExtensionContext conte return Stream.of( arguments(UUID.randomUUID().toString()), arguments(Collections.singletonList(UUID.randomUUID().toString())), + arguments(Collections.singleton(UUID.randomUUID().toString())), arguments(Collections.singletonMap(UUID.randomUUID().toString(), UUID.randomUUID().toString())), + arguments(new ArrayList<>()), + arguments(new LinkedList<>()), + arguments(new HashMap<>()), + arguments(new LinkedHashMap<>()), + arguments(new HashSet<>()), + arguments(new LinkedHashSet<>()), + arguments(Collections.unmodifiableList(new ArrayList<>())), + arguments(Collections.unmodifiableMap(new HashMap<>())), + arguments(Collections.unmodifiableSet(new HashSet<>())), + arguments(new Date()), arguments(Instant.now()), arguments(Duration.ofMinutes(5)), arguments(DefaultEventMetadata.builder().withEventType(UUID.randomUUID().toString()).build()) From c15ecb387e82a1dce9706eed03683308b0d106fa Mon Sep 17 00:00:00 2001 From: Asif Sohail Mohammed Date: Thu, 2 Mar 2023 11:38:54 -0600 Subject: [PATCH 2/5] Removed default service endpoint for otel sources (#2346) * Removed default service endpoint for otel sources Signed-off-by: Asif Sohail Mohammed --- data-prepper-plugins/http-source/README.md | 2 +- .../source/loghttp/HTTPSourceConfig.java | 2 ++ .../otel-logs-source/README.md | 2 +- .../source/otellogs/OTelLogsSource.java | 3 ++- .../source/otellogs/OTelLogsSourceConfig.java | 8 ++++++ .../source/otellogs/OTelLogsSourceTest.java | 13 +++------ .../otellogs/OtelLogsSourceConfigTests.java | 27 ++++++++++++++++++- .../otel-metrics-source/README.md | 2 +- .../source/otelmetrics/OTelMetricsSource.java | 3 ++- .../otelmetrics/OTelMetricsSourceConfig.java | 8 ++++++ .../otelmetrics/OTelMetricsSourceTest.java | 14 ++++------ .../OtelMetricsSourceConfigTests.java | 27 ++++++++++++++++++- .../otel-trace-source/README.md | 4 +-- .../source/oteltrace/OTelTraceSource.java | 3 ++- .../oteltrace/OTelTraceSourceConfig.java | 8 ++++++ .../source/oteltrace/OTelTraceSourceTest.java | 13 +++------ .../oteltrace/OtelTraceSourceConfigTests.java | 27 ++++++++++++++++++- 17 files changed, 128 insertions(+), 38 deletions(-) diff --git a/data-prepper-plugins/http-source/README.md b/data-prepper-plugins/http-source/README.md index 50087de671..2a20d45b11 100644 --- a/data-prepper-plugins/http-source/README.md +++ b/data-prepper-plugins/http-source/README.md @@ -22,7 +22,7 @@ source: ## Configurations * port (Optional) => An `int` between 0 and 65535 represents the port source is running on. Default is ```2021```. -* path (Optional) => A `string` which represents the URI path for log ingestion, and it should start with `/`. Path can contain `${pipelineName}` placeholder which will be replaced with pipeline name. Default value is `/log/ingest`. +* path (Optional) => A `string` which represents the URI path for log ingestion. It should start with `/` and length should be at least 1. Path can contain `${pipelineName}` placeholder which will be replaced with pipeline name. Default value is `/log/ingest`. * health_check_service (Optional) => A `boolean` that determines if a `/health` endpoint on the defined port will be home to a health check. Default is `false` * unauthenticated_health_check (Optional) => A `boolean` that determines if the health endpoint will require authentication. This option is ignored if no authentication is defined. Default is `false` * request_timeout (Optional) => An `int` larger than 0 represents request timeout in millis. Default is ```10_000```. diff --git a/data-prepper-plugins/http-source/src/main/java/org/opensearch/dataprepper/plugins/source/loghttp/HTTPSourceConfig.java b/data-prepper-plugins/http-source/src/main/java/org/opensearch/dataprepper/plugins/source/loghttp/HTTPSourceConfig.java index f93275586a..0465727475 100644 --- a/data-prepper-plugins/http-source/src/main/java/org/opensearch/dataprepper/plugins/source/loghttp/HTTPSourceConfig.java +++ b/data-prepper-plugins/http-source/src/main/java/org/opensearch/dataprepper/plugins/source/loghttp/HTTPSourceConfig.java @@ -5,6 +5,7 @@ package org.opensearch.dataprepper.plugins.source.loghttp; +import jakarta.validation.constraints.Size; import org.opensearch.dataprepper.model.configuration.PluginModel; import com.fasterxml.jackson.annotation.JsonProperty; import io.micrometer.core.instrument.util.StringUtils; @@ -36,6 +37,7 @@ public class HTTPSourceConfig { private int port = DEFAULT_PORT; @JsonProperty("path") + @Size(min = 1, message = "path length should be at least 1") private String path = DEFAULT_LOG_INGEST_URI; @JsonProperty("request_timeout") diff --git a/data-prepper-plugins/otel-logs-source/README.md b/data-prepper-plugins/otel-logs-source/README.md index 870ddd386e..ac572a1ea4 100644 --- a/data-prepper-plugins/otel-logs-source/README.md +++ b/data-prepper-plugins/otel-logs-source/README.md @@ -15,7 +15,7 @@ source: ## Configurations * port(Optional) => An `int` represents the port OTel logs source is running on. Default is ```21892```. -* path(Optional) => A `String` which represents the path for sending unframed HTTP requests. This will add a new service which can be used for supporting unframed gRPC with HTTP idiomatic path to a configurable path other than `/opentelemetry.proto.collector.logs.v1.LogsService/Export` which can still be used. Path can contain `${pipelineName}` placeholder which will be replaced with pipeline name. +* path(Optional) => A `String` which represents the path for sending unframed HTTP requests. This can be used for supporting unframed gRPC with HTTP idiomatic path to a configurable path. It should start with `/` and length should be at least 1. `/opentelemetry.proto.collector.logs.v1.LogsService/Export` endpoint will be disabled for both gRPC and HTTP requests if path is configured. Path can contain `${pipelineName}` placeholder which will be replaced with pipeline name. * request_timeout(Optional) => An `int` represents request timeout in millis. Default is ```10_000```. * health_check_service(Optional) => A boolean enables a gRPC health check service under ```grpc.health.v1 / Health / Check```. Default is ```false```. * proto_reflection_service(Optional) => A boolean enables a reflection service for Protobuf services (see [ProtoReflectionService](https://grpc.github.io/grpc-java/javadoc/io/grpc/protobuf/services/ProtoReflectionService.html) and [gRPC reflection](https://github.com/grpc/grpc-java/blob/master/documentation/server-reflection-tutorial.md) docs). Default is ```false```. diff --git a/data-prepper-plugins/otel-logs-source/src/main/java/com/amazon/dataprepper/plugins/source/otellogs/OTelLogsSource.java b/data-prepper-plugins/otel-logs-source/src/main/java/com/amazon/dataprepper/plugins/source/otellogs/OTelLogsSource.java index 088c53bfa5..78818f0f4d 100644 --- a/data-prepper-plugins/otel-logs-source/src/main/java/com/amazon/dataprepper/plugins/source/otellogs/OTelLogsSource.java +++ b/data-prepper-plugins/otel-logs-source/src/main/java/com/amazon/dataprepper/plugins/source/otellogs/OTelLogsSource.java @@ -93,7 +93,6 @@ public void start(Buffer> buffer) { final GrpcServiceBuilder grpcServiceBuilder = GrpcService .builder() - .addService(ServerInterceptors.intercept(oTelLogsGrpcService, serverInterceptors)) .useClientTimeoutHeader(false) .useBlockingTaskExecutor(true); @@ -103,6 +102,8 @@ public void start(Buffer> buffer) { final String transformedOTelLogsSourcePath = oTelLogsSourcePath.replace(PIPELINE_NAME_PLACEHOLDER, pipelineName); grpcServiceBuilder.addService(transformedOTelLogsSourcePath, ServerInterceptors.intercept(oTelLogsGrpcService, serverInterceptors), methodDescriptor); + } else { + grpcServiceBuilder.addService(ServerInterceptors.intercept(oTelLogsGrpcService, serverInterceptors)); } if (oTelLogsSourceConfig.hasHealthCheck()) { diff --git a/data-prepper-plugins/otel-logs-source/src/main/java/com/amazon/dataprepper/plugins/source/otellogs/OTelLogsSourceConfig.java b/data-prepper-plugins/otel-logs-source/src/main/java/com/amazon/dataprepper/plugins/source/otellogs/OTelLogsSourceConfig.java index 42acc74ec3..aaed3e43e5 100644 --- a/data-prepper-plugins/otel-logs-source/src/main/java/com/amazon/dataprepper/plugins/source/otellogs/OTelLogsSourceConfig.java +++ b/data-prepper-plugins/otel-logs-source/src/main/java/com/amazon/dataprepper/plugins/source/otellogs/OTelLogsSourceConfig.java @@ -7,6 +7,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.Size; import org.apache.commons.lang3.StringUtils; import org.opensearch.dataprepper.model.configuration.PluginModel; @@ -46,6 +48,7 @@ public class OTelLogsSourceConfig { private int port = DEFAULT_PORT; @JsonProperty(PATH) + @Size(min = 1, message = "path length should be at least 1") private String path; @JsonProperty(HEALTH_CHECK_SERVICE) @@ -92,6 +95,11 @@ public class OTelLogsSourceConfig { @JsonProperty("authentication") private PluginModel authentication; + @AssertTrue(message = "path should start with /") + boolean isPathValid() { + return path == null || path.startsWith("/"); + } + public void validateAndInitializeCertAndKeyFileInS3() { boolean certAndKeyFileInS3 = false; if (useAcmCertForSSL) { diff --git a/data-prepper-plugins/otel-logs-source/src/test/java/com/amazon/dataprepper/plugins/source/otellogs/OTelLogsSourceTest.java b/data-prepper-plugins/otel-logs-source/src/test/java/com/amazon/dataprepper/plugins/source/otellogs/OTelLogsSourceTest.java index 24e50cc04b..9e7c1bce2b 100644 --- a/data-prepper-plugins/otel-logs-source/src/test/java/com/amazon/dataprepper/plugins/source/otellogs/OTelLogsSourceTest.java +++ b/data-prepper-plugins/otel-logs-source/src/test/java/com/amazon/dataprepper/plugins/source/otellogs/OTelLogsSourceTest.java @@ -696,7 +696,7 @@ void gRPC_with_auth_request_writes_to_buffer_with_successful_response() throws E } @Test - void gRPC_request_writes_to_buffer_with_successful_response_with_custom_path() throws Exception { + void gRPC_request_with_custom_path_throws_when_written_to_default_path() { when(oTelLogsSourceConfig.getPath()).thenReturn(TEST_PATH); when(oTelLogsSourceConfig.enableUnframedRequests()).thenReturn(true); @@ -705,15 +705,10 @@ void gRPC_request_writes_to_buffer_with_successful_response_with_custom_path() t final LogsServiceGrpc.LogsServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) .build(LogsServiceGrpc.LogsServiceBlockingStub.class); - final ExportLogsServiceResponse exportResponse = client.export(createExportLogsRequest()); - assertThat(exportResponse, notNullValue()); - - final ArgumentCaptor>> bufferWriteArgumentCaptor = ArgumentCaptor.forClass(Collection.class); - verify(buffer).writeAll(bufferWriteArgumentCaptor.capture(), anyInt()); - final Collection> actualBufferWrites = bufferWriteArgumentCaptor.getValue(); - assertThat(actualBufferWrites, notNullValue()); - assertThat(actualBufferWrites, hasSize(1)); + final StatusRuntimeException actualException = assertThrows(StatusRuntimeException.class, () -> client.export(createExportLogsRequest())); + assertThat(actualException.getStatus(), notNullValue()); + assertThat(actualException.getStatus().getCode(), equalTo(Status.UNIMPLEMENTED.getCode())); } @ParameterizedTest diff --git a/data-prepper-plugins/otel-logs-source/src/test/java/com/amazon/dataprepper/plugins/source/otellogs/OtelLogsSourceConfigTests.java b/data-prepper-plugins/otel-logs-source/src/test/java/com/amazon/dataprepper/plugins/source/otellogs/OtelLogsSourceConfigTests.java index 9e9fa1763f..880ae8ac02 100644 --- a/data-prepper-plugins/otel-logs-source/src/test/java/com/amazon/dataprepper/plugins/source/otellogs/OtelLogsSourceConfigTests.java +++ b/data-prepper-plugins/otel-logs-source/src/test/java/com/amazon/dataprepper/plugins/source/otellogs/OtelLogsSourceConfigTests.java @@ -205,7 +205,7 @@ void testInvalidConfigWithEmptyKeyFile() { @Test void testValidConfigWithCustomPath() { - final String testPath = "testPath"; + final String testPath = "/testPath"; // Prepare final PluginSetting customPathPluginSetting = completePluginSettingForOtelLogsSource( DEFAULT_REQUEST_TIMEOUT_MS, @@ -224,6 +224,31 @@ void testValidConfigWithCustomPath() { // When/Then assertThat(oTelLogsSourceConfig.getPath(), equalTo(testPath)); + assertThat(oTelLogsSourceConfig.isPathValid(), equalTo(true)); + } + + @Test + void testInValidConfigWithCustomPath() { + final String testPath = "invalidPath"; + // Prepare + final PluginSetting customPathPluginSetting = completePluginSettingForOtelLogsSource( + DEFAULT_REQUEST_TIMEOUT_MS, + DEFAULT_PORT, + testPath, + false, + false, + false, + true, + TEST_KEY_CERT, + "", + DEFAULT_THREAD_COUNT, + DEFAULT_MAX_CONNECTION_COUNT); + + final OTelLogsSourceConfig oTelLogsSourceConfig = OBJECT_MAPPER.convertValue(customPathPluginSetting.getSettings(), OTelLogsSourceConfig.class); + + // When/Then + assertThat(oTelLogsSourceConfig.getPath(), equalTo(testPath)); + assertThat(oTelLogsSourceConfig.isPathValid(), equalTo(false)); } private PluginSetting completePluginSettingForOtelLogsSource(final int requestTimeoutInMillis, diff --git a/data-prepper-plugins/otel-metrics-source/README.md b/data-prepper-plugins/otel-metrics-source/README.md index b4923388bb..0db871c80d 100644 --- a/data-prepper-plugins/otel-metrics-source/README.md +++ b/data-prepper-plugins/otel-metrics-source/README.md @@ -15,7 +15,7 @@ source: ## Configurations * port(Optional) => An `int` represents the port Otel metrics source is running on. Default is ```21891```. -* path(Optional) => A `String` which represents the path for sending unframed HTTP requests. This will add a new service which can be used for supporting unframed gRPC with HTTP idiomatic path to a configurable path other than `/opentelemetry.proto.collector.metrics.v1.MetricsService/Export` which can still be used. Path can contain `${pipelineName}` placeholder which will be replaced with pipeline name. +* path(Optional) => A `String` which represents the path for sending unframed HTTP requests. This can be used for supporting unframed gRPC with HTTP idiomatic path to a configurable path. It should start with `/` and length should be at least 1. `/opentelemetry.proto.collector.metrics.v1.MetricsService/Export` endpoint will be disabled for both gRPC and HTTP requests if path is configured. Path can contain `${pipelineName}` placeholder which will be replaced with pipeline name. * request_timeout(Optional) => An `int` represents request timeout in millis. Default is ```10_000```. * health_check_service(Optional) => A boolean enables health check service. When ```true``` enables a gRPC health check service under ```grpc.health.v1.Health/Check```. Default is ```false```. In order to use the health check service, you must also enable ```proto_reflection_service```. * proto_reflection_service(Optional) => A boolean enables a reflection service for Protobuf services (see [ProtoReflectionService](https://grpc.github.io/grpc-java/javadoc/io/grpc/protobuf/services/ProtoReflectionService.html) and [gRPC reflection](https://github.com/grpc/grpc-java/blob/master/documentation/server-reflection-tutorial.md) docs). Default is ```false```. diff --git a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource.java b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource.java index e09acc58d6..85bc5ba8a8 100644 --- a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource.java +++ b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource.java @@ -94,7 +94,6 @@ public void start(Buffer> buffer) { final GrpcServiceBuilder grpcServiceBuilder = GrpcService .builder() - .addService(ServerInterceptors.intercept(oTelMetricsGrpcService, serverInterceptors)) .useClientTimeoutHeader(false) .useBlockingTaskExecutor(true); @@ -104,6 +103,8 @@ public void start(Buffer> buffer) { final String transformedOTelMetricsSourcePath = oTelMetricsSourcePath.replace(PIPELINE_NAME_PLACEHOLDER, pipelineName); grpcServiceBuilder.addService(transformedOTelMetricsSourcePath, ServerInterceptors.intercept(oTelMetricsGrpcService, serverInterceptors), methodDescriptor); + } else { + grpcServiceBuilder.addService(ServerInterceptors.intercept(oTelMetricsGrpcService, serverInterceptors)); } if (oTelMetricsSourceConfig.hasHealthCheck()) { diff --git a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceConfig.java b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceConfig.java index 93d1523c43..068251ac9a 100644 --- a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceConfig.java +++ b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceConfig.java @@ -6,6 +6,8 @@ package org.opensearch.dataprepper.plugins.source.otelmetrics; import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.Size; import org.apache.commons.lang3.StringUtils; import org.opensearch.dataprepper.model.configuration.PluginModel; @@ -46,6 +48,7 @@ public class OTelMetricsSourceConfig { private int port = DEFAULT_PORT; @JsonProperty(PATH) + @Size(min = 1, message = "path length should be at least 1") private String path; @JsonProperty(HEALTH_CHECK_SERVICE) @@ -95,6 +98,11 @@ public class OTelMetricsSourceConfig { @JsonProperty(UNAUTHENTICATED_HEALTH_CHECK) private boolean unauthenticatedHealthCheck = false; + @AssertTrue(message = "path should start with /") + boolean isPathValid() { + return path == null || path.startsWith("/"); + } + public void validateAndInitializeCertAndKeyFileInS3() { boolean certAndKeyFileInS3 = false; if (useAcmCertForSSL) { diff --git a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceTest.java b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceTest.java index 15a0fc30be..4bb564ab19 100644 --- a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceTest.java +++ b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceTest.java @@ -913,23 +913,19 @@ void gRPC_with_auth_request_writes_to_buffer_with_successful_response() throws E } @Test - void gRPC_request_writes_to_buffer_with_successful_response_with_custom_path() throws Exception { + void gRPC_request_with_custom_path_throws_when_written_to_default_path() { when(oTelMetricsSourceConfig.getPath()).thenReturn(TEST_PATH); when(oTelMetricsSourceConfig.enableUnframedRequests()).thenReturn(true); + configureObjectUnderTest(); SOURCE.start(buffer); final MetricsServiceGrpc.MetricsServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) .build(MetricsServiceGrpc.MetricsServiceBlockingStub.class); - final ExportMetricsServiceResponse exportResponse = client.export(createExportMetricsRequest()); - assertThat(exportResponse, notNullValue()); - - final ArgumentCaptor> bufferWriteArgumentCaptor = ArgumentCaptor.forClass(Record.class); - verify(buffer).write(bufferWriteArgumentCaptor.capture(), anyInt()); - final Record actualBufferWrites = bufferWriteArgumentCaptor.getValue(); - assertThat(actualBufferWrites, notNullValue()); - assertThat(actualBufferWrites.getData().getResourceMetricsCount(), equalTo(1)); + final StatusRuntimeException actualException = assertThrows(StatusRuntimeException.class, () -> client.export(createExportMetricsRequest())); + assertThat(actualException.getStatus(), notNullValue()); + assertThat(actualException.getStatus().getCode(), equalTo(Status.UNIMPLEMENTED.getCode())); } @ParameterizedTest diff --git a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OtelMetricsSourceConfigTests.java b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OtelMetricsSourceConfigTests.java index 381ae38d29..52df0b571d 100644 --- a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OtelMetricsSourceConfigTests.java +++ b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OtelMetricsSourceConfigTests.java @@ -232,7 +232,7 @@ void testInvalidConfigWithEmptyKeyFile() { @Test void testValidConfigWithCustomPath() { - final String testPath = "testPath"; + final String testPath = "/testPath"; // Prepare final PluginSetting customPathPluginSetting = completePluginSettingForOtelMetricsSource( DEFAULT_REQUEST_TIMEOUT_MS, @@ -251,6 +251,31 @@ void testValidConfigWithCustomPath() { // When/Then assertThat(oTelMetricsSourceConfig.getPath(), equalTo(testPath)); + assertThat(oTelMetricsSourceConfig.isPathValid(), equalTo(true)); + } + + @Test + void testInValidConfigWithCustomPath() { + final String testPath = "invalidPath"; + // Prepare + final PluginSetting customPathPluginSetting = completePluginSettingForOtelMetricsSource( + DEFAULT_REQUEST_TIMEOUT_MS, + DEFAULT_PORT, + testPath, + false, + false, + false, + true, + TEST_KEY_CERT, + "", + DEFAULT_THREAD_COUNT, + DEFAULT_MAX_CONNECTION_COUNT); + + final OTelMetricsSourceConfig oTelMetricsSourceConfig = OBJECT_MAPPER.convertValue(customPathPluginSetting.getSettings(), OTelMetricsSourceConfig.class); + + // When/Then + assertThat(oTelMetricsSourceConfig.getPath(), equalTo(testPath)); + assertThat(oTelMetricsSourceConfig.isPathValid(), equalTo(false)); } private PluginSetting completePluginSettingForOtelMetricsSource(final int requestTimeoutInMillis, diff --git a/data-prepper-plugins/otel-trace-source/README.md b/data-prepper-plugins/otel-trace-source/README.md index f6281eafae..fa07a24714 100644 --- a/data-prepper-plugins/otel-trace-source/README.md +++ b/data-prepper-plugins/otel-trace-source/README.md @@ -28,7 +28,7 @@ For more information on migrating from Data Prepper 1.x to Data Prepper 2.x, see ## Configurations * port(Optional) => An `int` represents the port Otel trace source is running on. Default is ```21890```. -* path(Optional) => A `String` which represents the path for sending unframed HTTP requests. This will add a new service which can be used for supporting unframed gRPC with HTTP idiomatic path to a configurable path other than `/opentelemetry.proto.collector.trace.v1.TraceService/Export` which can still be used. Path can contain `${pipelineName}` placeholder which will be replaced with pipeline name. +* path(Optional) => A `String` which represents the path for sending unframed HTTP requests. This can be used for supporting unframed gRPC with HTTP idiomatic path to a configurable path. It should start with `/` and length should be at least 1. `/opentelemetry.proto.collector.trace.v1.TraceService/Export` endpoint will be disabled for both gRPC and HTTP requests if path is configured. Path can contain `${pipelineName}` placeholder which will be replaced with pipeline name. * request_timeout(Optional) => An `int` represents request timeout in millis. Default is ```10_000```. * health_check_service(Optional) => A boolean enables health check service. When ```true``` enables a gRPC health check service under ```grpc.health.v1.Health/Check```. Default is ```false```. In order to use the health check service, you must also enable ```proto_reflection_service```. * unauthenticated_health_check (Optional) => A `boolean` that determines if the health endpoint will require authentication. This option is ignored if no authentication is defined. Default is `false` @@ -106,7 +106,7 @@ Send a sample span with the following https curl command: curl -k -H 'Content-Type: application/json; charset=utf-8' -d '{"resourceSpans":[{"instrumentationLibrarySpans":[{"spans":[{"spanId":"AAAAAAAAAAM=","name":"test-span"}]}]}]}' https://localhost:21890/opentelemetry.proto.collector.trace.v1.TraceService/Export ``` -If `path` option is configured you can send a sample span with the following https curl command: +If `path` option is configured, you can send a sample span to the custom path with the following https curl command: ``` curl -k -H 'Content-Type: application/json; charset=utf-8' -d '{"resourceSpans":[{"instrumentationLibrarySpans":[{"spans":[{"spanId":"AAAAAAAAAAM=","name":"test-span"}]}]}]}' https://localhost:21890/ ``` diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java index 151770e498..f0f07c8ed1 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java @@ -96,7 +96,6 @@ public void start(Buffer> buffer) { final GrpcServiceBuilder grpcServiceBuilder = GrpcService .builder() - .addService(ServerInterceptors.intercept(oTelTraceGrpcService, serverInterceptors)) .useClientTimeoutHeader(false) .useBlockingTaskExecutor(true); @@ -106,6 +105,8 @@ public void start(Buffer> buffer) { final String transformedOTelTraceSourcePath = oTelTraceSourcePath.replace(PIPELINE_NAME_PLACEHOLDER, pipelineName); grpcServiceBuilder.addService(transformedOTelTraceSourcePath, ServerInterceptors.intercept(oTelTraceGrpcService, serverInterceptors), methodDescriptor); + } else { + grpcServiceBuilder.addService(ServerInterceptors.intercept(oTelTraceGrpcService, serverInterceptors)); } if (oTelTraceSourceConfig.hasHealthCheck()) { diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java index bc8eeb2ad1..61ca51791d 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java @@ -5,6 +5,8 @@ package org.opensearch.dataprepper.plugins.source.oteltrace; +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.Size; import org.opensearch.dataprepper.model.configuration.PluginModel; import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.commons.lang3.StringUtils; @@ -46,6 +48,7 @@ public class OTelTraceSourceConfig { private int port = DEFAULT_PORT; @JsonProperty(PATH) + @Size(min = 1, message = "path length should be at least 1") private String path; @JsonProperty(HEALTH_CHECK_SERVICE) @@ -95,6 +98,11 @@ public class OTelTraceSourceConfig { @JsonProperty(UNAUTHENTICATED_HEALTH_CHECK) private boolean unauthenticatedHealthCheck = false; + @AssertTrue(message = "path should start with /") + boolean isPathValid() { + return path == null || path.startsWith("/"); + } + public void validateAndInitializeCertAndKeyFileInS3() { boolean certAndKeyFileInS3 = false; if (useAcmCertForSSL) { diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java index 32a17d9671..4ecfbf9e59 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java @@ -976,7 +976,7 @@ void gRPC_with_auth_request_writes_to_buffer_with_successful_response() throws E } @Test - void gRPC_request_writes_to_buffer_with_successful_response_with_custom_path() throws Exception { + void gRPC_request_with_custom_path_throws_when_written_to_default_path() { when(oTelTraceSourceConfig.getPath()).thenReturn(TEST_PATH); when(oTelTraceSourceConfig.enableUnframedRequests()).thenReturn(true); @@ -985,15 +985,10 @@ void gRPC_request_writes_to_buffer_with_successful_response_with_custom_path() t final TraceServiceGrpc.TraceServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) .build(TraceServiceGrpc.TraceServiceBlockingStub.class); - final ExportTraceServiceResponse exportResponse = client.export(createExportTraceRequest()); - assertThat(exportResponse, notNullValue()); - - final ArgumentCaptor>> bufferWriteArgumentCaptor = ArgumentCaptor.forClass(Collection.class); - verify(buffer).writeAll(bufferWriteArgumentCaptor.capture(), anyInt()); - final Collection> actualBufferWrites = bufferWriteArgumentCaptor.getValue(); - assertThat(actualBufferWrites, notNullValue()); - assertThat(actualBufferWrites, hasSize(1)); + final StatusRuntimeException actualException = assertThrows(StatusRuntimeException.class, () -> client.export(createExportTraceRequest())); + assertThat(actualException.getStatus(), notNullValue()); + assertThat(actualException.getStatus().getCode(), equalTo(Status.UNIMPLEMENTED.getCode())); } @ParameterizedTest diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java index 0afb7a7abd..a495ca045a 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java @@ -232,7 +232,7 @@ void testInvalidConfigWithEmptyKeyFile() { @Test void testValidConfigWithCustomPath() { - final String testPath = "testPath"; + final String testPath = "/testPath"; // Prepare final PluginSetting customPathPluginSetting = completePluginSettingForOtelTraceSource( DEFAULT_REQUEST_TIMEOUT_MS, @@ -251,6 +251,31 @@ void testValidConfigWithCustomPath() { // When/Then assertThat(otelTraceSourceConfig.getPath(), equalTo(testPath)); + assertThat(otelTraceSourceConfig.isPathValid(), equalTo(true)); + } + + @Test + void testInValidConfigWithCustomPath() { + final String testPath = "invalidPath"; + // Prepare + final PluginSetting customPathPluginSetting = completePluginSettingForOtelTraceSource( + DEFAULT_REQUEST_TIMEOUT_MS, + DEFAULT_PORT, + testPath, + false, + false, + false, + true, + TEST_KEY_CERT, + "", + DEFAULT_THREAD_COUNT, + DEFAULT_MAX_CONNECTION_COUNT); + + final OTelTraceSourceConfig otelTraceSourceConfig = OBJECT_MAPPER.convertValue(customPathPluginSetting.getSettings(), OTelTraceSourceConfig.class); + + // When/Then + assertThat(otelTraceSourceConfig.getPath(), equalTo(testPath)); + assertThat(otelTraceSourceConfig.isPathValid(), equalTo(false)); } private PluginSetting completePluginSettingForOtelTraceSource(final int requestTimeoutInMillis, From 99f2cfed423b20abe657eeb95a3baf3b98ecdb54 Mon Sep 17 00:00:00 2001 From: Asif Sohail Mohammed Date: Thu, 2 Mar 2023 12:14:20 -0600 Subject: [PATCH 3/5] Added backoff for SQS to reduce logging (#2326) --- data-prepper-plugins/s3-source/build.gradle | 1 + .../plugins/source/SqsWorkerIT.java | 7 ++++- .../plugins/source/SqsService.java | 8 ++++- .../dataprepper/plugins/source/SqsWorker.java | 31 ++++++++++++++++--- .../SqsRetriesExhaustedException.java | 13 ++++++++ .../plugins/source/SqsWorkerTest.java | 23 +++++++++++++- 6 files changed, 76 insertions(+), 7 deletions(-) create mode 100644 data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/exception/SqsRetriesExhaustedException.java diff --git a/data-prepper-plugins/s3-source/build.gradle b/data-prepper-plugins/s3-source/build.gradle index 8b291dce43..c7c5d7e129 100644 --- a/data-prepper-plugins/s3-source/build.gradle +++ b/data-prepper-plugins/s3-source/build.gradle @@ -13,6 +13,7 @@ repositories { dependencies { implementation project(':data-prepper-api') + implementation libs.armeria.core implementation 'io.micrometer:micrometer-core' implementation 'software.amazon.awssdk:s3' implementation 'software.amazon.awssdk:sts' diff --git a/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/SqsWorkerIT.java b/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/SqsWorkerIT.java index dd3936f93c..6b57a5b28c 100644 --- a/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/SqsWorkerIT.java +++ b/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/SqsWorkerIT.java @@ -5,6 +5,7 @@ package org.opensearch.dataprepper.plugins.source; +import com.linecorp.armeria.client.retry.Backoff; import io.micrometer.core.instrument.DistributionSummary; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.plugins.source.configuration.OnErrorOption; @@ -44,6 +45,7 @@ class SqsWorkerIT { private PluginMetrics pluginMetrics; private S3ObjectGenerator s3ObjectGenerator; private String bucket; + private Backoff backoff; @BeforeEach void setUp() { @@ -57,6 +59,9 @@ void setUp() { .region(Region.of(System.getProperty("tests.s3source.region"))) .build(); + backoff = Backoff.exponential(SqsService.INITIAL_DELAY, SqsService.MAXIMUM_DELAY).withJitter(SqsService.JITTER_RATE) + .withMaxAttempts(Integer.MAX_VALUE); + s3SourceConfig = mock(S3SourceConfig.class); s3Service = mock(S3Service.class); @@ -79,7 +84,7 @@ void setUp() { } private SqsWorker createObjectUnderTest() { - return new SqsWorker(sqsClient, s3Service, s3SourceConfig, pluginMetrics); + return new SqsWorker(sqsClient, s3Service, s3SourceConfig, pluginMetrics, backoff); } @AfterEach diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/SqsService.java b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/SqsService.java index 2ac97e0149..8190612366 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/SqsService.java +++ b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/SqsService.java @@ -5,6 +5,7 @@ package org.opensearch.dataprepper.plugins.source; +import com.linecorp.armeria.client.retry.Backoff; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -14,6 +15,9 @@ public class SqsService { private static final Logger LOG = LoggerFactory.getLogger(SqsService.class); + static final int INITIAL_DELAY = 1000; + static final int MAXIMUM_DELAY = 5 * 60 * 1000; + static final double JITTER_RATE = 0.20; private final S3SourceConfig s3SourceConfig; private final S3Service s3Accessor; @@ -32,7 +36,9 @@ public SqsService(final S3SourceConfig s3SourceConfig, } public void start() { - sqsWorkerThread = new Thread(new SqsWorker(sqsClient, s3Accessor, s3SourceConfig, pluginMetrics)); + final Backoff backoff = Backoff.exponential(INITIAL_DELAY, MAXIMUM_DELAY).withJitter(JITTER_RATE) + .withMaxAttempts(Integer.MAX_VALUE); + sqsWorkerThread = new Thread(new SqsWorker(sqsClient, s3Accessor, s3SourceConfig, pluginMetrics, backoff)); sqsWorkerThread.start(); } diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/SqsWorker.java b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/SqsWorker.java index aee6cf83ca..2e2a2a0766 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/SqsWorker.java +++ b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/SqsWorker.java @@ -6,11 +6,13 @@ package org.opensearch.dataprepper.plugins.source; import com.fasterxml.jackson.core.JsonProcessingException; +import com.linecorp.armeria.client.retry.Backoff; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.Timer; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.plugins.source.configuration.OnErrorOption; import org.opensearch.dataprepper.plugins.source.configuration.SqsOptions; +import org.opensearch.dataprepper.plugins.source.exception.SqsRetriesExhaustedException; import org.opensearch.dataprepper.plugins.source.filter.ObjectCreatedFilter; import org.opensearch.dataprepper.plugins.source.filter.S3EventFilter; import org.slf4j.Logger; @@ -49,16 +51,21 @@ public class SqsWorker implements Runnable { private final Counter sqsMessagesDeletedCounter; private final Counter sqsMessagesFailedCounter; private final Timer sqsMessageDelayTimer; + private final Backoff standardBackoff; + private int failedAttemptCount; public SqsWorker(final SqsClient sqsClient, final S3Service s3Service, final S3SourceConfig s3SourceConfig, - final PluginMetrics pluginMetrics) { + final PluginMetrics pluginMetrics, + final Backoff backoff) { this.sqsClient = sqsClient; this.s3Service = s3Service; this.s3SourceConfig = s3SourceConfig; + this.standardBackoff = backoff; sqsOptions = s3SourceConfig.getSqsOptions(); objectCreatedFilter = new ObjectCreatedFilter(); + failedAttemptCount = 0; sqsMessagesReceivedCounter = pluginMetrics.counter(SQS_MESSAGES_RECEIVED_METRIC_NAME); sqsMessagesDeletedCounter = pluginMetrics.counter(SQS_MESSAGES_DELETED_METRIC_NAME); @@ -108,13 +115,29 @@ int processSqsMessages() { private List getMessagesFromSqs() { try { final ReceiveMessageRequest receiveMessageRequest = createReceiveMessageRequest(); - return sqsClient.receiveMessage(receiveMessageRequest).messages(); - } catch (SqsException e) { - LOG.error("Error reading from SQS: {}", e.getMessage()); + final List messages = sqsClient.receiveMessage(receiveMessageRequest).messages(); + failedAttemptCount = 0; + return messages; + } catch (final SqsException e) { + LOG.error("Error reading from SQS: {}. Retrying with exponential backoff.", e.getMessage()); + applyBackoff(); return Collections.emptyList(); } } + private void applyBackoff() { + final long delayMillis = standardBackoff.nextDelayMillis(++failedAttemptCount); + if (delayMillis < 0) { + Thread.currentThread().interrupt(); + throw new SqsRetriesExhaustedException("SQS retries exhausted. Make sure that SQS configuration is valid and queue exists."); + } + try { + Thread.sleep(delayMillis); + } catch (final InterruptedException e){ + LOG.error("Thread is interrupted while polling SQS with retry.", e); + } + } + private ReceiveMessageRequest createReceiveMessageRequest() { return ReceiveMessageRequest.builder() .queueUrl(sqsOptions.getSqsUrl()) diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/exception/SqsRetriesExhaustedException.java b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/exception/SqsRetriesExhaustedException.java new file mode 100644 index 0000000000..0c6a90b9fd --- /dev/null +++ b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/exception/SqsRetriesExhaustedException.java @@ -0,0 +1,13 @@ +package org.opensearch.dataprepper.plugins.source.exception; + +/** + * This exception is thrown when SQS retries are exhausted + * + * @since 2.1 + */ +public class SqsRetriesExhaustedException extends RuntimeException { + + public SqsRetriesExhaustedException(final String errorMessage) { + super(errorMessage); + } +} diff --git a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/SqsWorkerTest.java b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/SqsWorkerTest.java index 8149b96de0..96f8d7512f 100644 --- a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/SqsWorkerTest.java +++ b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/SqsWorkerTest.java @@ -5,10 +5,12 @@ package org.opensearch.dataprepper.plugins.source; +import com.linecorp.armeria.client.retry.Backoff; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.plugins.source.configuration.AwsAuthenticationOptions; import org.opensearch.dataprepper.plugins.source.configuration.OnErrorOption; import org.opensearch.dataprepper.plugins.source.configuration.SqsOptions; +import org.opensearch.dataprepper.plugins.source.exception.SqsRetriesExhaustedException; import org.opensearch.dataprepper.plugins.source.filter.ObjectCreatedFilter; import org.opensearch.dataprepper.plugins.source.filter.S3EventFilter; import io.micrometer.core.instrument.Counter; @@ -37,6 +39,8 @@ import java.util.Collections; import java.util.UUID; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.anyInt; import static org.opensearch.dataprepper.plugins.source.SqsWorker.SQS_MESSAGES_DELETED_METRIC_NAME; import static org.opensearch.dataprepper.plugins.source.SqsWorker.SQS_MESSAGES_FAILED_METRIC_NAME; import static org.opensearch.dataprepper.plugins.source.SqsWorker.SQS_MESSAGES_RECEIVED_METRIC_NAME; @@ -62,6 +66,7 @@ class SqsWorkerTest { private S3SourceConfig s3SourceConfig; private S3EventFilter objectCreatedFilter; private PluginMetrics pluginMetrics; + private Backoff backoff; private Counter sqsMessagesReceivedCounter; private Counter sqsMessagesDeletedCounter; private Counter sqsMessagesFailedCounter; @@ -73,6 +78,7 @@ void setUp() { s3Service = mock(S3Service.class); s3SourceConfig = mock(S3SourceConfig.class); objectCreatedFilter = new ObjectCreatedFilter(); + backoff = mock(Backoff.class); AwsAuthenticationOptions awsAuthenticationOptions = mock(AwsAuthenticationOptions.class); when(awsAuthenticationOptions.getAwsRegion()).thenReturn(Region.US_EAST_1); @@ -99,7 +105,7 @@ void setUp() { when(pluginMetrics.counter(SQS_MESSAGES_FAILED_METRIC_NAME)).thenReturn(sqsMessagesFailedCounter); when(pluginMetrics.timer(SQS_MESSAGE_DELAY_METRIC_NAME)).thenReturn(sqsMessageDelayTimer); - sqsWorker = new SqsWorker(sqsClient, s3Service, s3SourceConfig, pluginMetrics); + sqsWorker = new SqsWorker(sqsClient, s3Service, s3SourceConfig, pluginMetrics, backoff); } @AfterEach @@ -182,6 +188,21 @@ void processSqsMessages_should_return_zero_messages_when_a_SqsException_is_throw verify(sqsClient, never()).deleteMessageBatch(any(DeleteMessageBatchRequest.class)); } + @Test + void processSqsMessages_should_return_zero_messages_with_backoff_when_a_SqsException_is_thrown() { + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenThrow(SqsException.class); + final int messagesProcessed = sqsWorker.processSqsMessages(); + verify(backoff).nextDelayMillis(1); + assertThat(messagesProcessed, equalTo(0)); + } + + @Test + void processSqsMessages_should_throw_when_a_SqsException_is_thrown_with_max_retries() { + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenThrow(SqsException.class); + when(backoff.nextDelayMillis(anyInt())).thenReturn((long) -1); + assertThrows(SqsRetriesExhaustedException.class, () -> sqsWorker.processSqsMessages()); + } + @ParameterizedTest @ValueSource(strings = {"", "{\"foo\": \"bar\""}) void processSqsMessages_should_not_interact_with_S3Service_if_input_is_not_valid_JSON(String inputString) { From 24db8a6f510bb0fe369fe726e5c00468e8cf1976 Mon Sep 17 00:00:00 2001 From: Hai Yan <8153134+oeyh@users.noreply.github.com> Date: Thu, 2 Mar 2023 13:52:16 -0600 Subject: [PATCH 4/5] Add release notes for 2.1.0 (#2354) * Add release notes for 2.1.0 Signed-off-by: Hai Yan --- .../data-prepper.release-notes-2.1.0.md | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 release/release-notes/data-prepper.release-notes-2.1.0.md diff --git a/release/release-notes/data-prepper.release-notes-2.1.0.md b/release/release-notes/data-prepper.release-notes-2.1.0.md new file mode 100644 index 0000000000..d52716517a --- /dev/null +++ b/release/release-notes/data-prepper.release-notes-2.1.0.md @@ -0,0 +1,72 @@ +## 2023-03-02 Version 2.1.0 + +--- + +### Features +* Support OpenTelementry Logs ([#1372](https://github.com/opensearch-project/data-prepper/pull/1372)) +* New type conversion processor ([#2010](https://github.com/opensearch-project/data-prepper/issues/2010)) +* New anomaly detection processor ([#2040](https://github.com/opensearch-project/data-prepper/issues/2040)) +* Support histogram action in aggregate processor ([#2078](https://github.com/opensearch-project/data-prepper/pull/2078)) +* Support percent sampler action in aggregate processor ([#2094](https://github.com/opensearch-project/data-prepper/issues/2094)) +* Support rate limiter action in aggregate processor ([#2088](https://github.com/opensearch-project/data-prepper/issues/2088)) +* Provide a circuit breaking capability ([#2150](https://github.com/opensearch-project/data-prepper/issues/2150)) + +### Enhancements +* Support custom headers when assuming AWS IAM roles ([#1888](https://github.com/opensearch-project/data-prepper/issues/1888)) +* Provide additional metrics for S3 source ([#2025](https://github.com/opensearch-project/data-prepper/issues/2025)) +* Support "when" condition in aggregate processor actions ([#2021](https://github.com/opensearch-project/data-prepper/issues/2021)) +* Support dynamic index name in OpenSearch sink ([#1459](https://github.com/opensearch-project/data-prepper/issues/1459)) +* Support latest OTEL metrics features ([#1310](https://github.com/opensearch-project/data-prepper/issues/1310)) +* Support loading files from S3 in OpenSearch sink ([#2120](https://github.com/opensearch-project/data-prepper/issues/2120)) +* Support metric tags based on metric name pattern ([#2229](https://github.com/opensearch-project/data-prepper/issues/2229)) +* Support retry when sink fails to initialize ([#936](https://github.com/opensearch-project/data-prepper/issues/936)) +* Add more metrics for OpenSearch sink ([#2168](https://github.com/opensearch-project/data-prepper/issues/2168)) +* Validate file content from S3 for OpenSearch sink ([#2178](https://github.com/opensearch-project/data-prepper/issues/2178)) +* Add more configurable values to OTEL trace raw processor ([#2165](https://github.com/opensearch-project/data-prepper/issues/2165)) +* Add more metrics for OTEL trace raw processor ([#2166](https://github.com/opensearch-project/data-prepper/issues/2166)) +* Provide logging markers to filter sensitive data and events ([#1990](https://github.com/opensearch-project/data-prepper/issues/1990)) +* Support sensitive data masking in logs ([#2187](https://github.com/opensearch-project/data-prepper/issues/2187)) +* Emit Metrics for ACM and AWS CloudMap related calls ([#2087](https://github.com/opensearch-project/data-prepper/issues/2087)) +* Add buffer overflow metric in AbstractBuffer ([#2167](https://github.com/opensearch-project/data-prepper/issues/2167)) +* Support configurable read batch delay for peer-forwarder ([#2158](https://github.com/opensearch-project/data-prepper/issues/2158)) +* Improve ReceiveBuffer Performance ([#2142](https://github.com/opensearch-project/data-prepper/issues/2142)) +* Support OTEL scope logs ([#2115](https://github.com/opensearch-project/data-prepper/issues/2115)) +* Implement batching for peer forwarder request documents ([#2197](https://github.com/opensearch-project/data-prepper/pull/2197)) +* Add more metrics to OTEL metrics source ([#2283](https://github.com/opensearch-project/data-prepper/pull/2283)) +* Support a version property in pipeline YAML configurations ([#2263](https://github.com/opensearch-project/data-prepper/issues/2263)) +* Change the path prefix for OTEL endpoints ([#2257](https://github.com/opensearch-project/data-prepper/issues/2257)) +* Change the path for HTTP source ([#2258](https://github.com/opensearch-project/data-prepper/issues/2258)) +* Improve peer-forwarding serialization ([#2242](https://github.com/opensearch-project/data-prepper/issues/2242)) +* Add null support to Data Prepper Expressions ([#1136](https://github.com/opensearch-project/data-prepper/issues/1136)) +* Support for complex document ID and routing ID ([#1966](https://github.com/opensearch-project/data-prepper/pull/1966)) +* Add PluginMetrics in the Auth Plugin for HTTP, OTEL and Metrics Source ([#2023](https://github.com/opensearch-project/data-prepper/pull/2023)) +* Support Long type in expressions ([#2265](https://github.com/opensearch-project/data-prepper/pull/2265)) +* Add backoff for SQS in S3 source to reduce logging ([#2326](https://github.com/opensearch-project/data-prepper/pull/2326)) +* Add several performance improvement changes ([#2190](https://github.com/opensearch-project/data-prepper/pull/2190), +[#2228](https://github.com/opensearch-project/data-prepper/pull/2228), +[#2238](https://github.com/opensearch-project/data-prepper/pull/2238), +[#2239](https://github.com/opensearch-project/data-prepper/pull/2239), +[#2241](https://github.com/opensearch-project/data-prepper/pull/2241), +[#2246](https://github.com/opensearch-project/data-prepper/pull/2246), +[#2250](https://github.com/opensearch-project/data-prepper/pull/2250), +[#2290](https://github.com/opensearch-project/data-prepper/pull/2290)) + +### Bug Fixes +* Fix logging by updating related packages for SLF4J 2.0 ([#2113](https://github.com/opensearch-project/data-prepper/pull/2113)) +* Fix null pointer exception in remote peer forwarding ([#2124](https://github.com/opensearch-project/data-prepper/pull/2124)) +* Fix Data Prepper to not terminate on invalid open telemetry metric/trace data ([#2176](https://github.com/opensearch-project/data-prepper/pull/2176)) +* Fix Armeria exception when under pressure ([#2274](https://github.com/opensearch-project/data-prepper/pull/2274)) +* Fix release build issue ([#2345](https://github.com/opensearch-project/data-prepper/pull/2345)) + +### Security +* Fix CVE-2022-3509, CVE-2022-3510 ([#2079](https://github.com/opensearch-project/data-prepper/pull/2079)) +* Fix CVE-2022-41881, CVE-2021-21290 and CVE-2022-41915 ([#2093](https://github.com/opensearch-project/data-prepper/pull/2093)) +* Fix CVE-2020-8908, CVE-2018-10237 ([#2254](https://github.com/opensearch-project/data-prepper/pull/2254)) +* Update OpenTelemetry dependencies to address CVE-2023-25577 and CVE-2023-23934 ([#2288](https://github.com/opensearch-project/data-prepper/pull/2288)) +* Use secure Java serialization with Java serialization ([#2310](https://github.com/opensearch-project/data-prepper/issues/2310)) + +### Maintenance +* Test against multiple OTEL version ([#1963](https://github.com/opensearch-project/data-prepper/issues/1963)) +* Document trace_peer_forwarder in README ([#2141](https://github.com/opensearch-project/data-prepper/issues/2141)) +* Use consistent AWS pipeline configurations ([#2184](https://github.com/opensearch-project/data-prepper/issues/2184)) +* Use snake case for all configurations ([#2203](https://github.com/opensearch-project/data-prepper/issues/2203)) From 567e3bf5efbe89731a79c5e55f08f44270d02bd6 Mon Sep 17 00:00:00 2001 From: Asif Sohail Mohammed Date: Thu, 2 Mar 2023 15:20:55 -0600 Subject: [PATCH 5/5] Added 2.1 change log (#2360) Signed-off-by: Asif Sohail Mohammed --- .../data-prepper.change-log-2.1.0.md | 2568 +++++++++++++++++ 1 file changed, 2568 insertions(+) create mode 100644 release/release-notes/data-prepper.change-log-2.1.0.md diff --git a/release/release-notes/data-prepper.change-log-2.1.0.md b/release/release-notes/data-prepper.change-log-2.1.0.md new file mode 100644 index 0000000000..6197eeba20 --- /dev/null +++ b/release/release-notes/data-prepper.change-log-2.1.0.md @@ -0,0 +1,2568 @@ + +* __Add release notes for 2.1.0 (#2354)__ + + [Hai Yan](mailto:8153134+oeyh@users.noreply.github.com) - Thu, 2 Mar 2023 13:52:16 -0600 + + EAD -> refs/heads/change-log-2.1, refs/remotes/upstream/main, refs/remotes/origin/main, refs/remotes/origin/HEAD, refs/heads/main + * Add release notes for 2.1.0 + Signed-off-by: Hai Yan <oeyh@amazon.com> + +* __Added backoff for SQS to reduce logging (#2326)__ + + [Asif Sohail Mohammed](mailto:nsifmoh@amazon.com) - Thu, 2 Mar 2023 12:14:20 -0600 + + + + +* __Removed default service endpoint for otel sources (#2346)__ + + [Asif Sohail Mohammed](mailto:nsifmoh@amazon.com) - Thu, 2 Mar 2023 11:38:54 -0600 + + + * Removed default service endpoint for otel sources + Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com> + + +* __Replace the java.util.* allowed pattern in the ObjectInputFilter with specific classes which are commonly used in Data Prepper. (#2351)__ + + [David Venable](mailto:dlv@amazon.com) - Thu, 2 Mar 2023 09:53:00 -0600 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Explicitly set the GitHub Actions thumbprint to resolve #2343. Updated the AWS CDK as well. (#2345)__ + + [David Venable](mailto:dlv@amazon.com) - Wed, 1 Mar 2023 15:40:05 -0600 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Updated version to 2.2 on main (#2342)__ + + [Asif Sohail Mohammed](mailto:nsifmoh@amazon.com) - Wed, 1 Mar 2023 13:27:15 -0600 + + + Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com> + +* __Generated THIRD-PARTY file for 062ae95 (#2344)__ + + [opensearch-trigger-bot[bot]](mailto:98922864+opensearch-trigger-bot[bot]@users.noreply.github.com) - Wed, 1 Mar 2023 13:26:20 -0600 + + + Signed-off-by: GitHub <noreply@github.com> + Co-authored-by: asifsmohammed + <asifsmohammed@users.noreply.github.com> + +* __Use an ObjectInputFilter to serialize allow deserialization of only certain objects in peer-to-peer connections. Additionally, it refactors some application configurations to improve integration testing. Fixes #2310. (#2311)__ + + [David Venable](mailto:dlv@amazon.com) - Wed, 1 Mar 2023 10:06:52 -0600 + + efs/heads/changelog-2.1 + Signed-off-by: David Venable <dlv@amazon.com> + +* __Bump org.apache.logging.log4j:log4j-bom in /data-prepper-core (#2333)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Wed, 1 Mar 2023 09:47:42 -0600 + + + Bumps org.apache.logging.log4j:log4j-bom from 2.19.0 to 2.20.0. + + --- + updated-dependencies: + - dependency-name: org.apache.logging.log4j:log4j-bom + dependency-type: direct:production + update-type: version-update:semver-minor + ... + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Bump org.assertj:assertj-core from 3.21.0 to 3.24.2 (#2331)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Wed, 1 Mar 2023 09:47:06 -0600 + + + Bumps org.assertj:assertj-core from 3.21.0 to 3.24.2. + + --- + updated-dependencies: + - dependency-name: org.assertj:assertj-core + dependency-type: direct:production + update-type: version-update:semver-minor + ... + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Bump org.junit.jupiter:junit-jupiter-api from 5.9.0 to 5.9.2 (#2332)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Wed, 1 Mar 2023 09:46:26 -0600 + + + Bumps + [org.junit.jupiter:junit-jupiter-api](https://github.com/junit-team/junit5) + from 5.9.0 to 5.9.2. + - [Release notes](https://github.com/junit-team/junit5/releases) + - [Commits](https://github.com/junit-team/junit5/compare/r5.9.0...r5.9.2) + + --- + updated-dependencies: + - dependency-name: org.junit.jupiter:junit-jupiter-api + dependency-type: direct:production + update-type: version-update:semver-patch + ... + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Support for path in OTel sources (#2297)__ + + [Asif Sohail Mohammed](mailto:nsifmoh@amazon.com) - Tue, 28 Feb 2023 19:57:38 -0600 + + + Initial commit for OTel trace path changes + Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com> + Signed-off-by: Asif + Sohail Mohammed <mdasifsohail7@gmail.com> + +* __Fix grok processor to not create a new record (#2325)__ + + [kkondaka](mailto:41027584+kkondaka@users.noreply.github.com) - Tue, 28 Feb 2023 19:56:04 -0600 + + + * Fix grok processor to not create a new record + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Fixed checkStyleMain failure + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + --------- + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + Co-authored-by: Krishna + Kondaka <krishkdk@amazon.com> + +* __Bump org.springframework:spring-context in /data-prepper-expression (#2223)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Mon, 27 Feb 2023 18:27:04 -0600 + + efs/heads/otel-paths + Bumps + [org.springframework:spring-context](https://github.com/spring-projects/spring-framework) + from 5.3.23 to 5.3.25. + - [Release notes](https://github.com/spring-projects/spring-framework/releases) + + - + [Commits](https://github.com/spring-projects/spring-framework/compare/v5.3.23...v5.3.25) + + + --- + updated-dependencies: + - dependency-name: org.springframework:spring-context + dependency-type: direct:production + update-type: version-update:semver-patch + ... + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Bump org.springframework:spring-context in /data-prepper-core (#2214)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Mon, 27 Feb 2023 16:52:20 -0600 + + + Bumps + [org.springframework:spring-context](https://github.com/spring-projects/spring-framework) + from 5.3.23 to 5.3.25. + - [Release notes](https://github.com/spring-projects/spring-framework/releases) + + - + [Commits](https://github.com/spring-projects/spring-framework/compare/v5.3.23...v5.3.25) + + + --- + updated-dependencies: + - dependency-name: org.springframework:spring-context + dependency-type: direct:production + update-type: version-update:semver-patch + ... + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Bump io.spring.dependency-management from 1.0.11.RELEASE to 1.1.0 (#2106)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Mon, 27 Feb 2023 16:48:25 -0600 + + + Bumps io.spring.dependency-management from 1.0.11.RELEASE to 1.1.0. + + --- + updated-dependencies: + - dependency-name: io.spring.dependency-management + dependency-type: direct:production + update-type: version-update:semver-minor + ... + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Bump com.fasterxml.jackson.datatype:jackson-datatype-jdk8 (#2217)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Mon, 27 Feb 2023 16:47:36 -0600 + + + Bumps com.fasterxml.jackson.datatype:jackson-datatype-jdk8 from 2.14.1 to + 2.14.2. + + --- + updated-dependencies: + - dependency-name: com.fasterxml.jackson.datatype:jackson-datatype-jdk8 + dependency-type: direct:production + update-type: version-update:semver-patch + ... + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Added path support for HTTP source (#2277)__ + + [Asif Sohail Mohammed](mailto:nsifmoh@amazon.com) - Mon, 27 Feb 2023 15:26:58 -0600 + + + * Added path support for HTTP source + Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com> + +* __Updated Data Prepper base image for e2e tests (#2269)__ + + [Asif Sohail Mohammed](mailto:nsifmoh@amazon.com) - Mon, 27 Feb 2023 15:08:13 -0600 + + + Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com> + +* __Loop until interrupted in SqsWorker (#2306)__ + + [Chase](mailto:62891993+engechas@users.noreply.github.com) - Fri, 24 Feb 2023 00:10:26 -0600 + + + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + +* __Performance Improvement - Avoid event copying for the first sub-pipeline (#2290)__ + + [kkondaka](mailto:41027584+kkondaka@users.noreply.github.com) - Thu, 23 Feb 2023 22:08:47 -0600 + + + Avoid event copying for the first sub-pipeline + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + +* __Bump slf4j-simple from 1.7.36 to 2.0.6 (#2112)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Wed, 22 Feb 2023 10:39:21 -0600 + + + Bumps [slf4j-simple](https://github.com/qos-ch/slf4j) from 1.7.36 to 2.0.6. + - [Release notes](https://github.com/qos-ch/slf4j/releases) + - [Commits](https://github.com/qos-ch/slf4j/compare/v_1.7.36...v_2.0.6) + + --- + updated-dependencies: + - dependency-name: org.slf4j:slf4j-simple + dependency-type: direct:production + update-type: version-update:semver-major + ... + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Upgrades opentelemetry dependencies (#2288)__ + + [Shivani Shukla](mailto:67481911+sshivanii@users.noreply.github.com) - Tue, 21 Feb 2023 21:16:58 -0600 + + + Signed-off-by: Shivani Shukla <sshkamz@amazon.com> + +* __Fix error message printed when OpenSearch Sink fails to initialize (#2296)__ + + [kkondaka](mailto:41027584+kkondaka@users.noreply.github.com) - Tue, 21 Feb 2023 14:38:36 -0600 + + + Fix error message printed when OpenSearch Sink fails to initialize + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + --------- + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + Co-authored-by: Krishna + Kondaka <krishkdk@amazon.com> + +* __adding backwards compatibility support in versioning check (#2295)__ + + [Christopher Manning](mailto:cmanning09@users.noreply.github.com) - Tue, 21 Feb 2023 14:13:50 -0600 + + + Signed-off-by: Christopher Manning <cmanning09@users.noreply.github.com> + +* __[2263] adding versioning property to pipeline yaml configuration (#2292)__ + + [Christopher Manning](mailto:cmanning09@users.noreply.github.com) - Tue, 21 Feb 2023 09:08:39 -0600 + + + * [2263] adding versioning property to pipeline yaml configuration + Signed-off-by: Christopher Manning <cmanning09@users.noreply.github.com> + + * adding hashCode support and providing documentation for updating the version + Signed-off-by: Christopher Manning <cmanning09@users.noreply.github.com> + + --------- + Signed-off-by: Christopher Manning <cmanning09@users.noreply.github.com> + +* __Updates to opensearch-java 2.2.0 (#2287)__ + + [David Venable](mailto:dlv@amazon.com) - Mon, 20 Feb 2023 15:18:55 -0600 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Replace an unnecessary builder pattern in trace raw processor with a factory method. This is a little simpler code and it reduces an object creation. (#2271)__ + + [David Venable](mailto:dlv@amazon.com) - Fri, 17 Feb 2023 14:34:21 -0600 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Create DataPrepper server after pipeline initialization (#2284)__ + + [Chase](mailto:62891993+engechas@users.noreply.github.com) - Fri, 17 Feb 2023 11:12:08 -0600 + + + * Create DataPrepper HttpServer after pipeline initialization + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + * Remove injection provider implementation + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + --------- + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + +* __Refactor PeerForwarderReceiveBuffer to extend AbstractBuffer to pick up metric logic (#2286)__ + + [Chase](mailto:62891993+engechas@users.noreply.github.com) - Fri, 17 Feb 2023 10:48:29 -0600 + + + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + +* __Add more metrics to OTEL metrics source (#2283)__ + + [Chase](mailto:62891993+engechas@users.noreply.github.com) - Fri, 17 Feb 2023 10:24:30 -0600 + + + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + +* __Make batching queue depth configurable, fill in some missing documentation around batching (#2278)__ + + [Chase](mailto:62891993+engechas@users.noreply.github.com) - Fri, 17 Feb 2023 10:24:06 -0600 + + + * Make batching queue depth configurable, fill in some missing documentation + around batching + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + * Fix validation message + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + * Actually fix validation message + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + --------- + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + +* __Update to opensearch 1.3.8. Resolves #2192, #2193 (#2285)__ + + [David Venable](mailto:dlv@amazon.com) - Thu, 16 Feb 2023 15:56:36 -0600 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __OTel Trace Source: Logging and integration tests (#2262)__ + + [David Venable](mailto:dlv@amazon.com) - Thu, 16 Feb 2023 11:12:26 -0600 + + + Modified some of the logging for the OTel Trace source. Added integration tests + which actually verify that gRPC requests operate as expected. Print out SLF4J + logs from tests to stdout to help with debugging. + Signed-off-by: David Venable <dlv@amazon.com> + +* __Created untriaged issue workflow. (#2276)__ + + [Daniel (dB.) Doubrovkine](mailto:dblock@amazon.com) - Wed, 15 Feb 2023 13:15:53 -0600 + + + Signed-off-by: dblock <dblock@amazon.com> + +* __Long int fix (#2265)__ + + [kkondaka](mailto:41027584+kkondaka@users.noreply.github.com) - Tue, 14 Feb 2023 16:13:20 -0600 + + + * OpenSearchSink should close open files before retrying initialization + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Minor fixes to data prepper plugins documentation + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Fix Long Integer comparisons + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Added more tests for long integer comparisons + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + --------- + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + Co-authored-by: Krishna + Kondaka <krishkdk@amazon.com> + +* __CVE: updated guava version (#2254)__ + + [Asif Sohail Mohammed](mailto:nsifmoh@amazon.com) - Tue, 14 Feb 2023 15:08:18 -0600 + + + * CVE: updated guava version + Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com> + Signed-off-by: Asif + Sohail Mohammed <mdasifsohail7@gmail.com> + + * Updated guava to use catalog + Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com> + Signed-off-by: Asif + Sohail Mohammed <mdasifsohail7@gmail.com> + + --------- + Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com> + Signed-off-by: Asif + Sohail Mohammed <mdasifsohail7@gmail.com> + +* __Update to Armeria 1.22.1 which fixes #2206. (#2274)__ + + [David Venable](mailto:dlv@amazon.com) - Tue, 14 Feb 2023 13:36:21 -0600 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Fix: e2e tests and added documentation (#2267)__ + + [Asif Sohail Mohammed](mailto:nsifmoh@amazon.com) - Mon, 13 Feb 2023 14:40:27 -0600 + + + Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com> + +* __Removes some unnecessary methods from ServiceMapRelationship and adds additional tests to verify behavior. (#2200)__ + + [David Venable](mailto:dlv@amazon.com) - Mon, 13 Feb 2023 13:45:31 -0600 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Created a metric for the overall JVM memory usage - both heap and non-heap. (#2266)__ + + [David Venable](mailto:dlv@amazon.com) - Mon, 13 Feb 2023 13:24:48 -0600 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Fixes index_type convention (#2261)__ + + [Shivani Shukla](mailto:67481911+sshivanii@users.noreply.github.com) - Fri, 10 Feb 2023 14:47:20 -0600 + + + Signed-off-by: Shivani Shukla <sshkamz@amazon.com> + +* __Minor fixes to data prepper plugins documentation (#2260)__ + + [kkondaka](mailto:41027584+kkondaka@users.noreply.github.com) - Fri, 10 Feb 2023 13:39:12 -0600 + + + * OpenSearchSink should close open files before retrying initialization + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Minor fixes to data prepper plugins documentation + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + --------- + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + Co-authored-by: Krishna + Kondaka <krishkdk@amazon.com> + +* __ENH: peer forwarding codec and model (#2256)__ + + [Qi Chen](mailto:qchea@amazon.com) - Fri, 10 Feb 2023 09:19:51 -0600 + + + Signed-off-by: George Chen <qchea@amazon.com> + +* __Use the Gradle version catalog for common software versions to the extent that it can replace the versionMap. (#2253)__ + + [David Venable](mailto:dlv@amazon.com) - Thu, 9 Feb 2023 13:49:11 -0600 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __OpenSearchSink should close open files before retrying initialization (#2255)__ + + [kkondaka](mailto:41027584+kkondaka@users.noreply.github.com) - Thu, 9 Feb 2023 11:15:19 -0600 + + + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + Co-authored-by: Krishna + Kondaka <krishkdk@amazon.com> + +* __Added trace peer forwarder doc (#2245)__ + + [Asif Sohail Mohammed](mailto:nsifmoh@amazon.com) - Wed, 8 Feb 2023 18:10:26 -0600 + + + * Added trace peer forwarder doc + Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com> + +* __Refactor to remove stream in RemotePeerForwarder as micro optimization (#2250)__ + + [Chase](mailto:62891993+engechas@users.noreply.github.com) - Wed, 8 Feb 2023 18:09:17 -0600 + + + * Refactor to remove stream in RemotePeerForwarder as micro optimization + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + * Remove unused import + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + --------- + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + +* __Bump org.assertj:assertj-core (#2221)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Wed, 8 Feb 2023 16:01:58 -0600 + + + Bumps org.assertj:assertj-core from 3.23.1 to 3.24.2. + + --- + updated-dependencies: + - dependency-name: org.assertj:assertj-core + dependency-type: direct:production + update-type: version-update:semver-minor + ... + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Bump certifi in /release/smoke-tests/otel-span-exporter (#2063)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Wed, 8 Feb 2023 15:51:47 -0600 + + + Bumps [certifi](https://github.com/certifi/python-certifi) from 2021.10.8 to + 2022.12.7. + - [Release notes](https://github.com/certifi/python-certifi/releases) + - + [Commits](https://github.com/certifi/python-certifi/compare/2021.10.08...2022.12.07) + + + --- + updated-dependencies: + - dependency-name: certifi + dependency-type: direct:production + ... + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Bump com.google.guava:guava in /data-prepper-plugins/aggregate-processor (#2219)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Wed, 8 Feb 2023 15:50:29 -0600 + + + Bumps [com.google.guava:guava](https://github.com/google/guava) from 10.0.1 to + 23.0. + - [Release notes](https://github.com/google/guava/releases) + - [Commits](https://github.com/google/guava/compare/v10.0.1...v23.0) + + --- + updated-dependencies: + - dependency-name: com.google.guava:guava + dependency-type: direct:production + update-type: version-update:semver-major + ... + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Bump org.assertj:assertj-core (#2226)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Wed, 8 Feb 2023 15:42:11 -0600 + + + Bumps org.assertj:assertj-core from 3.23.1 to 3.24.2. + + --- + updated-dependencies: + - dependency-name: org.assertj:assertj-core + dependency-type: direct:production + update-type: version-update:semver-minor + ... + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Bump org.springframework:spring-test in /data-prepper-expression (#2224)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Wed, 8 Feb 2023 14:53:37 -0600 + + + Bumps + [org.springframework:spring-test](https://github.com/spring-projects/spring-framework) + from 5.3.23 to 5.3.25. + - [Release notes](https://github.com/spring-projects/spring-framework/releases) + + - + [Commits](https://github.com/spring-projects/spring-framework/compare/v5.3.23...v5.3.25) + + + --- + updated-dependencies: + - dependency-name: org.springframework:spring-test + dependency-type: direct:production + update-type: version-update:semver-patch + ... + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Moved Random Cut Forest libraries to the Anomaly Detection processor. (#2252)__ + + [David Venable](mailto:dlv@amazon.com) - Wed, 8 Feb 2023 11:44:14 -0600 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Sink doInitialize() should be able throw exception to stop execution (#2249)__ + + [kkondaka](mailto:41027584+kkondaka@users.noreply.github.com) - Wed, 8 Feb 2023 09:53:30 -0600 + + + * Sink doInitialize() should be able throw exception to stop execution + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Sink doInitialize() should be able throw exception to stop execution - fixed + code coverage failures + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + --------- + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + Co-authored-by: Krishna + Kondaka <krishkdk@amazon.com> + +* __Consistent AWS Pipeline Configurations - #2184 (#2248)__ + + [kkondaka](mailto:41027584+kkondaka@users.noreply.github.com) - Tue, 7 Feb 2023 15:58:59 -0600 + + + * Consistent AWS Pipeline Configurations - #2184 + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Consistent AWS Pipeline Configurations - updated documentation + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Consistent AWS Pipeline Configurations - fixed documentation + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + --------- + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + Co-authored-by: Krishna + Kondaka <krishkdk@amazon.com> + +* __Made some changes to the :release:archives:linux:linuxTar task to reduce dependencies and get a better build time when building multiple times. (#2240)__ + + [David Venable](mailto:dlv@amazon.com) - Tue, 7 Feb 2023 14:30:30 -0600 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Added SDK client metrics and metric filters (#2232)__ + + [Asif Sohail Mohammed](mailto:nsifmoh@amazon.com) - Tue, 7 Feb 2023 14:04:10 -0600 + + + * Added SDK client metrics and metric filters + Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com> + +* __Improved how the PipelineConnector copies events by using a new static method to perform the copy. For JacksonEvent, this performs a deepCopy() which appears to be more efficient than the old process. (#2241)__ + + [David Venable](mailto:dlv@amazon.com) - Tue, 7 Feb 2023 10:06:18 -0600 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Use snake case for all configurations #2203 (#2243)__ + + [kkondaka](mailto:41027584+kkondaka@users.noreply.github.com) - Mon, 6 Feb 2023 17:30:14 -0600 + + + * Use snake case for all configurations #2203 + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + +* __Add back LinkedBlockingQueue::poll to avoid busy wait (#2246)__ + + [Chase](mailto:62891993+engechas@users.noreply.github.com) - Mon, 6 Feb 2023 14:12:32 -0600 + + + * Add back LinkedBlockingQueue::poll to avoid busy wait + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + * Drop poll timeout when delay=0 to 5 millis + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + --------- + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + +* __Use Armeria's BlockingTaskExecutor to configure thread names (#2235)__ + + [David Venable](mailto:dlv@amazon.com) - Mon, 6 Feb 2023 09:32:47 -0600 + + + Use Armeria's BlockingTaskExecutor to configure thread name + Signed-off-by: David Venable <dlv@amazon.com> + +* __Update default values in buffer documentation (#2233)__ + + [JannikBrand](mailto:jannik.brand@sap.com) - Sat, 4 Feb 2023 14:04:56 -0600 + + + + +* __The DefaultEventMetadata almost always creates an empty attributes. This uses the static ImmutableMap.of() which uses a shared instance underneath so that new objects are not created each time. (#2239)__ + + [David Venable](mailto:dlv@amazon.com) - Fri, 3 Feb 2023 13:21:31 -0600 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __The RecordMetadata default metadata is used by most records and it creates a new instance each time. This is an immutable class, so just return the same metadata instance for all. (#2238)__ + + [David Venable](mailto:dlv@amazon.com) - Fri, 3 Feb 2023 12:47:25 -0600 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Abstract sink should not create a new retry thread everytime initialization fails (#2231)__ + + [kkondaka](mailto:41027584+kkondaka@users.noreply.github.com) - Thu, 2 Feb 2023 10:25:41 -0600 + + + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + Co-authored-by: Krishna + Kondaka <krishkdk@amazon.com> + +* __Add Support for retry when Sink fails to initialize (#2198)__ + + [kkondaka](mailto:41027584+kkondaka@users.noreply.github.com) - Wed, 1 Feb 2023 16:39:50 -0600 + + + * Remove opensearch availability dependence - Issue #936 + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Add support for retry when Sink fails to initialize - issue #936 + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Add support for retry when Sink fails to initialize - added number of retries + to dataprepper execute + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Add support for retry when Sink fails to initialize - Addressed review + comments. Fixed test failures + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Add support for retry when Sink fails to initialize - Addressed review + comments + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Add support for retry when Sink fails to initialize - Fixes for failing tests + + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Add support for retry when Sink fails to initialize - Fixes for failing tests + + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Add support for retry when Sink fails to initialize - Fixes for code coverage + failures + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Add support for retry when Sink fails to initialize - Modified to check for + retryable exception + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Add support for retry when Sink fails to initialize - addressed review + comments + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + --------- + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + Co-authored-by: Krishna + Kondaka <krishkdk@amazon.com> + +* __DynamicIndexTemplate can cause NPE that shuts down pipeline - Issue #2210 (#2211)__ + + [kkondaka](mailto:41027584+kkondaka@users.noreply.github.com) - Wed, 1 Feb 2023 15:15:01 -0600 + + + * DynamicIndexTemplate can cause NPE that shuts down pipeline - Issue #2210 + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * DynamicIndexTemplate can cause NPE that shuts down pipeline - addressed + review comments + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * DynamicIndexTemplate can cause NPE that shuts down pipeline - addressed + review comments + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * DynamicIndexTemplate can cause NPE that shuts down pipeline - fixed build + failure + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + --------- + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + Co-authored-by: Krishna + Kondaka <krishkdk@amazon.com> + +* __Add missing metrics for Opensearch Sink #2168 (#2205)__ + + [kkondaka](mailto:41027584+kkondaka@users.noreply.github.com) - Wed, 1 Feb 2023 15:10:21 -0600 + + + * Add missing metrics for Opensearch Sink #2168 + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Add missing metrics for Opensearch Sink -- updated the documentation + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Add missing metrics for Opensearch Sink -- addressed review comments + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Add missing metrics for Opensearch Sink -- addressed review comments + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Add missing metrics for Opensearch Sink -- modified documentation + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Add missing metrics for Opensearch Sink -- addressed review comments + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Add missing metrics for Opensearch Sink -- fixed build failure + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + --------- + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + Co-authored-by: Krishna + Kondaka <krishkdk@amazon.com> + +* __Perform serialization sequentially, block for each batch of forwards (#2228)__ + + [Chase](mailto:62891993+engechas@users.noreply.github.com) - Wed, 1 Feb 2023 11:04:43 -0600 + + + * Perform serialization sequentially, block for each batch of forwards + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + * Add test for mixed future results + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + * Rename forwardRecords to forwardBatchedRecords to avoid naming duplication + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + * Add missing log statement + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + --------- + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + +* __Removed getObjectAttributes API call to check object size (#2179)__ + + [Asif Sohail Mohammed](mailto:nsifmoh@amazon.com) - Wed, 1 Feb 2023 10:16:49 -0600 + + + * Removed getObjectAttributes API call to check object size + Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com> + +* __Bump org.springframework:spring-core in /data-prepper-expression (#2222)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Tue, 31 Jan 2023 21:00:33 -0600 + + + Bumps + [org.springframework:spring-core](https://github.com/spring-projects/spring-framework) + from 5.3.23 to 5.3.25. + - [Release notes](https://github.com/spring-projects/spring-framework/releases) + + - + [Commits](https://github.com/spring-projects/spring-framework/compare/v5.3.23...v5.3.25) + + + --- + updated-dependencies: + - dependency-name: org.springframework:spring-core + dependency-type: direct:production + update-type: version-update:semver-patch + ... + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Bump org.assertj:assertj-core (#2225)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Tue, 31 Jan 2023 20:46:44 -0600 + + + Bumps org.assertj:assertj-core from 3.23.1 to 3.24.2. + + --- + updated-dependencies: + - dependency-name: org.assertj:assertj-core + dependency-type: direct:production + update-type: version-update:semver-minor + ... + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Bump org.assertj:assertj-core in /data-prepper-plugins/http-source (#2227)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Tue, 31 Jan 2023 20:46:18 -0600 + + + Bumps org.assertj:assertj-core from 3.23.1 to 3.24.2. + + --- + updated-dependencies: + - dependency-name: org.assertj:assertj-core + dependency-type: direct:production + update-type: version-update:semver-minor + ... + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Bump org.assertj:assertj-core in /data-prepper-plugins/otel-trace-source (#2212)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Tue, 31 Jan 2023 20:45:50 -0600 + + + Bumps org.assertj:assertj-core from 3.23.1 to 3.24.2. + + --- + updated-dependencies: + - dependency-name: org.assertj:assertj-core + dependency-type: direct:production + update-type: version-update:semver-minor + ... + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Bump org.springframework:spring-test in /data-prepper-core (#2215)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Tue, 31 Jan 2023 20:41:19 -0600 + + + Bumps + [org.springframework:spring-test](https://github.com/spring-projects/spring-framework) + from 5.3.23 to 5.3.25. + - [Release notes](https://github.com/spring-projects/spring-framework/releases) + + - + [Commits](https://github.com/spring-projects/spring-framework/compare/v5.3.23...v5.3.25) + + + --- + updated-dependencies: + - dependency-name: org.springframework:spring-test + dependency-type: direct:production + update-type: version-update:semver-patch + ... + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Bump org.springframework:spring-core in /data-prepper-core (#2216)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Tue, 31 Jan 2023 20:40:24 -0600 + + + Bumps + [org.springframework:spring-core](https://github.com/spring-projects/spring-framework) + from 5.3.23 to 5.3.25. + - [Release notes](https://github.com/spring-projects/spring-framework/releases) + + - + [Commits](https://github.com/spring-projects/spring-framework/compare/v5.3.23...v5.3.25) + + + --- + updated-dependencies: + - dependency-name: org.springframework:spring-core + dependency-type: direct:production + update-type: version-update:semver-patch + ... + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Implement batching for peer forwarder request documents (#2197)__ + + [Chase](mailto:62891993+engechas@users.noreply.github.com) - Tue, 31 Jan 2023 16:42:25 -0600 + + + * Implement batching for peer forwarder request documents + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + * Add configurable forwarding_batch_timeout for low-traffic scenarios + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + * Slight refactors and add unit tests for batching + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + * Increase YAML deserialization size + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + * Refactor for clarity and flush all available batches on each iteration + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + * Fix typo in FORWARDING + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + * Use getOrDefault when checking last flushed time + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + --------- + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + +* __Integration test for a pipeline with multiple process workers (#2017)__ + + [David Venable](mailto:dlv@amazon.com) - Tue, 31 Jan 2023 14:29:38 -0600 + + + Created an integration test for a pipeline with multiple process workers. + Signed-off-by: David Venable <dlv@amazon.com> + +* __Add clientTimeout to peer forwarder configuration, optimize CPF seria… (#2190)__ + + [Chase](mailto:62891993+engechas@users.noreply.github.com) - Tue, 31 Jan 2023 11:53:15 -0600 + + + * Add clientTimeout to peer forwarder configuration, optimize CPF serialization + + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + * Fix miss in rename and use lower write timeout rather than higher request + timeout + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + * Add missing private + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + --------- + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + +* __Bump hibernate-validator in /data-prepper-core (#1854)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Mon, 30 Jan 2023 10:29:58 -0600 + + + Bumps [hibernate-validator](https://github.com/hibernate/hibernate-validator) + from 7.0.5.Final to 8.0.0.Final. + - [Release notes](https://github.com/hibernate/hibernate-validator/releases) + - + [Changelog](https://github.com/hibernate/hibernate-validator/blob/main/changelog.txt) + + - + [Commits](https://github.com/hibernate/hibernate-validator/compare/7.0.5.Final...8.0.0.Final) + + + --- + updated-dependencies: + - dependency-name: org.hibernate.validator:hibernate-validator + dependency-type: direct:production + update-type: version-update:semver-major + ... + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Bump jackson-datatype-jdk8 from 2.13.3 to 2.14.1 in /data-prepper-api (#2101)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Mon, 30 Jan 2023 10:25:40 -0600 + + + Bumps jackson-datatype-jdk8 from 2.13.3 to 2.14.1. + + --- + updated-dependencies: + - dependency-name: com.fasterxml.jackson.datatype:jackson-datatype-jdk8 + dependency-type: direct:production + update-type: version-update:semver-minor + ... + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Support OpenTelementry Logs (#1372)__ + + [kmssap](mailto:100778246+kmssap@users.noreply.github.com) - Sat, 28 Jan 2023 09:21:54 -0600 + + + * Introduce Support for OpenTelemetry Logs + Signed-off-by: Kai Sternad <kai.sternad@sap.com> + Co-authored-by: Tomas Longo + <tomas.longo@sap.com> + Co-authored-by: Karsten Schnitter <k.schnitter@sap.com> + + * Fix Unused Import Checkstyle Error + Signed-off-by: Kai Sternad <ksternad@sternad.de> + + * Refactor commit for new architecture + + - Move mapping from processor to source + - Move mapping logic to OtelProtoCodec + - Enhance metrics of LogsGrpcService + Signed-off-by: Kai Sternad <ksternad@sternad.de> + + * Remove Otel Logs Processor + Signed-off-by: Kai Sternad <ksternad@sternad.de> + + * Fix JavaDoc Version + Signed-off-by: Kai Sternad <ksternad@sternad.de> + + * Improve documentation of metrics + Signed-off-by: Kai Sternad <ksternad@sternad.de> + + * Remove Otel logs processor from settings.gradle + Signed-off-by: Kai Sternad <ksternad@sternad.de> + + --------- + Signed-off-by: Kai Sternad <kai.sternad@sap.com> + Signed-off-by: Kai Sternad + <ksternad@sternad.de> + Co-authored-by: Kai Sternad <kai.sternad@sap.com> + + Co-authored-by: Tomas Longo <tomas.longo@sap.com> + Co-authored-by: Karsten + Schnitter <k.schnitter@sap.com> + Co-authored-by: Kai Sternad + <ksternad@sternad.de> + +* __Optimize buffer reads, allow delay to be 0 (#2189)__ + + [Chase](mailto:62891993+engechas@users.noreply.github.com) - Fri, 27 Jan 2023 17:11:49 -0600 + + + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + +* __ENH: implement custom LogEventPatternConverter for desensitization on Data Prepper logging (#2188)__ + + [Qi Chen](mailto:qchea@amazon.com) - Fri, 27 Jan 2023 16:27:09 -0600 + + + * ADD: SensitiveArgumentMaskingConverter + Signed-off-by: George Chen <qchea@amazon.com> + +* __Generated THIRD-PARTY file for bd60dcc (#2201)__ + + [opensearch-trigger-bot[bot]](mailto:98922864+opensearch-trigger-bot[bot]@users.noreply.github.com) - Fri, 27 Jan 2023 14:30:58 -0600 + + + Signed-off-by: GitHub <noreply@github.com> + Co-authored-by: asifsmohammed + <asifsmohammed@users.noreply.github.com> + +* __Updated OpenSearch version to 1.3.7 (#2191)__ + + [Asif Sohail Mohammed](mailto:nsifmoh@amazon.com) - Fri, 27 Jan 2023 09:24:13 -0600 + + + Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com> + Signed-off-by: Asif + Sohail Mohammed <mdasifsohail7@gmail.com> + +* __Fix Data Prepper to not terminate on invalid open telemetry metric/trace data (#2176)__ + + [kkondaka](mailto:41027584+kkondaka@users.noreply.github.com) - Thu, 26 Jan 2023 19:03:00 -0600 + + + * Rebased to latest and removed unnecessary System.out.println + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Modified the test to send one valid record and one invalid record, as per the + review comments + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + Co-authored-by: Krishna + Kondaka <krishkdk@amazon.com> + +* __Add configurations and metrics to OTelTraceRawProcessor (#2164)__ + + [David Venable](mailto:dlv@amazon.com) - Thu, 26 Jan 2023 19:01:47 -0600 + + + Updated the OTelTraceRawProcessor to use the new plugin configuration model. + Added two plugin metrics to tracking the usage of two of the collections used + in this processor. + Signed-off-by: David Venable <dlv@amazon.com> + +* __Don't block on forwarding request response, populate records that failed to forward in local CPF buffer (#2175)__ + + [Chase](mailto:62891993+engechas@users.noreply.github.com) - Thu, 26 Jan 2023 17:26:14 -0600 + + + * Handle peer forwarding in the background + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + * Rework tests to pass with new implementation + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + * Small tweaks to fix the diff + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + * Refactor to use Consumer<AggregateHttpResponse> and ExecutorService for + background forwarding work + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + * Add unit tests for processFailedRequestsLocally + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + * Add more unit tests + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + * Move requestsReceivedFromPeers to HttpService to avoid gauge usage + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + * Coming full circle + an ExecutorService + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + * Add missing private modifier + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + * Reuse existing clientThreadCount parameter, cleanup unused code + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + * Fix log message + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + +* __Add support to convert metrics to json strings without flattening attributes - issue #2146 (#2163)__ + + [kkondaka](mailto:41027584+kkondaka@users.noreply.github.com) - Thu, 26 Jan 2023 15:45:02 -0600 + + + Add support to convert metrics to json strings without flattening attributes + field - issue #2146 + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + +* __ENH: add and populate markers (#2180)__ + + [Qi Chen](mailto:qchea@amazon.com) - Thu, 26 Jan 2023 11:31:25 -0600 + + + * ENH: add and populate markers + Signed-off-by: George Chen <qchea@amazon.com> + +* __Fixes links to use org/opensearch from com/amazon now that this is the correct the package name. (#2186)__ + + [David Venable](mailto:dlv@amazon.com) - Wed, 25 Jan 2023 08:29:54 -0600 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Added AWS STS header override configurations for OpenSearch sink/S3 source (#1898)__ + + [David Venable](mailto:dlv@amazon.com) - Tue, 24 Jan 2023 17:17:13 -0600 + + + Added AWS STS header override configurations for the OpenSearch sink and the + Amazon S3 source. Resolves #1888. + Signed-off-by: David Venable <dlv@amazon.com> + +* __ENH: add buffer records overflow metrics (#2170)__ + + [Qi Chen](mailto:qchea@amazon.com) - Tue, 24 Jan 2023 09:51:27 -0600 + + + * ENH: add buffer records overflow metrics + Signed-off-by: George Chen <qchea@amazon.com> + +* __Bump decode-uri-component in /release/staging-resources-cdk (#2064)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Mon, 23 Jan 2023 19:49:14 -0600 + + + Bumps + [decode-uri-component](https://github.com/SamVerschueren/decode-uri-component) + from 0.2.0 to 0.2.2. + - [Release + notes](https://github.com/SamVerschueren/decode-uri-component/releases) + - + [Commits](https://github.com/SamVerschueren/decode-uri-component/compare/v0.2.0...v0.2.2) + + + --- + updated-dependencies: + - dependency-name: decode-uri-component + dependency-type: indirect + ... + Signed-off-by: dependabot[bot] <support@github.com> + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __New aggregate action - percent sampler (#2096)__ + + [kkondaka](mailto:41027584+kkondaka@users.noreply.github.com) - Mon, 23 Jan 2023 17:36:31 -0600 + + + Percent Sampler aggregate action + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + +* __Implemented a heap-based circuit breaker (#2155)__ + + [David Venable](mailto:dlv@amazon.com) - Mon, 23 Jan 2023 17:11:16 -0600 + + + Implemented a heap-based circuit breaker. This circuit breaker will prevent + entry buffers from accepting events after the heap usage reaches a specified + value. This checks for heap usage in a background thread and updates the state, + which the buffer will then use to determine if the circuit breaker is open or + closed. This also signals to the JVM to start a GC when the threshold is + reached. Resolves #2150. + Signed-off-by: David Venable <dlv@amazon.com> + +* __Test against multiple OTel version - Issue #1963 (#2154)__ + + [kkondaka](mailto:41027584+kkondaka@users.noreply.github.com) - Mon, 23 Jan 2023 15:08:18 -0600 + + + * Test against multiple OTel version - Issue #1963 + Signed-off-by: Krishna + Kondaka <krishkdk@amazon.com> + Co-authored-by: Krishna Kondaka + <krishkdk@amazon.com> + +* __Bump byte-buddy in /data-prepper-plugins/opensearch (#2161)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Sat, 21 Jan 2023 14:48:03 -0600 + + + Bumps [byte-buddy](https://github.com/raphw/byte-buddy) from 1.12.18 to + 1.12.22. + - [Release notes](https://github.com/raphw/byte-buddy/releases) + - [Changelog](https://github.com/raphw/byte-buddy/blob/master/release-notes.md) + + - + [Commits](https://github.com/raphw/byte-buddy/compare/byte-buddy-1.12.18...byte-buddy-1.12.22) + + + --- + updated-dependencies: + - dependency-name: net.bytebuddy:byte-buddy + dependency-type: direct:production + update-type: version-update:semver-patch + ... + Signed-off-by: dependabot[bot] <support@github.com> + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Bump byte-buddy-agent in /data-prepper-plugins/opensearch (#2160)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Fri, 20 Jan 2023 11:11:23 -0600 + + + Bumps [byte-buddy-agent](https://github.com/raphw/byte-buddy) from 1.12.20 to + 1.12.22. + - [Release notes](https://github.com/raphw/byte-buddy/releases) + - [Changelog](https://github.com/raphw/byte-buddy/blob/master/release-notes.md) + + - + [Commits](https://github.com/raphw/byte-buddy/compare/byte-buddy-1.12.20...byte-buddy-1.12.22) + + + --- + updated-dependencies: + - dependency-name: net.bytebuddy:byte-buddy-agent + dependency-type: direct:production + update-type: version-update:semver-patch + ... + Signed-off-by: dependabot[bot] <support@github.com> + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __New Aggregate Action - Event Rate Limiter (#2090)__ + + [kkondaka](mailto:41027584+kkondaka@users.noreply.github.com) - Thu, 19 Jan 2023 17:28:12 -0600 + + + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + Co-authored-by: Krishna + Kondaka <krishkdk@amazon.com> + +* __Bump byte-buddy-agent in /data-prepper-plugins/opensearch (#2100)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Thu, 19 Jan 2023 16:22:01 -0600 + + + Bumps [byte-buddy-agent](https://github.com/raphw/byte-buddy) from 1.12.18 to + 1.12.20. + - [Release notes](https://github.com/raphw/byte-buddy/releases) + - [Changelog](https://github.com/raphw/byte-buddy/blob/master/release-notes.md) + + - + [Commits](https://github.com/raphw/byte-buddy/compare/byte-buddy-1.12.18...byte-buddy-1.12.20) + + + --- + updated-dependencies: + - dependency-name: net.bytebuddy:byte-buddy-agent + dependency-type: direct:production + update-type: version-update:semver-patch + ... + Signed-off-by: dependabot[bot] <support@github.com> + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Added s3 support in Opensearch sink (#2121)__ + + [Asif Sohail Mohammed](mailto:nsifmoh@amazon.com) - Thu, 19 Jan 2023 16:21:32 -0600 + + + * Added implementation of s3 support in Opensearch sink + Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com> + +* __Added batch delay to CPF configuration (#2159)__ + + [Asif Sohail Mohammed](mailto:nsifmoh@amazon.com) - Thu, 19 Jan 2023 13:05:26 -0600 + + + Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com> + Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com> + +* __Fix for null pointer exception in remote peer forwarding (fix for issue 2123) (#2124)__ + + [kkondaka](mailto:41027584+kkondaka@users.noreply.github.com) - Wed, 18 Jan 2023 20:56:57 -0600 + + + * Fix for null pointer exception in remote peer forwarding (fix for issue + #2123) + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Addressed review comments to add a counter and not skip when an + identification key is missing + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Addressed review comments. Modified to increment the counter only when all + identification keys are missing + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Added 'final' to the local variable + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Addressed review comments + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Added a test with all missing keys + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + Co-authored-by: Krishna + Kondaka <krishkdk@amazon.com> + +* __Adds ScheduledExecutorService for Polling the RSS feed (#2140)__ + + [Shivani Shukla](mailto:67481911+sshivanii@users.noreply.github.com) - Wed, 18 Jan 2023 15:10:08 -0600 + + + * Adds ScheduledExecutor Service and runnable task + Signed-off-by: Shivani + Shukla <sshkamz@amazon.com> + + + +* __Combined two integration tests for conditional routes into one. Also fixed a bug in the tests where the data was not sent to the correct sink. (#2061)__ + + [David Venable](mailto:dlv@amazon.com) - Tue, 17 Jan 2023 21:04:00 -0600 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Add e2etest for testing log metrics (#2127)__ + + [kkondaka](mailto:41027584+kkondaka@users.noreply.github.com) - Fri, 13 Jan 2023 15:19:16 -0600 + + + * Add e2etest for testing log metrics + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Addressed review comments. + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + Co-authored-by: Krishna + Kondaka <krishkdk@amazon.com> + +* __Adds count metrics to the service_map_stateful processor (#2130)__ + + [David Venable](mailto:dlv@amazon.com) - Thu, 12 Jan 2023 15:17:03 -0600 + + + Adds new metrics to the service_map_stateful processor. These count the number + of items in the collections used by the service map. These do not have byte + sizes, but use object counts. + Signed-off-by: David Venable <dlv@amazon.com> + +* __Bump guava from 10.0.1 to 23.0 in /data-prepper-plugins/opensearch (#2050)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Wed, 11 Jan 2023 12:07:17 -0600 + + + Bumps [guava](https://github.com/google/guava) from 10.0.1 to 23.0. + - [Release notes](https://github.com/google/guava/releases) + - [Commits](https://github.com/google/guava/compare/v10.0.1...v23.0) + + --- + updated-dependencies: + - dependency-name: com.google.guava:guava + dependency-type: direct:production + update-type: version-update:semver-major + ... + Signed-off-by: dependabot[bot] <support@github.com> + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Bump json5 from 2.2.0 to 2.2.3 in /release/staging-resources-cdk (#2119)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Wed, 11 Jan 2023 12:06:05 -0600 + + + Bumps [json5](https://github.com/json5/json5) from 2.2.0 to 2.2.3. + - [Release notes](https://github.com/json5/json5/releases) + - [Changelog](https://github.com/json5/json5/blob/main/CHANGELOG.md) + - [Commits](https://github.com/json5/json5/compare/v2.2.0...v2.2.3) + + --- + updated-dependencies: + - dependency-name: json5 + dependency-type: indirect + ... + Signed-off-by: dependabot[bot] <support@github.com> + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Updated MAINTAINERS.md to match recommended opensearch-project format. (#2117)__ + + [Daniel (dB.) Doubrovkine](mailto:dblock@dblock.org) - Thu, 5 Jan 2023 14:58:26 -0600 + + + Signed-off-by: dblock <dblock@amazon.com> + Signed-off-by: dblock <dblock@amazon.com> + +* __Update OTelProtoCodec for InstrumentationLibrary to InstrumentationScope rename (#2114)__ + + [Chase](mailto:62891993+engechas@users.noreply.github.com) - Wed, 4 Jan 2023 09:40:06 -0600 + + + * Add support for ScopeSpans migration to otel-trace-raw-processor + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + * Remove unused import + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + * Run spotless + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + +* __Update slf4j-simple; update log4j-slf4j-impl to log4j-slf4j2-impl (#2113)__ + + [Hai Yan](mailto:8153134+oeyh@users.noreply.github.com) - Tue, 3 Jan 2023 17:16:03 -0600 + + + Signed-off-by: Hai Yan <oeyh@amazon.com> + +* __MOD: fix typos (#2084)__ + + [Shanelle Marasigan](mailto:39988782+rmarasigan@users.noreply.github.com) - Tue, 3 Jan 2023 15:51:49 -0600 + + + Signed-off-by: Russianhielle Marasigan <russianhielle@gmail.com> + +* __Created a GitHub Action to generate the Third Party report. (#2033)__ + + [David Venable](mailto:dlv@amazon.com) - Thu, 22 Dec 2022 15:01:37 -0800 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __updating S3 source documentation to include all codecs (#2091)__ + + [Christopher Manning](mailto:cmanning09@users.noreply.github.com) - Thu, 22 Dec 2022 16:04:08 -0600 + + + Signed-off-by: Christopher Manning <cmanning09@users.noreply.github.com> + +* __Provide a type conversion / cast processor #2010 (#2020)__ + + [kkondaka](mailto:41027584+kkondaka@users.noreply.github.com) - Thu, 22 Dec 2022 15:50:01 -0600 + + + * Provide a type conversion / cast processor #2010 + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + Co-authored-by: Krishna + Kondaka <krishkdk@amazon.com> + +* __Histogram Aggregate Action - Added duration and fixed end time in the aggregated output (#2085)__ + + [kkondaka](mailto:41027584+kkondaka@users.noreply.github.com) - Thu, 22 Dec 2022 12:20:08 -0600 + + + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + +* __Fix CVE-2022-41881, CVE-2021-21290 and CVE-2022-41915 (#2093)__ + + [Asif Sohail Mohammed](mailto:nsifmoh@amazon.com) - Wed, 21 Dec 2022 16:36:04 -0800 + + + Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com> + +* __Fix: CVE-2022-3509, CVE-2022-3510 (#2079)__ + + [Asif Sohail Mohammed](mailto:nsifmoh@amazon.com) - Mon, 19 Dec 2022 10:14:22 -0800 + + + Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com> + +* __Add robust retry strategy to AcmClients (#2082)__ + + [Chase](mailto:62891993+engechas@users.noreply.github.com) - Fri, 16 Dec 2022 19:29:07 -0600 + + + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + +* __Updated info logs to debug level (#2083)__ + + [Asif Sohail Mohammed](mailto:nsifmoh@amazon.com) - Fri, 16 Dec 2022 16:35:28 -0800 + + + Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com> + +* __Fix: CVE-2022-36944 (#2080)__ + + [Asif Sohail Mohammed](mailto:nsifmoh@amazon.com) - Fri, 16 Dec 2022 15:59:07 -0800 + + + Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com> + + +* __Histogram aggregate action (#2078)__ + + [kkondaka](mailto:41027584+kkondaka@users.noreply.github.com) - Fri, 16 Dec 2022 11:22:23 -0800 + + + * Add Histogram aggregation action + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + Co-authored-by: Krishna + Kondaka <krishkdk@amazon.com> + +* __Updated jackson bom dependency (#2068)__ + + [Asif Sohail Mohammed](mailto:nsifmoh@amazon.com) - Thu, 15 Dec 2022 14:44:59 -0800 + + + Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com> + +* __Anomaly detector (#2058)__ + + [kkondaka](mailto:41027584+kkondaka@users.noreply.github.com) - Thu, 15 Dec 2022 16:28:28 -0600 + + + * Add support for anomaly detection in the pipeline with new anomaly detector + processor + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + Co-authored-by: Krishna + Kondaka <krishkdk@amazon.com> + +* __Add retry strategy to StsClient used for sigv4 auth against OpenSearch sinks (#2069)__ + + [Chase](mailto:62891993+engechas@users.noreply.github.com) - Thu, 15 Dec 2022 11:26:50 -0600 + + + Add retry strategy to STS client used for sigv4 auth + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + +* __Parse RSS feed URL items and convert Item to Event (#2073)__ + + [Shivani Shukla](mailto:67481911+sshivanii@users.noreply.github.com) - Wed, 14 Dec 2022 13:16:46 -0600 + + + Signed-off-by: Shivani Shukla <sshkamz@amazon.com> + + * Adds Jackson Jdk8Module to enable usage of Optional in Rss Item model + + Signed-off-by: Shivani Shukla <sshkamz@amazon.com> + + +* __Fix: Updated parse json processor documentation (#2071)__ + + [Asif Sohail Mohammed](mailto:nsifmoh@amazon.com) - Wed, 14 Dec 2022 08:47:12 -0800 + + + Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com> + +* __Count Aggregate Action - fix aggregation temporality (#2067)__ + + [kkondaka](mailto:41027584+kkondaka@users.noreply.github.com) - Thu, 8 Dec 2022 17:21:41 -0600 + + + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + +* __Add support for count aggregate action (#2034)__ + + [kkondaka](mailto:41027584+kkondaka@users.noreply.github.com) - Wed, 7 Dec 2022 12:51:11 -0600 + + + Add support for count aggregate action with raw and otel_metrics output format + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + +* __Setup boilerplate for RSS Source Plugin (#2062)__ + + [Shivani Shukla](mailto:67481911+sshivanii@users.noreply.github.com) - Wed, 7 Dec 2022 12:44:37 -0600 + + + * Adds boilerplate config and code for rss source + Signed-off-by: Shivani + Shukla <sshkamz@amazon.com> + + * Adds RSS Source plugin class + Signed-off-by: Shivani Shukla + <sshkamz@amazon.com> + + * Adds Document Event Type + Signed-off-by: Shivani Shukla <sshkamz@amazon.com> + + * Adds Document interface and JacksonDocument class + Signed-off-by: Shivani + Shukla <sshkamz@amazon.com> + + * Adds a simple unit test checking default for pollingFrequency + Signed-off-by: + Shivani Shukla <sshkamz@amazon.com> + +* __Bump slf4j-simple in /data-prepper-logstash-configuration (#2053)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Fri, 2 Dec 2022 14:04:27 -0600 + + + Bumps [slf4j-simple](https://github.com/qos-ch/slf4j) from 1.7.36 to 2.0.5. + - [Release notes](https://github.com/qos-ch/slf4j/releases) + - [Commits](https://github.com/qos-ch/slf4j/compare/v_1.7.36...v_2.0.5) + + --- + updated-dependencies: + - dependency-name: org.slf4j:slf4j-simple + dependency-type: direct:production + update-type: version-update:semver-major + ... + Signed-off-by: dependabot[bot] <support@github.com> + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Bump slf4j-api from 1.7.36 to 2.0.5 (#2057)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Fri, 2 Dec 2022 14:00:34 -0600 + + + Bumps [slf4j-api](https://github.com/qos-ch/slf4j) from 1.7.36 to 2.0.5. + - [Release notes](https://github.com/qos-ch/slf4j/releases) + - [Commits](https://github.com/qos-ch/slf4j/compare/v_1.7.36...v_2.0.5) + + --- + updated-dependencies: + - dependency-name: org.slf4j:slf4j-api + dependency-type: direct:production + update-type: version-update:semver-major + ... + Signed-off-by: dependabot[bot] <support@github.com> + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Bump jsonassert from 1.5.0 to 1.5.1 in /data-prepper-api (#2043)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Thu, 1 Dec 2022 17:51:03 -0600 + + + Bumps [jsonassert](https://github.com/skyscreamer/JSONassert) from 1.5.0 to + 1.5.1. + - [Release notes](https://github.com/skyscreamer/JSONassert/releases) + - + [Changelog](https://github.com/skyscreamer/JSONassert/blob/master/CHANGELOG.md) + + - + [Commits](https://github.com/skyscreamer/JSONassert/compare/jsonassert-1.5.0...jsonassert-1.5.1) + + + --- + updated-dependencies: + - dependency-name: org.skyscreamer:jsonassert + dependency-type: direct:production + update-type: version-update:semver-patch + ... + Signed-off-by: dependabot[bot] <support@github.com> + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Bump protobuf-java-util in /data-prepper-plugins/otel-trace-raw-prepper (#2042)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Thu, 1 Dec 2022 17:50:08 -0600 + + + Bumps protobuf-java-util from 3.21.9 to 3.21.10. + + --- + updated-dependencies: + - dependency-name: com.google.protobuf:protobuf-java-util + dependency-type: direct:production + update-type: version-update:semver-patch + ... + Signed-off-by: dependabot[bot] <support@github.com> + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Router integration tests to verify that conditional pipeline routes work as expected. (#1988)__ + + [David Venable](mailto:dlv@amazon.com) - Thu, 1 Dec 2022 15:51:52 -0600 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Updated the OpenSearch libraries to 1.3.6 and run the integration tests against 1.3.6 instead of 1.3.5. Fixes #2022 (#2030)__ + + [David Venable](mailto:dlv@amazon.com) - Wed, 30 Nov 2022 12:57:39 -0600 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Removes double brace initialization and includes this as a checkstyle rule to prevent future use. (#2035)__ + + [David Venable](mailto:dlv@amazon.com) - Tue, 22 Nov 2022 16:16:14 -0600 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Implemented additional metrics for the S3 source. (#2028)__ + + [David Venable](mailto:dlv@amazon.com) - Tue, 22 Nov 2022 14:25:20 -0600 + + + Implemented additional metrics for the S3 source. Resolves #2024 + Signed-off-by: David Venable <dlv@amazon.com> + +* __Use the same compression engines for GZIP and NONE in the AUTOMATIC compression engine. Fixes #2026. (#2027)__ + + [David Venable](mailto:dlv@amazon.com) - Mon, 21 Nov 2022 11:15:05 -0600 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Log the number of messages received from the SQS queue, including a count of the number of messages that will need to be processed. Also, include logging of deletes at the debug level. (#2011)__ + + [David Venable](mailto:dlv@amazon.com) - Sat, 19 Nov 2022 13:06:23 -0600 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Add PluginMetrics in the Auth Plugin for Http, OTel and Metrics Source (#2023)__ + + [Dinu John](mailto:86094133+dinujoh@users.noreply.github.com) - Fri, 18 Nov 2022 21:27:15 -0600 + + + * Add PluginMetrics in the Auth Plugin for Http, OTel and Metrics Source + Issue: https://github.com/opensearch-project/data-prepper/issues/2007 + + Signed-off-by: Dinu John <dinujohn@amazon.com> + + * Added unit test for verifying PipelineDescription in DefaultPluginFactory + class and fixed review comments on unit test + Signed-off-by: Dinu John <86094133+dinujoh@users.noreply.github.com> + + * Unit test update + Signed-off-by: Dinu John <86094133+dinujoh@users.noreply.github.com> + + * Unit test update + Signed-off-by: Dinu John <86094133+dinujoh@users.noreply.github.com> + Signed-off-by: Dinu John <dinujohn@amazon.com> + Signed-off-by: Dinu John + <86094133+dinujoh@users.noreply.github.com> + +* __Add when condition to aggregate processor (#2018)__ + + [kkondaka](mailto:41027584+kkondaka@users.noreply.github.com) - Fri, 18 Nov 2022 15:31:54 -0600 + + + * Add when condition to aggregate processor + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Add when condition to aggregate processor - addressed review comments + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Add when condition to aggregate processor - Fixed check style test errors + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + Co-authored-by: Krishna + Kondaka <krishkdk@amazon.com> + +* __Dynamic Index Name in OpenSearch sink - Resolves #1459 (#1999)__ + + [kkondaka](mailto:41027584+kkondaka@users.noreply.github.com) - Thu, 17 Nov 2022 20:36:47 -0600 + + + * Dynamic Index Name in OpenSearch sink #1459 + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Dynamic Index Name in OpenSearch sink #1459 -- updated documentation + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Dynamic Index Name in OpenSearch sink #1459 -- fixed a bug and increased test + coverage + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Addressed review comments to add more tests and re-design index manager to + accommodate dynamic indexes + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Added tests for DynamicIndexManager + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Dynamic Index Name in OpenSearch sink #1459 -- changed cache weigher to have + constant value + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Dynamic Index Name in OpenSearch sink #1459 -- addressed review comments + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Dynamic Index Name in OpenSearch sink - Resolves Issue #1459 -- addressed + review comments + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Dynamic Index Name in OpenSearch sink - Fixed checkSytleMain issues in + opensearch + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Dynamic Index Name in OpenSearch sink - Fixed spotlessJavaCheck issues in + opensearch + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + Co-authored-by: Krishna + Kondaka <krishkdk@amazon.com> + +* __updated samples for dataperpper 2 (#2019)__ + + [Arunachalam Lakshmanan](mailto:arnlaksh@amazon.com) - Thu, 17 Nov 2022 10:02:31 -0600 + + + * updated samples for dataperpper 2 + Signed-off-by: Arun Lakshmanan <arnlaksh@amazon.com> + + * updated samples for dataperpper 2 + Signed-off-by: Arun Lakshmanan <arnlaksh@amazon.com> + Signed-off-by: Arun Lakshmanan <arnlaksh@amazon.com> + +* __Add IntelliJ's .iml file extension to the .gitignore. (#2016)__ + + [David Venable](mailto:dlv@amazon.com) - Wed, 16 Nov 2022 14:15:00 -0600 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Bump protobuf-java-util in /data-prepper-plugins/otel-trace-raw-prepper (#2001)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Fri, 11 Nov 2022 09:58:40 -0600 + + + Bumps protobuf-java-util from 3.19.4 to 3.21.9. + + --- + updated-dependencies: + - dependency-name: com.google.protobuf:protobuf-java-util + dependency-type: direct:production + update-type: version-update:semver-minor + ... + Signed-off-by: dependabot[bot] <support@github.com> + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Data Prepper Core integration tests (#1949)__ + + [David Venable](mailto:dlv@amazon.com) - Thu, 10 Nov 2022 09:50:03 -0600 + + + Creates an integration test source set for data-prepper-core and adds a small + framework for running integration tests on data-prepper-core functionality. + Signed-off-by: David Venable <dlv@amazon.com> + +* __Adding Chase Engelbrecht to the MAINTAINERS.md. (#2005)__ + + [David Venable](mailto:dlv@amazon.com) - Wed, 9 Nov 2022 09:55:59 -0600 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Adding Hai Yan to the maintainers. (#2004)__ + + [David Venable](mailto:dlv@amazon.com) - Tue, 8 Nov 2022 16:13:31 -0600 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Bump opensearch-java from 1.0.0 to 2.1.0 (#1733)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Tue, 8 Nov 2022 08:17:20 -0600 + + + Bumps [opensearch-java](https://github.com/opensearch-project/opensearch-java) + from 1.0.0 to 2.1.0. + - [Release + notes](https://github.com/opensearch-project/opensearch-java/releases) + - + [Commits](https://github.com/opensearch-project/opensearch-java/compare/v1.0.0...v2.1.0) + + + --- + updated-dependencies: + - dependency-name: org.opensearch.client:opensearch-java + dependency-type: direct:production + update-type: version-update:semver-major + ... + Signed-off-by: dependabot[bot] <support@github.com> + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Bump assertj-core in /data-prepper-plugins/otel-trace-raw-prepper (#2003)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Tue, 8 Nov 2022 08:15:44 -0600 + + + Bumps [assertj-core](https://github.com/assertj/assertj-core) from 3.22.0 to + 3.23.1. + - [Release notes](https://github.com/assertj/assertj-core/releases) + - + [Commits](https://github.com/assertj/assertj-core/compare/assertj-core-3.22.0...assertj-core-3.23.1) + + + --- + updated-dependencies: + - dependency-name: org.assertj:assertj-core + dependency-type: direct:production + update-type: version-update:semver-minor + ... + Signed-off-by: dependabot[bot] <support@github.com> + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Bump awaitility in /data-prepper-plugins/otel-trace-raw-prepper (#2002)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Tue, 8 Nov 2022 08:15:12 -0600 + + + Bumps [awaitility](https://github.com/awaitility/awaitility) from 4.1.1 to + 4.2.0. + - [Release notes](https://github.com/awaitility/awaitility/releases) + - + [Changelog](https://github.com/awaitility/awaitility/blob/master/changelog.txt) + + - + [Commits](https://github.com/awaitility/awaitility/compare/awaitility-4.1.1...awaitility-4.2.0) + + + --- + updated-dependencies: + - dependency-name: org.awaitility:awaitility + dependency-type: direct:production + update-type: version-update:semver-minor + ... + Signed-off-by: dependabot[bot] <support@github.com> + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Support remaining OpenTelemetry Metrics proto spec features (#1335)__ + + [kmssap](mailto:ksternad@sternad.de) - Fri, 4 Nov 2022 10:58:14 -0500 + + + * Bump OTEL proto version + Signed-off-by: Kai Sternad <kai.sternad@sap.com> + Co-authored-by: Tomas Longo + <tomas.longo@sap.com> + Co-authored-by: Karsten Schnitter <k.schnitter@sap.com> + + * Support OTEL ScopeMetrics + Signed-off-by: Kai Sternad <kai.sternad@sap.com> + Co-authored-by: Tomas Longo + <tomas.longo@sap.com> + Co-authored-by: Karsten Schnitter <k.schnitter@sap.com> + + * Add support for OTEL schemaUrl + Signed-off-by: Kai Sternad <kai.sternad@sap.com> + Co-authored-by: Tomas Longo + <tomas.longo@sap.com> + Co-authored-by: Karsten Schnitter <k.schnitter@sap.com> + + * Add exemplars to metrics plugin + Signed-off-by: Kai Sternad <kai.sternad@sap.com> + Co-authored-by: Tomas Longo + <tomas.longo@sap.com> + Co-authored-by: Karsten Schnitter <k.schnitter@sap.com> + + * Add metrics flags + Signed-off-by: Kai Sternad <kai.sternad@sap.com> + Co-authored-by: Tomas Longo + <tomas.longo@sap.com> + Co-authored-by: Karsten Schnitter <k.schnitter@sap.com> + + * Add support for Exponential Histogram + Signed-off-by: Kai Sternad <kai.sternad@sap.com> + Co-authored-by: Tomas Longo + <tomas.longo@sap.com> + Co-authored-by: Karsten Schnitter <k.schnitter@sap.com> + + * Add config switch for histogram bucket calculation + Signed-off-by: Kai Sternad <kai.sternad@sap.com> + Co-authored-by: Tomas Longo + <tomas.longo@sap.com> + Co-authored-by: Karsten Schnitter <k.schnitter@sap.com> + + * Refactor Otel Metrics Proto + Signed-off-by: Kai Sternad <kai.sternad@sap.com> + Co-authored-by: Tomas Longo + <tomas.longo@sap.com> + Co-authored-by: Karsten Schnitter <k.schnitter@sap.com> + + * Change config property to snake_case + Signed-off-by: Kai Sternad <kai.sternad@sap.com> + Co-authored-by: Tomas Longo + <tomas.longo@sap.com> + Co-authored-by: Karsten Schnitter <k.schnitter@sap.com> + + * Fix JavaDoc + Signed-off-by: Kai Sternad <kai.sternad@sap.com> + Co-authored-by: Tomas Longo + <tomas.longo@sap.com> + Co-authored-by: Karsten Schnitter <k.schnitter@sap.com> + + * Remove Clock from tests + Signed-off-by: Kai Sternad <kai.sternad@sap.com> + Co-authored-by: Tomas Longo + <tomas.longo@sap.com> + Co-authored-by: Karsten Schnitter <k.schnitter@sap.com> + + * Change config parameters + + - Introduce allowed max scale + - Invert histogram calculation params + Signed-off-by: Kai Sternad <kai.sternad@sap.com> + Co-authored-by: Tomas Longo + <tomas.longo@sap.com> + Co-authored-by: Karsten Schnitter <k.schnitter@sap.com> + + * Address review comments + + - Remove unused import, breaking Checkstyle + - Change Exponential Histogram filter + - Add lenient to some Mockito calls + - Clarify metrics processor documentation + Signed-off-by: Kai Sternad <kai.sternad@sap.com> + Co-authored-by: Tomas Longo + <tomas.longo@sap.com> + Co-authored-by: Karsten Schnitter <k.schnitter@sap.com> + + * Fix OtelMetricsRawProcessorConfigTest + Signed-off-by: Kai Sternad <kai.sternad@sap.com> + Co-authored-by: Tomas Longo + <tomas.longo@sap.com> + Co-authored-by: Karsten Schnitter <k.schnitter@sap.com> + + * Change ExponentialHistogram Bucket Calculation + + - Precompute all possible bucket bounds + - Consider negative offset + Signed-off-by: Kai Sternad <kai.sternad@sap.com> + Co-authored-by: Tomas Longo + <tomas.longo@sap.com> + Co-authored-by: Karsten Schnitter <k.schnitter@sap.com> + + * Fix e2e otel dependency coordinates + Signed-off-by: Kai Sternad <kai.sternad@sap.com> + Co-authored-by: Tomas Longo + <tomas.longo@sap.com> + Co-authored-by: Karsten Schnitter <k.schnitter@sap.com> + + * Fix dependency coordinate for otel + Signed-off-by: Kai Sternad <kai.sternad@sap.com> + Signed-off-by: Kai Sternad <kai.sternad@sap.com> + Co-authored-by: Kai Sternad + <kai.sternad@sap.com> + Co-authored-by: Tomas Longo <tomas.longo@sap.com> + + Co-authored-by: Karsten Schnitter <k.schnitter@sap.com> + +* __Bump byte-buddy in /data-prepper-plugins/opensearch (#1993)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Wed, 2 Nov 2022 20:22:12 -0500 + + + Bumps [byte-buddy](https://github.com/raphw/byte-buddy) from 1.12.17 to + 1.12.18. + - [Release notes](https://github.com/raphw/byte-buddy/releases) + - [Changelog](https://github.com/raphw/byte-buddy/blob/master/release-notes.md) + + - + [Commits](https://github.com/raphw/byte-buddy/compare/byte-buddy-1.12.17...byte-buddy-1.12.18) + + + --- + updated-dependencies: + - dependency-name: net.bytebuddy:byte-buddy + dependency-type: direct:production + update-type: version-update:semver-patch + ... + Signed-off-by: dependabot[bot] <support@github.com> + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Bump byte-buddy-agent in /data-prepper-plugins/opensearch (#1992)__ + + [dependabot[bot]](mailto:49699333+dependabot[bot]@users.noreply.github.com) - Wed, 2 Nov 2022 19:20:41 -0500 + + + Bumps [byte-buddy-agent](https://github.com/raphw/byte-buddy) from 1.12.17 to + 1.12.18. + - [Release notes](https://github.com/raphw/byte-buddy/releases) + - [Changelog](https://github.com/raphw/byte-buddy/blob/master/release-notes.md) + + - + [Commits](https://github.com/raphw/byte-buddy/compare/byte-buddy-1.12.17...byte-buddy-1.12.18) + + + --- + updated-dependencies: + - dependency-name: net.bytebuddy:byte-buddy-agent + dependency-type: direct:production + update-type: version-update:semver-patch + ... + Signed-off-by: dependabot[bot] <support@github.com> + Signed-off-by: dependabot[bot] <support@github.com> + Co-authored-by: + dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> + +* __Fixes for issues #1456 and #1458 - support for complex document ID and routing ID (#1966)__ + + [kkondaka](mailto:41027584+kkondaka@users.noreply.github.com) - Wed, 2 Nov 2022 18:57:46 -0500 + + + * Fixes for issues #1456 and #1458 - support for complex document ID and + routing ID + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Fixes for issues #1456 and #1458 - updated README.md and added unit tests + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Fixes for issues #1456 and #1458 - addressed review comments + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Fixes for issues #1456 and #1458 - fixed check style build failure + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Add OpenSearch e2e test to github workflows + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Addressed review comments. Deleted e2e test for OpenSearch + DocumentId/RoutingField testing + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Remove e2e test entry for removed e2e test + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Addressed review comments and added tests for fromStringAndOptionals method + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Fixed checkstyle failure + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + Co-authored-by: Krishna + Kondaka <krishkdk@amazon.com> + +* __Update a few logs that were dumping the actual customer logs and traces into the DataPrepper logs on failure. Logging the request sizes instead, to avoid customer data being logged (#1989)__ + + [Deep Datta](mailto:18663532+deepdatta@users.noreply.github.com) - Mon, 31 Oct 2022 14:30:12 -0500 + + + Signed-off-by: Deep Datta <deedatta@amazon.com> + +* __Updated release notes and change log with spring change (#1981)__ + + [Asif Sohail Mohammed](mailto:nsifmoh@amazon.com) - Thu, 27 Oct 2022 17:02:02 -0500 + + + Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com> + + +* __Created release notes for Data Prepper 2.0.1. (#1969)__ + + [David Venable](mailto:dlv@amazon.com) - Thu, 27 Oct 2022 13:28:47 -0500 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Added release notes for 1.5.2 (#1976)__ + + [Asif Sohail Mohammed](mailto:nsifmoh@amazon.com) - Thu, 27 Oct 2022 12:44:57 -0500 + + + Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com> + +* __Added change log for 1.5.2 (#1977)__ + + [Asif Sohail Mohammed](mailto:nsifmoh@amazon.com) - Thu, 27 Oct 2022 12:28:10 -0500 + + + Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com> + +* __Added the change log for 2.0.1. (#1968)__ + + [David Venable](mailto:dlv@amazon.com) - Thu, 27 Oct 2022 10:38:17 -0500 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Expressions with null (#1946)__ + + [kkondaka](mailto:41027584+kkondaka@users.noreply.github.com) - Tue, 25 Oct 2022 15:00:55 -0500 + + + * fix for issue #1136 Add null support to DataPrepperExpressions + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * fix for issue #1136 Add null support to DataPrepperExpressions - added more + tests + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * fix for issue #1136 Add null support to DataPrepperExpressions - updated docs + + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Addressed review comments - added test cases and updated document + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + Co-authored-by: Krishna + Kondaka <krishkdk@amazon.com> + +* __Reduce smoke test timeout to 8 minutes from 30 minutes. These tests tend to pass within 3 minutes in my personal GitHub branch. So this leaves quite a bit of buffer time. It helps speed up retrying failures from flaky tests. (#1956)__ + + [David Venable](mailto:dlv@amazon.com) - Fri, 21 Oct 2022 09:48:44 -0500 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Attempt to reduce flakiness in RandomStringSourceTests by using awaitility. Split tests into two. JUnit 5. (#1921)__ + + [David Venable](mailto:dlv@amazon.com) - Fri, 21 Oct 2022 09:48:29 -0500 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Run smoke tests against OpenSearch 1.3.6. (#1955)__ + + [David Venable](mailto:dlv@amazon.com) - Fri, 21 Oct 2022 06:47:01 -0500 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Use Python grpcio 1.50.0 in smoke tests to reduce time to run. (#1954)__ + + [David Venable](mailto:dlv@amazon.com) - Fri, 21 Oct 2022 06:46:43 -0500 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Delete s3:TestEvent objects and log them when they are found in the SQS queue. Resolves #1924. (#1939)__ + + [David Venable](mailto:dlv@amazon.com) - Thu, 20 Oct 2022 12:56:56 -0500 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Add ExecutorService to DataPrepperServer (#1948)__ + + [Chase](mailto:62891993+engechas@users.noreply.github.com) - Thu, 20 Oct 2022 11:18:27 -0500 + + + * Add ExecutorService to DataPrepperServer + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + + * Shutdown executor service after stopping server + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + Signed-off-by: Chase Engelbrecht <engechas@amazon.com> + +* __Updated k8s manifest to suit Data Prepper 2.0 (#1928)__ + + [Rafael Gumiero](mailto:rafael.gumiero@gmail.com) - Wed, 19 Oct 2022 17:04:42 -0500 + + + * Updated new paths for pepelines/config and new processor name + Signed-off-by: Rafael Gumiero <rafael.gumiero@gmail.com> + + * Updated image version + Signed-off-by: Rafael Gumiero <rafael.gumiero@gmail.com> + + * Moved Peer Forwarder to config file + Signed-off-by: Rafael Gumiero <rafael.gumiero@gmail.com> + Signed-off-by: Rafael Gumiero <rafael.gumiero@gmail.com> + +* __Require protobuf-java-util 3.21.7 to fix #1891 (#1938)__ + + [David Venable](mailto:dlv@amazon.com) - Wed, 19 Oct 2022 12:35:33 -0500 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Bug Fix: S3 source key (#1926)__ + + [Asif Sohail Mohammed](mailto:nsifmoh@amazon.com) - Wed, 19 Oct 2022 11:39:56 -0500 + + + * Fix: S3 source key bug fix + Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com> + +* __Jackson 2.13.4.2 (#1925)__ + + [David Venable](mailto:dlv@amazon.com) - Tue, 18 Oct 2022 16:52:21 -0500 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Refactors the Data Prepper CLI argument parsing into data-prepper-main. Added an interface for the parts of DataPrepperArgs that client classes really need. (#1920)__ + + [David Venable](mailto:dlv@amazon.com) - Fri, 14 Oct 2022 10:03:06 -0500 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Fix PipelineConnector to duplicate the events (#1897)__ + + [kkondaka](mailto:41027584+kkondaka@users.noreply.github.com) - Wed, 12 Oct 2022 16:32:00 -0500 + + + * Fix string mutate processors to duplicate the events + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Fix string mutate processors to duplicate the events - made changes as per + David's suggestions + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Removed unnecessary changes leftover from 1st commit + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Modified PipelineConnector to duplicate JacksonSpan type events too. Added + testcases in PipelineConnectorTest + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Addressed review comments + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Addressed review comment and added a new testcase for JacksonSpan withData() + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * Addressed review comment and added parallel pipeline test to github/workflows + + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + + * fixed workflow failure + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> + Co-authored-by: Krishna + Kondaka <krishkdk@amazon.com> + +* __Simple duration regex did not allow for 0s or 0ms (#1910)__ + + [Taylor Gray](mailto:tylgry@amazon.com) - Tue, 11 Oct 2022 10:58:40 -0500 + + + Signed-off-by: Taylor Gray <tylgry@amazon.com> + Signed-off-by: Taylor Gray <tylgry@amazon.com> + +* __Updated the release notes for 2.0.0 (#1911)__ + + [David Venable](mailto:dlv@amazon.com) - Mon, 10 Oct 2022 17:46:26 -0500 + + + Updated the release notes for 2.0.0 + Signed-off-by: David Venable <dlv@amazon.com> + +* __Updated the change log for 2.0.0 with most recent changes. (#1909)__ + + [David Venable](mailto:dlv@amazon.com) - Mon, 10 Oct 2022 17:33:31 -0500 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Update dev version to 2.1.0-SNAPSHOT (#1904)__ + + [Hai Yan](mailto:8153134+oeyh@users.noreply.github.com) - Mon, 10 Oct 2022 13:34:46 -0500 + + + Signed-off-by: Hai Yan <oeyh@amazon.com> + +* __Increase the default buffer configurations by 25. Capacity to 12,800 and batch size to 200. (#1906)__ + + [David Venable](mailto:dlv@amazon.com) - Mon, 10 Oct 2022 11:59:38 -0500 + + + Signed-off-by: David Venable <dlv@amazon.com> + +* __Conditional routing documentation (#1894)__ + + [David Venable](mailto:dlv@amazon.com) - Mon, 10 Oct 2022 10:35:15 -0500 + + + Add documentation for conditional routing. Resolves #1890 + Signed-off-by: David Venable <dlv@amazon.com> + +* __Added change log (#1901)__ + + [Asif Sohail Mohammed](mailto:nsifmoh@amazon.com) - Fri, 7 Oct 2022 20:52:38 -0500 + + + Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com> + + +* __Adds a stack-trace to failures from OpenSearch to help with debugging issues. (#1899)__ + + [David Venable](mailto:dlv@amazon.com) - Fri, 7 Oct 2022 18:32:26 -0500 + + + Signed-off-by: David Venable <dlv@amazon.com> + +