diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f6b07d4..f3126cba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,13 @@ All notable changes to this project will be documented in this file. - Default stackableVersion to operator version ([#458]). +### Changed + +- `vector` `0.26.0` -> `0.31.0` ([#459]). +- `operator-rs` `0.44.0` -> `0.45.1` ([#458]). + [#458]: https://github.com/stackabletech/druid-operator/pull/458 +[#459]: https://github.com/stackabletech/druid-operator/pull/459 ## [23.7.0] - 2023-07-14 diff --git a/tests/kuttl-test.yaml.jinja2 b/tests/kuttl-test.yaml.jinja2 index 76b502aa..9d0b5efb 100644 --- a/tests/kuttl-test.yaml.jinja2 +++ b/tests/kuttl-test.yaml.jinja2 @@ -9,3 +9,19 @@ testDirs: startKIND: false suppress: ["events"] parallel: 2 + +# The timeout (in seconds) is used when namespaces are created or +# deleted, and, if not overridden, in TestSteps, TestAsserts, and +# Commands. If not set, the timeout is 30 seconds by default. +# +# The deletion of a namespace can take a while until all resources are +# gracefully shut down. If the timeout is reached in the meantime, even +# a successful test case is considered a failure. +# +# For instance, the termination grace period of the Vector aggregator in +# the logging tests is set to 60 seconds. If there are logs entries +# which could not be forwarded yet to the external aggregator defined in +# the VECTOR_AGGREGATOR environment variable, then the test aggregator +# uses this period of time by trying to forward the events. In this +# case, deleting a namespace with several Pods takes about 90 seconds. +timeout: 120 diff --git a/tests/templates/kuttl/logging/01-install-druid-vector-aggregator.yaml b/tests/templates/kuttl/logging/01-install-druid-vector-aggregator.yaml index ff1d0187..6b8b58a3 100644 --- a/tests/templates/kuttl/logging/01-install-druid-vector-aggregator.yaml +++ b/tests/templates/kuttl/logging/01-install-druid-vector-aggregator.yaml @@ -5,7 +5,7 @@ commands: - script: >- helm install druid-vector-aggregator vector --namespace $NAMESPACE - --version 0.19.2 + --version 0.23.0 --repo https://helm.vector.dev --values druid-vector-aggregator-values.yaml --- diff --git a/tests/templates/kuttl/logging/druid-vector-aggregator-values.yaml.j2 b/tests/templates/kuttl/logging/druid-vector-aggregator-values.yaml.j2 index d362970f..d60005d9 100644 --- a/tests/templates/kuttl/logging/druid-vector-aggregator-values.yaml.j2 +++ b/tests/templates/kuttl/logging/druid-vector-aggregator-values.yaml.j2 @@ -20,192 +20,204 @@ customConfig: type: vector version: "2" transforms: - automaticLogConfigBrokerDruid: + filteredAutomaticLogConfigBrokerDruid: type: filter inputs: [vector] condition: >- .pod == "test-druid-broker-automatic-log-config-0" && .container == "druid" - automaticLogConfigBrokerPrepare: + filteredAutomaticLogConfigBrokerPrepare: type: filter inputs: [vector] condition: >- .pod == "test-druid-broker-automatic-log-config-0" && .container == "prepare" - automaticLogConfigBrokerVector: + filteredAutomaticLogConfigBrokerVector: type: filter inputs: [vector] condition: >- .pod == "test-druid-broker-automatic-log-config-0" && .container == "vector" - customLogConfigBrokerDruid: + filteredCustomLogConfigBrokerDruid: type: filter inputs: [vector] condition: >- .pod == "test-druid-broker-custom-log-config-0" && .container == "druid" - customLogConfigBrokerPrepare: + filteredCustomLogConfigBrokerPrepare: type: filter inputs: [vector] condition: >- .pod == "test-druid-broker-custom-log-config-0" && .container == "prepare" - customLogConfigBrokerVector: + filteredCustomLogConfigBrokerVector: type: filter inputs: [vector] condition: >- .pod == "test-druid-broker-custom-log-config-0" && .container == "vector" - automaticLogConfigCoordinatorDruid: + filteredAutomaticLogConfigCoordinatorDruid: type: filter inputs: [vector] condition: >- .pod == "test-druid-coordinator-automatic-log-config-0" && .container == "druid" - automaticLogConfigCoordinatorPrepare: + filteredAutomaticLogConfigCoordinatorPrepare: type: filter inputs: [vector] condition: >- .pod == "test-druid-coordinator-automatic-log-config-0" && .container == "prepare" - automaticLogConfigCoordinatorVector: + filteredAutomaticLogConfigCoordinatorVector: type: filter inputs: [vector] condition: >- .pod == "test-druid-coordinator-automatic-log-config-0" && .container == "vector" - customLogConfigCoordinatorDruid: + filteredCustomLogConfigCoordinatorDruid: type: filter inputs: [vector] condition: >- .pod == "test-druid-coordinator-custom-log-config-0" && .container == "druid" - customLogConfigCoordinatorPrepare: + filteredCustomLogConfigCoordinatorPrepare: type: filter inputs: [vector] condition: >- .pod == "test-druid-coordinator-custom-log-config-0" && .container == "prepare" - customLogConfigCoordinatorVector: + filteredCustomLogConfigCoordinatorVector: type: filter inputs: [vector] condition: >- .pod == "test-druid-coordinator-custom-log-config-0" && .container == "vector" - automaticLogConfigHistoricalDruid: + filteredAutomaticLogConfigHistoricalDruid: type: filter inputs: [vector] condition: >- .pod == "test-druid-historical-automatic-log-config-0" && .container == "druid" - automaticLogConfigHistoricalPrepare: + filteredAutomaticLogConfigHistoricalPrepare: type: filter inputs: [vector] condition: >- .pod == "test-druid-historical-automatic-log-config-0" && .container == "prepare" - automaticLogConfigHistoricalVector: + filteredAutomaticLogConfigHistoricalVector: type: filter inputs: [vector] condition: >- .pod == "test-druid-historical-automatic-log-config-0" && .container == "vector" - customLogConfigHistoricalDruid: + filteredCustomLogConfigHistoricalDruid: type: filter inputs: [vector] condition: >- .pod == "test-druid-historical-custom-log-config-0" && .container == "druid" - customLogConfigHistoricalPrepare: + filteredCustomLogConfigHistoricalPrepare: type: filter inputs: [vector] condition: >- .pod == "test-druid-historical-custom-log-config-0" && .container == "prepare" - customLogConfigHistoricalVector: + filteredCustomLogConfigHistoricalVector: type: filter inputs: [vector] condition: >- .pod == "test-druid-historical-custom-log-config-0" && .container == "vector" - automaticLogConfigMiddlemanagerDruid: + filteredAutomaticLogConfigMiddlemanagerDruid: type: filter inputs: [vector] condition: >- .pod == "test-druid-middlemanager-automatic-log-config-0" && .container == "druid" - automaticLogConfigMiddlemanagerPrepare: + filteredAutomaticLogConfigMiddlemanagerPrepare: type: filter inputs: [vector] condition: >- .pod == "test-druid-middlemanager-automatic-log-config-0" && .container == "prepare" - automaticLogConfigMiddlemanagerVector: + filteredAutomaticLogConfigMiddlemanagerVector: type: filter inputs: [vector] condition: >- .pod == "test-druid-middlemanager-automatic-log-config-0" && .container == "vector" - customLogConfigMiddlemanagerDruid: + filteredCustomLogConfigMiddlemanagerDruid: type: filter inputs: [vector] condition: >- .pod == "test-druid-middlemanager-custom-log-config-0" && .container == "druid" - customLogConfigMiddlemanagerPrepare: + filteredCustomLogConfigMiddlemanagerPrepare: type: filter inputs: [vector] condition: >- .pod == "test-druid-middlemanager-custom-log-config-0" && .container == "prepare" - customLogConfigMiddlemanagerVector: + filteredCustomLogConfigMiddlemanagerVector: type: filter inputs: [vector] condition: >- .pod == "test-druid-middlemanager-custom-log-config-0" && .container == "vector" - automaticLogConfigRouterDruid: + filteredAutomaticLogConfigRouterDruid: type: filter inputs: [vector] condition: >- .pod == "test-druid-router-automatic-log-config-0" && .container == "druid" - automaticLogConfigRouterPrepare: + filteredAutomaticLogConfigRouterPrepare: type: filter inputs: [vector] condition: >- .pod == "test-druid-router-automatic-log-config-0" && .container == "prepare" - automaticLogConfigRouterVector: + filteredAutomaticLogConfigRouterVector: type: filter inputs: [vector] condition: >- .pod == "test-druid-router-automatic-log-config-0" && .container == "vector" - customLogConfigRouterDruid: + filteredCustomLogConfigRouterDruid: type: filter inputs: [vector] condition: >- .pod == "test-druid-router-custom-log-config-0" && .container == "druid" - customLogConfigRouterPrepare: + filteredCustomLogConfigRouterPrepare: type: filter inputs: [vector] condition: >- .pod == "test-druid-router-custom-log-config-0" && .container == "prepare" - customLogConfigRouterVector: + filteredCustomLogConfigRouterVector: type: filter inputs: [vector] condition: >- .pod == "test-druid-router-custom-log-config-0" && .container == "vector" + filteredInvalidEvents: + type: filter + inputs: [vector] + condition: |- + .timestamp == from_unix_timestamp!(0) || + is_null(.level) || + is_null(.logger) || + is_null(.message) sinks: out: - inputs: [automaticLogConfig*, customLogConfig*] + inputs: [filtered*] {% if lookup('env', 'VECTOR_AGGREGATOR') %} type: vector address: {{ lookup('env', 'VECTOR_AGGREGATOR') }} + buffer: + # Avoid back pressure from VECTOR_AGGREGATOR. The test should + # not fail if the aggregator is not available. + when_full: drop_newest {% else %} type: blackhole {% endif %} diff --git a/tests/templates/kuttl/logging/test_log_aggregation.py b/tests/templates/kuttl/logging/test_log_aggregation.py index b8ecf18a..c11e8b90 100755 --- a/tests/templates/kuttl/logging/test_log_aggregation.py +++ b/tests/templates/kuttl/logging/test_log_aggregation.py @@ -2,7 +2,7 @@ import requests -def check_processed_events(): +def check_sent_events(): response = requests.post( 'http://druid-vector-aggregator:8686/graphql', json={ @@ -12,8 +12,8 @@ def check_processed_events(): nodes { componentId metrics { - processedEventsTotal { - processedEventsTotal + sentEventsTotal { + sentEventsTotal } } } @@ -30,12 +30,19 @@ def check_processed_events(): transforms = result['data']['transforms']['nodes'] for transform in transforms: - processedEvents = transform['metrics']['processedEventsTotal']['processedEventsTotal'] + sentEvents = transform['metrics']['sentEventsTotal'] componentId = transform['componentId'] - assert processedEvents > 0, \ - f'No events were processed in "{componentId}".' + + if componentId == 'filteredInvalidEvents': + assert sentEvents is None or \ + sentEvents['sentEventsTotal'] == 0, \ + 'Invalid log events were sent.' + else: + assert sentEvents is not None and \ + sentEvents['sentEventsTotal'] > 0, \ + f'No events were sent in "{componentId}".' if __name__ == '__main__': - check_processed_events() + check_sent_events() print('Test successful!')