Upgrade vector

CHANGELOG.md

@@ -8,7 +8,13 @@ All notable changes to this project will be documented in this file.
 
 - Default stackableVersion to operator version ([#458]).
 
+### Changed
+
+- `vector` `0.26.0` -> `0.31.0` ([#459]).
+- `operator-rs` `0.44.0` -> `0.45.1` ([#458]).
+
 [#458]: https://github.com/stackabletech/druid-operator/pull/458
+[#459]: https://github.com/stackabletech/druid-operator/pull/459
 
 ## [23.7.0] - 2023-07-14
 

tests/kuttl-test.yaml.jinja2

@@ -9,3 +9,19 @@ testDirs:
 startKIND: false
 suppress: ["events"]
 parallel: 2
+
+# The timeout (in seconds) is used when namespaces are created or
+# deleted, and, if not overridden, in TestSteps, TestAsserts, and
+# Commands. If not set, the timeout is 30 seconds by default.
+#
+# The deletion of a namespace can take a while until all resources are
+# gracefully shut down. If the timeout is reached in the meantime, even
+# a successful test case is considered a failure.
+#
+# For instance, the termination grace period of the Vector aggregator in
+# the logging tests is set to 60 seconds. If there are logs entries
+# which could not be forwarded yet to the external aggregator defined in
+# the VECTOR_AGGREGATOR environment variable, then the test aggregator
+# uses this period of time by trying to forward the events. In this
+# case, deleting a namespace with several Pods takes about 90 seconds.
+timeout: 120

tests/templates/kuttl/logging/01-install-druid-vector-aggregator.yaml

@@ -5,7 +5,7 @@ commands:
   - script: >-
       helm install druid-vector-aggregator vector
       --namespace $NAMESPACE
-      --version 0.19.2
+      --version 0.23.0
       --repo https://helm.vector.dev
       --values druid-vector-aggregator-values.yaml
 ---

tests/templates/kuttl/logging/druid-vector-aggregator-values.yaml.j2

@@ -20,192 +20,204 @@ customConfig:
       type: vector
       version: "2"
   transforms:
-    automaticLogConfigBrokerDruid:
+    filteredAutomaticLogConfigBrokerDruid:
       type: filter
       inputs: [vector]
       condition: >-
         .pod == "test-druid-broker-automatic-log-config-0" &&
         .container == "druid"
-    automaticLogConfigBrokerPrepare:
+    filteredAutomaticLogConfigBrokerPrepare:
       type: filter
       inputs: [vector]
       condition: >-
         .pod == "test-druid-broker-automatic-log-config-0" &&
         .container == "prepare"
-    automaticLogConfigBrokerVector:
+    filteredAutomaticLogConfigBrokerVector:
       type: filter
       inputs: [vector]
       condition: >-
         .pod == "test-druid-broker-automatic-log-config-0" &&
         .container == "vector"
-    customLogConfigBrokerDruid:
+    filteredCustomLogConfigBrokerDruid:
       type: filter
       inputs: [vector]
       condition: >-
         .pod == "test-druid-broker-custom-log-config-0" &&
         .container == "druid"
-    customLogConfigBrokerPrepare:
+    filteredCustomLogConfigBrokerPrepare:
       type: filter
       inputs: [vector]
       condition: >-
         .pod == "test-druid-broker-custom-log-config-0" &&
         .container == "prepare"
-    customLogConfigBrokerVector:
+    filteredCustomLogConfigBrokerVector:
       type: filter
       inputs: [vector]
       condition: >-
         .pod == "test-druid-broker-custom-log-config-0" &&
         .container == "vector"
-    automaticLogConfigCoordinatorDruid:
+    filteredAutomaticLogConfigCoordinatorDruid:
       type: filter
       inputs: [vector]
       condition: >-
         .pod == "test-druid-coordinator-automatic-log-config-0" &&
         .container == "druid"
-    automaticLogConfigCoordinatorPrepare:
+    filteredAutomaticLogConfigCoordinatorPrepare:
       type: filter
       inputs: [vector]
       condition: >-
         .pod == "test-druid-coordinator-automatic-log-config-0" &&
         .container == "prepare"
-    automaticLogConfigCoordinatorVector:
+    filteredAutomaticLogConfigCoordinatorVector:
       type: filter
       inputs: [vector]
       condition: >-
         .pod == "test-druid-coordinator-automatic-log-config-0" &&
         .container == "vector"
-    customLogConfigCoordinatorDruid:
+    filteredCustomLogConfigCoordinatorDruid:
       type: filter
       inputs: [vector]
       condition: >-
         .pod == "test-druid-coordinator-custom-log-config-0" &&
         .container == "druid"
-    customLogConfigCoordinatorPrepare:
+    filteredCustomLogConfigCoordinatorPrepare:
       type: filter
       inputs: [vector]
       condition: >-
         .pod == "test-druid-coordinator-custom-log-config-0" &&
         .container == "prepare"
-    customLogConfigCoordinatorVector:
+    filteredCustomLogConfigCoordinatorVector:
       type: filter
       inputs: [vector]
       condition: >-
         .pod == "test-druid-coordinator-custom-log-config-0" &&
         .container == "vector"
-    automaticLogConfigHistoricalDruid:
+    filteredAutomaticLogConfigHistoricalDruid:
       type: filter
       inputs: [vector]
       condition: >-
         .pod == "test-druid-historical-automatic-log-config-0" &&
         .container == "druid"
-    automaticLogConfigHistoricalPrepare:
+    filteredAutomaticLogConfigHistoricalPrepare:
       type: filter
       inputs: [vector]
       condition: >-
         .pod == "test-druid-historical-automatic-log-config-0" &&
         .container == "prepare"
-    automaticLogConfigHistoricalVector:
+    filteredAutomaticLogConfigHistoricalVector:
       type: filter
       inputs: [vector]
       condition: >-
         .pod == "test-druid-historical-automatic-log-config-0" &&
         .container == "vector"
-    customLogConfigHistoricalDruid:
+    filteredCustomLogConfigHistoricalDruid:
       type: filter
       inputs: [vector]
       condition: >-
         .pod == "test-druid-historical-custom-log-config-0" &&
         .container == "druid"
-    customLogConfigHistoricalPrepare:
+    filteredCustomLogConfigHistoricalPrepare:
       type: filter
       inputs: [vector]
       condition: >-
         .pod == "test-druid-historical-custom-log-config-0" &&
         .container == "prepare"
-    customLogConfigHistoricalVector:
+    filteredCustomLogConfigHistoricalVector:
       type: filter
       inputs: [vector]
       condition: >-
         .pod == "test-druid-historical-custom-log-config-0" &&
         .container == "vector"
-    automaticLogConfigMiddlemanagerDruid:
+    filteredAutomaticLogConfigMiddlemanagerDruid:
       type: filter
       inputs: [vector]
       condition: >-
         .pod == "test-druid-middlemanager-automatic-log-config-0" &&
         .container == "druid"
-    automaticLogConfigMiddlemanagerPrepare:
+    filteredAutomaticLogConfigMiddlemanagerPrepare:
       type: filter
       inputs: [vector]
       condition: >-
         .pod == "test-druid-middlemanager-automatic-log-config-0" &&
         .container == "prepare"
-    automaticLogConfigMiddlemanagerVector:
+    filteredAutomaticLogConfigMiddlemanagerVector:
       type: filter
       inputs: [vector]
       condition: >-
         .pod == "test-druid-middlemanager-automatic-log-config-0" &&
         .container == "vector"
-    customLogConfigMiddlemanagerDruid:
+    filteredCustomLogConfigMiddlemanagerDruid:
       type: filter
       inputs: [vector]
       condition: >-
         .pod == "test-druid-middlemanager-custom-log-config-0" &&
         .container == "druid"
-    customLogConfigMiddlemanagerPrepare:
+    filteredCustomLogConfigMiddlemanagerPrepare:
       type: filter
       inputs: [vector]
       condition: >-
         .pod == "test-druid-middlemanager-custom-log-config-0" &&
         .container == "prepare"
-    customLogConfigMiddlemanagerVector:
+    filteredCustomLogConfigMiddlemanagerVector:
       type: filter
       inputs: [vector]
       condition: >-
         .pod == "test-druid-middlemanager-custom-log-config-0" &&
         .container == "vector"
-    automaticLogConfigRouterDruid:
+    filteredAutomaticLogConfigRouterDruid:
       type: filter
       inputs: [vector]
       condition: >-
         .pod == "test-druid-router-automatic-log-config-0" &&
         .container == "druid"
-    automaticLogConfigRouterPrepare:
+    filteredAutomaticLogConfigRouterPrepare:
       type: filter
       inputs: [vector]
       condition: >-
         .pod == "test-druid-router-automatic-log-config-0" &&
         .container == "prepare"
-    automaticLogConfigRouterVector:
+    filteredAutomaticLogConfigRouterVector:
       type: filter
       inputs: [vector]
       condition: >-
         .pod == "test-druid-router-automatic-log-config-0" &&
         .container == "vector"
-    customLogConfigRouterDruid:
+    filteredCustomLogConfigRouterDruid:
       type: filter
       inputs: [vector]
       condition: >-
         .pod == "test-druid-router-custom-log-config-0" &&
         .container == "druid"
-    customLogConfigRouterPrepare:
+    filteredCustomLogConfigRouterPrepare:
       type: filter
       inputs: [vector]
       condition: >-
         .pod == "test-druid-router-custom-log-config-0" &&
         .container == "prepare"
-    customLogConfigRouterVector:
+    filteredCustomLogConfigRouterVector:
       type: filter
       inputs: [vector]
       condition: >-
         .pod == "test-druid-router-custom-log-config-0" &&
         .container == "vector"
+    filteredInvalidEvents:
+      type: filter
+      inputs: [vector]
+      condition: |-
+        .timestamp == from_unix_timestamp!(0) ||
+        is_null(.level) ||
+        is_null(.logger) ||
+        is_null(.message)
   sinks:
     out:
-      inputs: [automaticLogConfig*, customLogConfig*]
+      inputs: [filtered*]
 {% if lookup('env', 'VECTOR_AGGREGATOR') %}
       type: vector
       address: {{ lookup('env', 'VECTOR_AGGREGATOR') }}
+      buffer:
+        # Avoid back pressure from VECTOR_AGGREGATOR. The test should
+        # not fail if the aggregator is not available.
+        when_full: drop_newest
 {% else %}
       type: blackhole
 {% endif %}

tests/templates/kuttl/logging/test_log_aggregation.py

@@ -2,7 +2,7 @@
 import requests
 
 
-def check_processed_events():
+def check_sent_events():
     response = requests.post(
         'http://druid-vector-aggregator:8686/graphql',
         json={
@@ -12,8 +12,8 @@ def check_processed_events():
                         nodes {
                             componentId
                             metrics {
-                                processedEventsTotal {
-                                    processedEventsTotal
+                                sentEventsTotal {
+                                    sentEventsTotal
                                 }
                             }
                         }
@@ -30,12 +30,19 @@ def check_processed_events():
 
     transforms = result['data']['transforms']['nodes']
     for transform in transforms:
-        processedEvents = transform['metrics']['processedEventsTotal']['processedEventsTotal']
+        sentEvents = transform['metrics']['sentEventsTotal']
         componentId = transform['componentId']
-        assert processedEvents > 0, \
-            f'No events were processed in "{componentId}".'
+
+        if componentId == 'filteredInvalidEvents':
+            assert sentEvents is None or \
+                sentEvents['sentEventsTotal'] == 0, \
+                'Invalid log events were sent.'
+        else:
+            assert sentEvents is not None and \
+                sentEvents['sentEventsTotal'] > 0, \
+                f'No events were sent in "{componentId}".'
 
 
 if __name__ == '__main__':
-    check_processed_events()
+    check_sent_events()
     print('Test successful!')