Fix outputting to "errored" datastream.

NREL · Feb 15, 2024 · 488f347 · 488f347
1 parent c56e9e7
commit 488f347
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 3 deletions.
diff --git a/tasks/deps/fluent-bit b/tasks/deps/fluent-bit
@@ -36,7 +36,7 @@ cmake \
   -DFLB_FILTER_LOG_TO_METRICS="Off" \
   -DFLB_FILTER_LUA="Off" \
   -DFLB_FILTER_LUA_USE_MPACK="Off" \
-  -DFLB_FILTER_MODIFY="Off" \
+  -DFLB_FILTER_MODIFY="On" \
   -DFLB_FILTER_MULTILINE="Off" \
   -DFLB_FILTER_NEST="Off" \
   -DFLB_FILTER_NIGHTFALL="Off" \
@@ -48,7 +48,7 @@ cmake \
   -DFLB_FILTER_TENSORFLOW="Off" \
   -DFLB_FILTER_THROTTLE="Off" \
   -DFLB_FILTER_THROTTLE_SIZE="Off" \
-  -DFLB_FILTER_TYPE_CONVERTER="Off" \
+  -DFLB_FILTER_TYPE_CONVERTER="On" \
   -DFLB_FILTER_WASM="Off" \
   -DFLB_IN_CALYPTIA_FLEET="Off" \
   -DFLB_IN_COLLECTD="Off" \

diff --git a/templates/etc/fluent-bit/fluent-bit.yaml.etlua b/templates/etc/fluent-bit/fluent-bit.yaml.etlua
@@ -46,10 +46,20 @@ pipeline:
     - name: rewrite_tag
       match: analytics.allowed
       rule: "$gatekeeper_denied_code ^. analytics.denied false"
+      emitter_storage.type: filesystem
+
+    - name: type_converter
+      match: analytics.allowed
+      uint_key: response_status response_status_str string
 
     - name: rewrite_tag
       match: analytics.allowed
-      rule: "$response_status ^[4-9] analytics.errored false"
+      rule: "$response_status_str ^[4-9] analytics.errored false"
+      emitter_storage.type: filesystem
+
+    - name: modify
+      match: "analytics.*"
+      remove: response_status_str
 
   outputs:
     # Print what we would log to stdout for extra redundancy.

diff --git a/test/proxy/logging/test_basics.rb b/test/proxy/logging/test_basics.rb
@@ -681,6 +681,30 @@ def test_logs_server_header
     assert_equal(unique_test_id, record.fetch("response_server"))
   end
 
+  def test_logs_to_different_data_streams
+    response = Typhoeus.get("http://127.0.0.1:9080/api/hello", log_http_options.deep_merge({
+      :headers => {
+        "X-Api-Key" => "INVALID_KEY",
+      },
+    }))
+    assert_response_code(403, response)
+    hit = wait_for_log(response).fetch(:hit)
+    assert_match(/\A\.ds-api-umbrella-test-logs-v3-denied-\d+\z/, hit.fetch("_index"))
+    assert_equal(403, hit.fetch("_source").fetch("response_status"))
+
+    response = Typhoeus.get("http://127.0.0.1:9080/api/unknown-path-404", log_http_options)
+    assert_response_code(404, response)
+    hit = wait_for_log(response).fetch(:hit)
+    assert_match(/\A\.ds-api-umbrella-test-logs-v3-errored-\d+\z/, hit.fetch("_index"))
+    assert_equal(404, hit.fetch("_source").fetch("response_status"))
+
+    response = Typhoeus.get("http://127.0.0.1:9080/api/hello", log_http_options)
+    assert_response_code(200, response)
+    hit = wait_for_log(response).fetch(:hit)
+    assert_match(/\A\.ds-api-umbrella-test-logs-v3-allowed-\d+\z/, hit.fetch("_index"))
+    assert_equal(200, hit.fetch("_source").fetch("response_status"))
+  end
+
   private
 
   def refute_log(response)