New v4 template.

This requires a slightly different approach to testing due to the
opensearch plugin requirement.

.github/workflows/main.yml

@@ -141,31 +141,6 @@ jobs:
   test:
     needs: [build]
     runs-on: ubuntu-latest
-    services:
-      postgres:
-        image: postgres:12
-        env:
-          POSTGRES_PASSWORD: dev_password
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-      opensearch:
-        image: public.ecr.aws/opensearchproject/opensearch:2.0.1
-        env:
-          OPENSEARCH_JAVA_OPTS: "-Xms32m -Xmx512m"
-          DISABLE_INSTALL_DEMO_CONFIG: "true"
-          DISABLE_SECURITY_PLUGIN: "true"
-          DISABLE_SECURITY_DASHBOARDS_PLUGIN: "true"
-          DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI: "true"
-          discovery.type: single-node
-          cluster.routing.allocation.disk.threshold_enabled: "false"
-        options: >-
-          --health-cmd "curl -fsS http://localhost:9200/_cluster/health"
-          --health-interval 10s
-          --health-timeout 90s
-          --health-retries 10
     strategy:
       fail-fast: false
       matrix:
@@ -193,16 +168,16 @@ jobs:
           chmod +x /usr/local/bin/split_tests
 
           tests=$(split_tests --glob "${{ env.TESTS_GLOB }}" --split-total=${CI_NODE_TOTAL} --split-index=${CI_NODE_INDEX})
-          docker run --rm \
-            --network "${{ job.services.postgres.network }}" \
-            -v "$(pwd)/test/tmp/artifacts:/app/test/tmp/artifacts" \
-            -e CI=true \
+          docker compose run \
+            --rm \
             -e TESTS="${tests}" \
             -e MAXMIND_LICENSE_KEY="${MAXMIND_LICENSE_KEY}" \
-            "$DOCKER_IMAGE_CACHE_FROM" \
+            app \
             make test
         env:
-          DOCKER_IMAGE_CACHE_FROM: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}
+          COMPOSE_FILE: docker-compose.ci.yml
+          TEST_IMAGE_REPO: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          TEST_IMAGE_TAG: ${{ steps.meta.outputs.version }}
           SPLIT_TEST_VERSION: v0.3.0
           CI_NODE_TOTAL: ${{ matrix.ci_node_total }}
           CI_NODE_INDEX: ${{ matrix.ci_node_index }}

Dockerfile-opensearch

@@ -0,0 +1,2 @@
+FROM public.ecr.aws/opensearchproject/opensearch:2.11.1
+RUN /usr/share/opensearch/bin/opensearch-plugin install --batch mapper-murmur3

config/opensearch_templates_v3.json.etlua → config/opensearch_templates_v4.json.etlua

@@ -1,7 +1,7 @@
 {
-  "<%- config["opensearch"]["index_name_prefix"] %>-log-v3-template": {
+  "<%- config["opensearch"]["index_name_prefix"] %>-logs-v4-template": {
     "index_patterns": [
-      "<%- config["opensearch"]["index_name_prefix"] %>-logs-v3-*"
+      "<%- config["opensearch"]["index_name_prefix"] %>-logs-v4-*"
     ],
     "data_stream": {},
     "template": {
@@ -86,7 +86,12 @@
           },
           "request_ip": {
             "type": "keyword",
-            "normalizer": "lowercase_normalizer"
+            "normalizer": "lowercase_normalizer",
+            "fields": {
+              "hash": {
+                "type": "murmur3"
+              }
+            }
           },
           "request_ip_city": {
             "type": "keyword",
@@ -219,11 +224,21 @@
           },
           "user_id": {
             "type": "keyword",
-            "normalizer": "lowercase_normalizer"
+            "normalizer": "lowercase_normalizer",
+            "fields": {
+              "hash": {
+                "type": "murmur3"
+              }
+            }
           },
           "user_email": {
             "type": "keyword",
-            "normalizer": "lowercase_normalizer"
+            "normalizer": "lowercase_normalizer",
+            "fields": {
+              "hash": {
+                "type": "murmur3"
+              }
+            }
           },
           "user_registration_source": {
             "type": "keyword",

config/schema.cue

@@ -497,7 +497,7 @@ import "path"
       "http://opensearch:9200",
     ]
     index_name_prefix: string | *"api-umbrella"
-    template_version: uint | *3
+    template_version: uint | *4
     template: {
       index: {
         refresh_interval: string | *"5s"

docker-compose.ci.yml

@@ -0,0 +1,41 @@
+version: "3"
+services:
+  app:
+    image: "${TEST_IMAGE_REPO}:${TEST_IMAGE_TAG}"
+    volumes:
+      - ./test/tmp/artifacts:/app/test/tmp/artifacts
+    environment:
+      CI: "true"
+    depends_on:
+      - postgres
+      - opensearch
+  postgres:
+    image: postgres:12
+    environment:
+      POSTGRES_PASSWORD: dev_password
+    healthcheck:
+      test: ["CMD", "pg_isready"]
+      interval: 10s
+      timeout: 5s
+      start_period: 30s
+      start_interval: 2s
+      retries: 3
+  opensearch:
+    build:
+      context: .
+      dockerfile: Dockerfile-opensearch
+    environment:
+      OPENSEARCH_JAVA_OPTS: "-Xms32m -Xmx256m"
+      DISABLE_INSTALL_DEMO_CONFIG: "true"
+      DISABLE_SECURITY_PLUGIN: "true"
+      DISABLE_SECURITY_DASHBOARDS_PLUGIN: "true"
+      DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI: "true"
+      discovery.type: single-node
+      cluster.routing.allocation.disk.threshold_enabled: "false"
+    healthcheck:
+      test: ["CMD", "curl -fsS http://localhost:9200/_cluster/health"]
+      interval: 10s
+      timeout: 5s
+      start_period: 90s
+      start_interval: 2s
+      retries: 3

docker-compose.yml

@@ -33,7 +33,9 @@ services:
     ports:
       - "14011:5432"
   opensearch:
-    image: public.ecr.aws/opensearchproject/opensearch:2.11.1
+    build:
+      context: .
+      dockerfile: Dockerfile-opensearch
     environment:
       OPENSEARCH_JAVA_OPTS: "-Xms32m -Xmx256m"
       DISABLE_INSTALL_DEMO_CONFIG: "true"

src/api-umbrella/proxy/log_utils.lua

@@ -51,6 +51,16 @@ local function uppercase_truncate(value, max_length)
   return string.upper(truncate_string(value, max_length))
 end
 
+local function sanitize_request_header(value)
+  local cleaned_value, _, gsub_err = re_gsub(value, "\xC0\xA2", "??", "jo")
+  if gsub_err then
+    ngx.log(ngx.ERR, "regex error: ", gsub_err)
+    return value
+  end
+
+  return cleaned_value
+end
+
 local function remove_envoy_empty_header_value(value)
   if value and value == "-" then
     return nil
@@ -286,20 +296,20 @@ function _M.normalized_data(data)
     api_backend_response_flags = remove_envoy_empty_header_value(truncate(data["api_backend_response_flags"], 20)),
     gatekeeper_denied_code = lowercase_truncate(data["gatekeeper_denied_code"], 50),
     request_id = lowercase_truncate(data["request_id"], 20),
-    request_accept = truncate(data["request_accept"], 200),
-    request_accept_encoding = truncate(data["request_accept_encoding"], 200),
+    request_accept = sanitize_request_header(truncate(data["request_accept"], 200)),
+    request_accept_encoding = sanitize_request_header(truncate(data["request_accept_encoding"], 200)),
     request_basic_auth_username = truncate(data["request_basic_auth_username"], 200),
-    request_connection = truncate(data["request_connection"], 200),
-    request_content_type = truncate(data["request_content_type"], 200),
+    request_connection = sanitize_request_header(truncate(data["request_connection"], 200)),
+    request_content_type = sanitize_request_header(truncate(data["request_content_type"], 200)),
     request_ip = lowercase_truncate(data["request_ip"], 45),
     request_ip_city = truncate(data["request_ip_city"], 200),
     request_ip_country = uppercase_truncate(data["request_ip_country"], 2),
     request_ip_region = uppercase_truncate(data["request_ip_region"], 2),
     request_method = uppercase_truncate(data["request_method"], 10),
-    request_origin = truncate(data["request_origin"], 200),
-    request_referer = truncate(data["request_referer"], 200),
+    request_origin = sanitize_request_header(truncate(data["request_origin"], 200)),
+    request_referer = sanitize_request_header(truncate(data["request_referer"], 200)),
     request_size = tonumber(data["request_size"]),
-    request_host = lowercase_truncate(data["request_url_host"], 200),
+    request_host = sanitize_request_header(lowercase_truncate(data["request_url_host"], 200)),
     request_path = truncate(data["request_url_path"], 4000),
     request_url_hierarchy_level0 = truncate(data["request_url_hierarchy_level0"], 200),
     request_url_hierarchy_level1 = truncate(data["request_url_hierarchy_level1"], 200),
@@ -310,7 +320,7 @@ function _M.normalized_data(data)
     request_url_hierarchy_level6 = truncate(data["request_url_hierarchy_level6"], 200),
     request_url_query = truncate(data["request_url_query"], 4000),
     request_scheme = lowercase_truncate(data["request_url_scheme"], 10),
-    request_user_agent = truncate(data["request_user_agent"], 400),
+    request_user_agent = sanitize_request_header(truncate(data["request_user_agent"], 400)),
     request_user_agent_family = truncate(data["request_user_agent_family"], 100),
     request_user_agent_type = truncate(data["request_user_agent_type"], 100),
     response_age = tonumber(data["response_age"]),

tasks/test.thor

@@ -10,9 +10,9 @@ class Test < Thor
 
     args = []
     if ENV["CI"] == "true"
-      args += ["--ci-dir", "./test/tmp/artifacts/reports"]
+      args += ["--ci-dir", "/test/tmp/artifacts/reports"]
     end
 
-    exec "bundle", "exec", "minitest", *args, *tests
+    exec "bundle", "exec", "minitest", *(args + tests)
   end
 end

test/proxy/logging/test_basics.rb

@@ -689,19 +689,19 @@ def test_logs_to_different_data_streams
     }))
     assert_response_code(403, response)
     hit = wait_for_log(response).fetch(:hit)
-    assert_match(/\A\.ds-api-umbrella-test-logs-v3-denied-\d+\z/, hit.fetch("_index"))
+    assert_match(/\A\.ds-api-umbrella-test-logs-v#{$config["opensearch"]["template_version"]}-denied-\d+\z/, hit.fetch("_index"))
     assert_equal(403, hit.fetch("_source").fetch("response_status"))
 
     response = Typhoeus.get("http://127.0.0.1:9080/api/unknown-path-404", log_http_options)
     assert_response_code(404, response)
     hit = wait_for_log(response).fetch(:hit)
-    assert_match(/\A\.ds-api-umbrella-test-logs-v3-errored-\d+\z/, hit.fetch("_index"))
+    assert_match(/\A\.ds-api-umbrella-test-logs-v#{$config["opensearch"]["template_version"]}-errored-\d+\z/, hit.fetch("_index"))
     assert_equal(404, hit.fetch("_source").fetch("response_status"))
 
     response = Typhoeus.get("http://127.0.0.1:9080/api/hello", log_http_options)
     assert_response_code(200, response)
     hit = wait_for_log(response).fetch(:hit)
-    assert_match(/\A\.ds-api-umbrella-test-logs-v3-allowed-\d+\z/, hit.fetch("_index"))
+    assert_match(/\A\.ds-api-umbrella-test-logs-v#{$config["opensearch"]["template_version"]}-allowed-\d+\z/, hit.fetch("_index"))
     assert_equal(200, hit.fetch("_source").fetch("response_status"))
   end
 

test/proxy/logging/test_special_chars.rb

@@ -293,4 +293,18 @@ def test_slashes_and_backslashes
     assert_equal("/api/hello/extra//slash/some\\backslash/encoded%5Cbackslash/encoded%2Fslash", record["request_path"])
     assert_equal("&forward_slash=/slash&encoded_forward_slash=%2F&back_slash=\\&encoded_back_slash=%5C", record["request_url_query"])
   end
+
+  def test_invalid_quotes
+    response = Typhoeus.get("http://127.0.0.1:9080/api/hello", log_http_options.deep_merge({
+      :headers => {
+        "User-Agent" => Base64.decode64("eyJ1c2VyX2FnZW50IjogImZvbyDAp8CiIGJhciJ9"),
+        "Referer" => Base64.decode64("eyJ1c2VyX2FnZW50IjogImZvbyDAp8CiIGJhciJ9"),
+      },
+    }))
+    assert_response_code(200, response)
+
+    record = wait_for_log(response)[:hit_source]
+    assert_equal("{\"user_agent\": \"foo ?? bar\"}", record["request_referer"])
+    assert_equal("{\"user_agent\": \"foo ?? bar\"}", record["request_user_agent"])
+  end
 end