From 0f575d0fb4a97e883a46ab64879eb36c6d511586 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Wed, 6 May 2026 13:17:41 +0930 Subject: [PATCH] cel: add secret_state for encrypted credentials in CEL programs Add a secret_state variable (type: textarea, secret: true) that lets users store API keys and credentials encrypted by Fleet and reference them in CEL programs as state.secret.. Wire the variable into the agent input template and add a system test that validates the secret value reaches the CEL program via a header check against the mock server. Requires the matching beats change to accept string values for secret_state, since Fleet resolves secrets to strings. Bumps kibana.version to the versions that will include the Fleet fix. --- .../cel/_dev/deploy/docker/files/config.yml | 11 +++++ .../test/policy/test-secret-state.expected | 49 +++++++++++++++++++ .../_dev/test/policy/test-secret-state.yml | 14 ++++++ .../test/system/test-secret-state-config.yml | 20 ++++++++ packages/cel/agent/input/input.yml.hbs | 4 ++ packages/cel/changelog.yml | 5 ++ packages/cel/manifest.yml | 17 ++++++- 7 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 packages/cel/_dev/test/policy/test-secret-state.expected create mode 100644 packages/cel/_dev/test/policy/test-secret-state.yml create mode 100644 packages/cel/_dev/test/system/test-secret-state-config.yml diff --git a/packages/cel/_dev/deploy/docker/files/config.yml b/packages/cel/_dev/deploy/docker/files/config.yml index e3fe15e9264..9f6b11cae55 100644 --- a/packages/cel/_dev/deploy/docker/files/config.yml +++ b/packages/cel/_dev/deploy/docker/files/config.yml @@ -23,6 +23,17 @@ rules: - status_code: 200 body: | {"message": "success"} + - path: /testsecret/api + methods: [GET] + request_headers: + Accept: + - "application/json" + X-Api-Key: + - "test-secret-key" + responses: + - status_code: 200 + body: | + {"message": "success"} - path: /testoauth/token methods: [POST] query_params: diff --git a/packages/cel/_dev/test/policy/test-secret-state.expected b/packages/cel/_dev/test/policy/test-secret-state.expected new file mode 100644 index 00000000000..756eda8b0e7 --- /dev/null +++ b/packages/cel/_dev/test/policy/test-secret-state.expected @@ -0,0 +1,49 @@ +inputs: + - data_stream: + namespace: ep + meta: + package: + name: cel + name: test-secret-state-cel + streams: + - data_stream: + dataset: cel.cel + interval: 1m + program: |- + request("GET", state.url).with({ + "Header": {"X-API-Key": [state.secret.api_key]} + }).do_request().as(resp, { + "events": [resp.Body.decode_json()], + "secret": state.secret, + }) + publisher_pipeline.disable_host: true + redact.delete: false + regexp: null + resource.headers: null + resource.tracer: + enabled: false + filename: ../../logs/cel/http-request-trace-*.ndjson + maxbackups: 5 + resource.url: https://server.example.com:8089/api + secret_state: ${SECRET_0} + tags: + - forwarded + xsd: null + type: cel + use_output: default +output_permissions: + default: + _elastic_agent_checks: + cluster: + - monitor + _elastic_agent_monitoring: + indices: [] + uuid-for-permissions-on-related-indices: + indices: + - names: + - logs-*-* + privileges: + - auto_configure + - create_doc +secret_references: + - {} diff --git a/packages/cel/_dev/test/policy/test-secret-state.yml b/packages/cel/_dev/test/policy/test-secret-state.yml new file mode 100644 index 00000000000..dda71fae09e --- /dev/null +++ b/packages/cel/_dev/test/policy/test-secret-state.yml @@ -0,0 +1,14 @@ +vars: + url: http://example.com:9001 + program: |- + request("GET", state.url).with({ + "Header": {"X-API-Key": [state.secret.api_key]} + }).do_request().as(resp, { + "events": [resp.Body.decode_json()], + "secret": state.secret, + }) + secret_state: |- + api_key: my-secret-api-key + interval: 5m + preserve_original_event: true + preserve_duplicate_custom_fields: true diff --git a/packages/cel/_dev/test/system/test-secret-state-config.yml b/packages/cel/_dev/test/system/test-secret-state-config.yml new file mode 100644 index 00000000000..6e4f179729c --- /dev/null +++ b/packages/cel/_dev/test/system/test-secret-state-config.yml @@ -0,0 +1,20 @@ +vars: + redact_fields: [foo] + resource_url: http://{{Hostname}}:{{Port}}/testsecret/api + enable_request_tracer: true + secret_state: |- + api_key: test-secret-key + program: | + request("GET", state.url).with({ + "Header": { + "Accept": ["application/json"], + "X-Api-Key": [state.secret.api_key], + } + }).do_request().as(resp, resp.StatusCode == 200 ? + resp.Body.as(body, { + "events": [body.decode_json()], + "secret": state.secret, + }) + : + {"events": []} + ) diff --git a/packages/cel/agent/input/input.yml.hbs b/packages/cel/agent/input/input.yml.hbs index 6f568dbda89..8ccb198648e 100644 --- a/packages/cel/agent/input/input.yml.hbs +++ b/packages/cel/agent/input/input.yml.hbs @@ -12,6 +12,10 @@ program: {{escape_string program}} state: {{state}} {{/if}} +{{#if secret_state}} +secret_state: + {{secret_state}} +{{/if}} redact.delete: {{delete_redacted_fields}} {{#if redact_fields}} redact.fields: diff --git a/packages/cel/changelog.yml b/packages/cel/changelog.yml index 0e602fc6dbf..f704100bfb5 100644 --- a/packages/cel/changelog.yml +++ b/packages/cel/changelog.yml @@ -1,3 +1,8 @@ +- version: "1.20.0" + changes: + - description: Add secret state configuration for encrypted credentials in CEL programs. + type: enhancement + link: https://github.com/elastic/integrations/pull/18834 - version: "1.19.0" changes: - description: Add options for including global HTTP request headers. diff --git a/packages/cel/manifest.yml b/packages/cel/manifest.yml index 5ae52ba3900..4897cf48fed 100644 --- a/packages/cel/manifest.yml +++ b/packages/cel/manifest.yml @@ -3,12 +3,12 @@ name: cel title: Custom API using Common Expression Language description: Collect custom events from an API with Elastic agent type: input -version: "1.19.0" +version: "1.20.0" categories: - custom conditions: kibana: - version: "^8.19.0 || ^9.1.0" + version: "^8.19.17 || ^9.3.6 || ^9.4.1" elastic: subscription: "basic" policy_templates: @@ -81,10 +81,23 @@ policy_templates: title: Initial CEL evaluation state description: | State is the initial state to be provided to the program. If it has a cursor field, that field will be overwritten by any stored cursor, but will be available if no stored cursor exists. + The state must not contain a `secret` key; use the Secret State field instead. More information can be found in the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-cel.html#input-state-cel). show_user: true multi: false required: false + - name: secret_state + type: textarea + title: Secret CEL evaluation state + description: | + Secret state holds key-value pairs that are stored encrypted by Fleet and made available to the CEL program at `state.secret`. + Use this for API keys, tokens, and other credentials that should not be visible in the integration configuration. + Values are automatically redacted in debug logs. + More information can be found in the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-cel.html#secret-state-cel). + show_user: true + multi: false + required: false + secret: true - name: allowed_environment type: text title: Allowed environment variables