cel: add secret_state for encrypted credentials in CEL programs

packages/cel/_dev/deploy/docker/files/config.yml

@@ -23,6 +23,17 @@ rules:
       - status_code: 200
         body: |
           {"message": "success"}
+  - path: /testsecret/api
+    methods: [GET]
+    request_headers:
+      Accept:
+        - "application/json"
+      X-Api-Key:
+        - "test-secret-key"
+    responses:
+      - status_code: 200
+        body: |
+          {"message": "success"}
   - path: /testoauth/token
     methods: [POST]
     query_params:

packages/cel/_dev/test/policy/test-secret-state.expected

@@ -0,0 +1,49 @@
+inputs:
+    - data_stream:
+        namespace: ep
+      meta:
+        package:
+            name: cel
+      name: test-secret-state-cel
+      streams:
+        - data_stream:
+            dataset: cel.cel
+          interval: 1m
+          program: |-
+            request("GET", state.url).with({
+            "Header": {"X-API-Key": [state.secret.api_key]}
+            }).do_request().as(resp, {
+            "events": [resp.Body.decode_json()],
+            "secret": state.secret,
+            })
+          publisher_pipeline.disable_host: true
+          redact.delete: false
+          regexp: null
+          resource.headers: null
+          resource.tracer:
+            enabled: false
+            filename: ../../logs/cel/http-request-trace-*.ndjson
+            maxbackups: 5
+          resource.url: https://server.example.com:8089/api
+          secret_state: ${SECRET_0}
+          tags:
+            - forwarded
+          xsd: null
+      type: cel
+      use_output: default
+output_permissions:
+    default:
+        _elastic_agent_checks:
+            cluster:
+                - monitor
+        _elastic_agent_monitoring:
+            indices: []
+        uuid-for-permissions-on-related-indices:
+            indices:
+                - names:
+                    - logs-*-*
+                  privileges:
+                    - auto_configure
+                    - create_doc
+secret_references:
+    - {}

packages/cel/_dev/test/policy/test-secret-state.yml

@@ -0,0 +1,14 @@
+vars:
+  url: http://example.com:9001
+  program: |-
+    request("GET", state.url).with({
+    "Header": {"X-API-Key": [state.secret.api_key]}
+    }).do_request().as(resp, {
+    "events": [resp.Body.decode_json()],
+    "secret": state.secret,
+    })
+  secret_state: |-
+    api_key: my-secret-api-key
+  interval: 5m
+  preserve_original_event: true
+  preserve_duplicate_custom_fields: true

packages/cel/_dev/test/system/test-secret-state-config.yml

@@ -0,0 +1,20 @@
+vars:
+  redact_fields: [foo]
+  resource_url: http://{{Hostname}}:{{Port}}/testsecret/api
+  enable_request_tracer: true
+  secret_state: |-
+    api_key: test-secret-key
+  program: |
+    request("GET", state.url).with({
+      "Header": {
+        "Accept": ["application/json"],
+        "X-Api-Key": [state.secret.api_key],
+      }
+    }).do_request().as(resp, resp.StatusCode == 200 ?
+      resp.Body.as(body, {
+        "events": [body.decode_json()],
+        "secret": state.secret,
+      })
+    :
+      {"events": []}
+    )

packages/cel/agent/input/input.yml.hbs

@@ -12,6 +12,10 @@ program: {{escape_string program}}
 state:
   {{state}}
 {{/if}}
+{{#if secret_state}}
+secret_state:
+  {{secret_state}}
+{{/if}}
 redact.delete: {{delete_redacted_fields}}
 {{#if redact_fields}}
 redact.fields:

packages/cel/changelog.yml

@@ -1,3 +1,8 @@
+- version: "1.20.0"
+  changes:
+    - description: Add secret state configuration for encrypted credentials in CEL programs.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/18834
 - version: "1.19.0"
   changes:
     - description: Add options for including global HTTP request headers.

packages/cel/manifest.yml

@@ -3,12 +3,12 @@ name: cel
 title: Custom API using Common Expression Language
 description: Collect custom events from an API with Elastic agent
 type: input
-version: "1.19.0"
+version: "1.20.0"
 categories:
   - custom
 conditions:
   kibana:
-    version: "^8.19.0 || ^9.1.0"
+    version: "^8.19.17 || ^9.3.6 || ^9.4.1"
   elastic:
     subscription: "basic"
 policy_templates:
@@ -81,10 +81,23 @@ policy_templates:
         title: Initial CEL evaluation state
         description: |
           State is the initial state to be provided to the program. If it has a cursor field, that field will be overwritten by any stored cursor, but will be available if no stored cursor exists.
+          The state must not contain a `secret` key; use the Secret State field instead.
           More information can be found in the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-cel.html#input-state-cel).
         show_user: true
         multi: false
         required: false
+      - name: secret_state
+        type: textarea
+        title: Secret CEL evaluation state
+        description: |
+          Secret state holds key-value pairs that are stored encrypted by Fleet and made available to the CEL program at `state.secret`.
+          Use this for API keys, tokens, and other credentials that should not be visible in the integration configuration.
+          Values are automatically redacted in debug logs.
+          More information can be found in the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-cel.html#secret-state-cel).
+        show_user: true
+        multi: false
+        required: false
+        secret: true
       - name: allowed_environment
         type: text
         title: Allowed environment variables