gcp: add keep_json option for audit, dns, firewall and vpcflow datast…

…reams

packages/gcp/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.31.0"
+  changes:
+    - description: Allow users to retain otherwised discarded fields.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/8299
 - version: "2.30.1"
   changes:
     - description: Fix mappings of group fields

packages/gcp/data_stream/audit/agent/stream/gcp-pubsub.yml.hbs

@@ -27,7 +27,11 @@ tags:
 {{#contains "forwarded" tags}}
 publisher_pipeline.disable_host: true
 {{/contains}}
-{{#if processors}}
 processors:
+- add_fields:
+    target: '_conf'
+    fields:
+        keep_json: {{keep_json}}
+{{#if processors}}
 {{processors}}
 {{/if}}

packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml

@@ -113,20 +113,20 @@ processors:
 ##
 # email address of authenticated user (redacted) or service account
 # principalEmail -> client.user.email
-  - rename:
-      field: json.protoPayload.authenticationInfo.principalEmail
-      target_field: client.user.email
-      ignore_missing: true
+  - set:
+      field: client.user.email
+      copy_from: json.protoPayload.authenticationInfo.principalEmail
+      ignore_failure: true
 # identity of requesting first or third party
 # principalSubject -> client.user.id
-  - rename:
-      field: json.protoPayload.authenticationInfo.principalSubject
-      target_field: client.user.id
-      ignore_missing: true
-  - rename:
-      field: json.protoPayload.authenticationInfo.authoritySelector
-      target_field: gcp.audit.authentication_info.authority_selector
-      ignore_missing: true
+  - set:
+      field: client.user.id
+      copy_from: json.protoPayload.authenticationInfo.principalSubject
+      ignore_failure: true
+  - set:
+      field: gcp.audit.authentication_info.authority_selector
+      copy_from: json.protoPayload.authenticationInfo.authoritySelector
+      ignore_failure: true
 
   - rename:
       field: gcp.audit.authentication_info.principal_email
@@ -151,10 +151,10 @@ processors:
 # .protoPayload.authorizationInfo
 # https://cloud.google.com/logging/docs/reference/audit/auditlog/rest/Shared.Types/AuditLog#authorizationinfo
 ##
-  - rename:
-      field: json.protoPayload.authorizationInfo
-      target_field: gcp.audit.authorization_info
-      ignore_missing: true
+  - set:
+      field: gcp.audit.authorization_info
+      copy_from: json.protoPayload.authorizationInfo
+      ignore_failure: true
   - foreach:
       field: gcp.audit.authorization_info
       ignore_missing: true
@@ -184,10 +184,10 @@ processors:
       field: json.protoPayload.requestMetadata.callerIp
       target_field: source.ip
       ignore_missing: true
-  - rename:
-      field: json.protoPayload.requestMetadata.callerSuppliedUserAgent
-      target_field: user_agent.original
-      ignore_missing: true
+  - set:
+      field: user_agent.original
+      copy_from: json.protoPayload.requestMetadata.callerSuppliedUserAgent
+      ignore_failure: true
   - user_agent:
       field: user_agent.original
       ignore_missing: true
@@ -227,10 +227,10 @@ processors:
         }
 
 # TODO remove duplicate protoPayload.methodName
-  - rename:
-      field: json.protoPayload.methodName
-      target_field: event.action
-      ignore_missing: true
+  - set:
+      field: event.action
+      copy_from: json.protoPayload.methodName
+      ignore_failure: true
   - convert:
       field: json.protoPayload.numResponseItems
       target_field: gcp.audit.num_response_items
@@ -244,10 +244,10 @@ processors:
       field: gcp.audit.request.policy
       ignore_missing: true
       if: ctx.json?.protoPayload?.request?.policy != null && !(ctx.json?.protoPayload?.request.policy instanceof Map)
-  - rename:
-      field: json.protoPayload.request.policy
-      target_field: gcp.audit.request.policy_value
-      ignore_missing: true
+  - set:
+      field: gcp.audit.request.policy_value
+      copy_from: json.protoPayload.request.policy
+      ignore_failure: true
       if: ctx.json?.protoPayload?.request?.policy != null && !(ctx.json?.protoPayload?.request.policy instanceof Map)
   - set:
       field: gcp.audit.response
@@ -257,24 +257,24 @@ processors:
       field: gcp.audit.response.status
       ignore_missing: true
       if: ctx.json?.protoPayload?.response?.status != null && !(ctx.json?.protoPayload?.response.status instanceof Map)
-  - rename:
-      field: json.protoPayload.response.status
-      target_field: gcp.audit.response.status_value
-      ignore_missing: true
+  - set:
+      field: gcp.audit.response.status_value
+      copy_from: json.protoPayload.response.status
+      ignore_failure: true
       if: ctx.json?.protoPayload?.response?.status != null && !(ctx.json?.protoPayload?.response.status instanceof Map)
-  - rename:
-      field: json.protoPayload.resourceName
-      target_field: gcp.audit.resource_name
-      ignore_missing: true
+  - set:
+      field: gcp.audit.resource_name
+      copy_from: json.protoPayload.resourceName
+      ignore_failure: true
       if: ctx.orchestrator?.type != 'kubernetes'
-  - rename:
-      field: json.protoPayload.resourceLocation.currentLocations
-      target_field: gcp.audit.resource_location.current_locations
-      ignore_missing: true
-  - rename:
-      field: json.protoPayload.serviceName
-      target_field: gcp.audit.service_name
-      ignore_missing: true
+  - set:
+      field: gcp.audit.resource_location.current_locations
+      copy_from: json.protoPayload.resourceLocation.currentLocations
+      ignore_failure: true
+  - set:
+      field: gcp.audit.service_name
+      copy_from: json.protoPayload.serviceName
+      ignore_failure: true
   - rename:
       field: gcp.audit.service_name
       target_field: service.name
@@ -291,10 +291,10 @@ processors:
       target_field: gcp.audit.status.code
       type: long
       ignore_missing: true
-  - rename:
-      field: json.protoPayload.status.message
-      target_field: gcp.audit.status.message
-      ignore_missing: true
+  - set:
+      field: gcp.audit.status.message
+      copy_from: json.protoPayload.status.message
+      ignore_failure: true
   - set:
       field: event.outcome
       value: success
@@ -363,8 +363,13 @@ processors:
 ##
 # clean-up
 ##
+  - rename:
+      field: json
+      target_field: gcp.audit.flattened
+      if: ctx.json != null && ctx._conf?.keep_json == true
   - remove:
       field:
+        - _conf
         - _temp
         - json
       ignore_missing: true

packages/gcp/data_stream/audit/fields/fields.yml

@@ -113,3 +113,6 @@
         - name: message
           type: keyword
           description: "A developer-facing error message, which should be in English. Any user-facing  error message should be localized and sent in the google.rpc.Status.details  field, or localized by the client."
+    - name: flattened
+      type: flattened
+      description: Contains the full audit document as sent by GCP.

packages/gcp/data_stream/audit/manifest.yml

@@ -65,6 +65,14 @@ streams:
         type: bool
         multi: false
         default: false
+      - name: keep_json
+        required: true
+        show_user: false
+        title: Keep the JSON document as `gcp.audit.flattened`
+        description: Keeps a copy of the original document as a JSON field for processing in `@custom` pipelines.
+        type: bool
+        multi: false
+        default: false
       - name: processors
         type: yaml
         title: Processors

packages/gcp/data_stream/audit/sample_event.json

@@ -1,8 +1,8 @@
 {
     "@timestamp": "2019-12-19T00:44:25.051Z",
     "agent": {
-        "ephemeral_id": "7780bdcf-661a-4891-83bd-dd5233873f9d",
-        "id": "5872ddcf-0f11-4ff9-84ce-30e042fe8327",
+        "ephemeral_id": "a22278bb-5e1f-4ab7-b468-277c8c0b80a9",
+        "id": "c6b95057-2f5d-4b8f-b4b5-37cbdb995dec",
         "name": "docker-fleet-agent",
         "type": "filebeat",
         "version": "8.7.1"
@@ -27,7 +27,7 @@
         "version": "8.8.0"
     },
     "elastic_agent": {
-        "id": "5872ddcf-0f11-4ff9-84ce-30e042fe8327",
+        "id": "c6b95057-2f5d-4b8f-b4b5-37cbdb995dec",
         "snapshot": false,
         "version": "8.7.1"
     },
@@ -38,10 +38,10 @@
             "network",
             "configuration"
         ],
-        "created": "2023-07-19T18:53:36.388Z",
+        "created": "2023-10-25T04:18:46.637Z",
         "dataset": "gcp.audit",
         "id": "yonau2dg2zi",
-        "ingested": "2023-07-19T18:53:40Z",
+        "ingested": "2023-10-25T04:18:47Z",
         "kind": "event",
         "outcome": "success",
         "provider": "data_access",

packages/gcp/data_stream/dns/agent/stream/gcp-pubsub.yml.hbs

@@ -27,7 +27,11 @@ tags:
 {{#contains "forwarded" tags}}
 publisher_pipeline.disable_host: true
 {{/contains}}
-{{#if processors}}
 processors:
+- add_fields:
+    target: '_conf'
+    fields:
+        keep_json: {{keep_json}}
+{{#if processors}}
 {{processors}}
 {{/if}}