Add field limit check for AWS Cloudtrail flattened fields (elastic#21388

) (elastic#21432)

add 32k length check for
  - aws.cloudtrail.flattened.request_parameters
  - aws.cloudtrail.flattened.response_elements
  - aws.cloudtrail.flattened.additional_eventdata
  - aws.cloudtrail.flattened.service_event_details

Closes elastic#21382

(cherry picked from commit bfed554)

CHANGELOG.next.asciidoc

@@ -70,6 +70,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fixed `cloudfoundry.access` to have the correct `cloudfoundry.app.id` contents. {pull}17847[17847]
 - Fixing `ingress_controller.` fields to be of type keyword instead of text. {issue}17834[17834]
 - Fixed typo in log message. {pull}17897[17897]
+- Add field limit check for AWS Cloudtrail flattened fields. {pull}21388[21388] {issue}21382[21382]
 
 *Heartbeat*
 

x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml

@@ -146,37 +146,36 @@ processors:
       field: "json.errorMessage"
       target_field: "aws.cloudtrail.error_message"
       ignore_failure: true
-  - rename:
-      field: json.requestParameters
-      target_field: "aws.cloudtrail.flattened.request_parameters"
-      if: ctx?.json?.requestParameters != null
   - script:
       lang: painless
       source: |
-        if (ctx.aws.cloudtrail.flattened.request_parameters != null) {
-          ctx.aws.cloudtrail.request_parameters = ctx.aws.cloudtrail.flattened.request_parameters.toString();
+        if (ctx.aws.cloudtrail?.flattened == null) {
+            Map map = new HashMap();
+            ctx.aws.cloudtrail.put("flattened", map);
+          }
+        if (ctx.json.requestParameters != null) {
+          ctx.aws.cloudtrail.request_parameters = ctx.json.requestParameters.toString();
+          if (ctx.aws.cloudtrail.request_parameters.length() < 32766) {
+            ctx.aws.cloudtrail.flattened.put("request_parameters", ctx.json.requestParameters);
+          }
         }
-      ignore_failure: true
-  - rename:
-      field: json.responseElements
-      target_field: "aws.cloudtrail.flattened.response_elements"
-      if: ctx?.json?.responseElements != null
-  - script:
-      lang: painless
-      source: |
-        if (ctx.aws.cloudtrail.flattened.response_elements != null) {
-          ctx.aws.cloudtrail.response_elements = ctx.aws.cloudtrail.flattened.response_elements.toString();
+        if (ctx.json.responseElements != null) {
+          ctx.aws.cloudtrail.response_elements = ctx.json.responseElements.toString();
+          if (ctx.aws.cloudtrail.response_elements.length() < 32766) {
+            ctx.aws.cloudtrail.flattened.put("response_elements", ctx.json.responseElements);
+          }
         }
-      ignore_failure: true
-  - rename:
-      field: json.additionalEventData
-      target_field: "aws.cloudtrail.flattened.additional_eventdata"
-      if: ctx?.json?.additionalEventData != null
-  - script:
-      lang: painless
-      source: |
-        if (ctx.aws.cloudtrail.flattened.additional_eventdata != null) {
-          ctx.aws.cloudtrail.additional_eventdata = ctx.aws.cloudtrail.flattened.additional_eventdata.toString();
+        if (ctx.json.additionalEventData != null) {
+          ctx.aws.cloudtrail.additional_eventdata = ctx.json.additionalEventData.toString();
+          if (ctx.aws.cloudtrail.additional_eventdata.length() < 32766) {
+            ctx.aws.cloudtrail.flattened.put("additional_eventdata", ctx.json.additionalEventData);
+          }
+        }
+        if (ctx.json.serviceEventDetails != null) {
+          ctx.aws.cloudtrail.service_event_details = ctx.json.serviceEventDetails.toString();
+          if (ctx.aws.cloudtrail.service_event_details.length() < 32766) {
+            ctx.aws.cloudtrail.flattened.put("service_event_details", ctx.json.serviceEventDetails);
+          }
         }
       ignore_failure: true
   - rename:
@@ -219,17 +218,6 @@ processors:
       field: "json.recipientAccountId"
       target_field: "aws.cloudtrail.recipient_account_id"
       ignore_failure: true
-  - rename:
-      field: json.serviceEventDetails
-      target_field: "aws.cloudtrail.flattened.service_event_details"
-      if: ctx?.json?.serviceEventDetails != null
-  - script:
-      lang: painless
-      source: |
-        if (ctx.aws.cloudtrail.flattened.service_event_details != null) {
-          ctx.aws.cloudtrail.service_event_details = ctx.aws.cloudtrail.flattened.service_event_details.toString();
-        }
-      ignore_failure: true
   - rename:
       field: "json.sharedEventId"
       target_field: "aws.cloudtrail.shared_event_id"