[GCP] Convert some dashboards to lens (#7014)

* elastic-package format * changelog and manifest * changelog entry * changing array to nested fields * change event.category/type to array for audit/firewall/vpcflow
elastic · Jul 19, 2023 · d6b0497 · d6b0497
1 parent 8904a8f
commit d6b0497
Show file tree

Hide file tree

Showing 26 changed files with 8,936 additions and 7,535 deletions.
diff --git a/packages/gcp/changelog.yml b/packages/gcp/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.23.0"
+  changes:
+    - description: Convert security dashboards to lens.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/7014
 - version: "2.22.1"
   changes:
     - description: Change ownership in manifest.

diff --git a/packages/gcp/data_stream/audit/fields/fields.yml b/packages/gcp/data_stream/audit/fields/fields.yml
@@ -18,7 +18,7 @@
           type: keyword
           description: "String representation of identity of requesting party. Populated for both first and third party identities. Only present for APIs that support third-party identities."
     - name: authorization_info
-      type: array
+      type: nested
       description: |
         Authorization information for the operation.
       fields:
@@ -97,7 +97,7 @@
       type: group
       fields:
         - name: current_locations
-          type: array
+          type: keyword
           description: |
             Current locations of the resource.
     - name: service_name

diff --git a/packages/gcp/data_stream/audit/manifest.yml b/packages/gcp/data_stream/audit/manifest.yml
@@ -40,7 +40,7 @@ streams:
         description: The maximum number of unprocessed messages (unacknowledged but not yet expired). If the value is negative, then there will be no limit on the number of unprocessed messages. Default is 1000.
         multi: false
         required: false
-        show_user: false 
+        show_user: false
       - name: alternative_host
         type: text
         title: Alternative host

diff --git a/packages/gcp/data_stream/audit/sample_event.json b/packages/gcp/data_stream/audit/sample_event.json
@@ -1,11 +1,11 @@
 {
     "@timestamp": "2019-12-19T00:44:25.051Z",
     "agent": {
-        "ephemeral_id": "f4dde373-2ff7-464b-afdb-da94763f219b",
-        "id": "5d3eee86-91a9-4afa-af92-c6b79bd866c0",
+        "ephemeral_id": "7780bdcf-661a-4891-83bd-dd5233873f9d",
+        "id": "5872ddcf-0f11-4ff9-84ce-30e042fe8327",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "8.6.0"
+        "version": "8.7.1"
     },
     "client": {
         "user": {
@@ -27,9 +27,9 @@
         "version": "8.8.0"
     },
     "elastic_agent": {
-        "id": "5d3eee86-91a9-4afa-af92-c6b79bd866c0",
-        "snapshot": true,
-        "version": "8.6.0"
+        "id": "5872ddcf-0f11-4ff9-84ce-30e042fe8327",
+        "snapshot": false,
+        "version": "8.7.1"
     },
     "event": {
         "action": "beta.compute.instances.aggregatedList",
@@ -38,10 +38,10 @@
             "network",
             "configuration"
         ],
-        "created": "2023-01-13T14:59:20.459Z",
+        "created": "2023-07-19T18:53:36.388Z",
         "dataset": "gcp.audit",
         "id": "yonau2dg2zi",
-        "ingested": "2023-01-13T14:59:21Z",
+        "ingested": "2023-07-19T18:53:40Z",
         "kind": "event",
         "outcome": "success",
         "provider": "data_access",

diff --git a/packages/gcp/data_stream/firewall/_dev/test/pipeline/test-firewall.log-expected.json b/packages/gcp/data_stream/firewall/_dev/test/pipeline/test-firewall.log-expected.json
diff --git a/packages/gcp/data_stream/firewall/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/firewall/elasticsearch/ingest_pipeline/default.yml
@@ -26,7 +26,7 @@ processors:
   - set:
       field: event.kind
       value: event
-  - set:
+  - append:
       field: event.category
       value: network
   - set:
@@ -48,16 +48,16 @@ processors:
       copy_from: json.insertId
       ignore_empty_value: true
       ignore_failure: true
-  - rename:
+  - lowercase:
       field: json.jsonPayload.disposition
-      target_field: event.type
       if: ctx?.json?.jsonPayload?.disposition != null
-  - set:
+  - append:
       field: event.type
-      value: connection
-      if: ctx?.event?.type != null
-  - lowercase:
+      value: '{{json.jsonPayload.disposition}}'
+      if: ctx?.json?.jsonPayload?.disposition != null
+  - append:
       field: event.type
+      value: connection
   - set:
       field: network.direction
       value: inbound

diff --git a/packages/gcp/data_stream/firewall/fields/fields.yml b/packages/gcp/data_stream/firewall/fields/fields.yml
@@ -31,7 +31,7 @@
           description: |
             List of all the target tags that the firewall rule applies to.
         - name: ip_port_info
-          type: array
+          type: nested
           description: |
             List of ip protocols and applicable port ranges for rules.
         - name: source_service_account

diff --git a/packages/gcp/data_stream/firewall/sample_event.json b/packages/gcp/data_stream/firewall/sample_event.json
@@ -1,11 +1,11 @@
 {
     "@timestamp": "2019-10-30T13:52:42.191Z",
     "agent": {
-        "ephemeral_id": "f4dde373-2ff7-464b-afdb-da94763f219b",
-        "id": "5d3eee86-91a9-4afa-af92-c6b79bd866c0",
+        "ephemeral_id": "cf009128-e43c-42e4-9158-9b088bd6f3f5",
+        "id": "5872ddcf-0f11-4ff9-84ce-30e042fe8327",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "8.6.0"
+        "version": "8.7.1"
     },
     "cloud": {
         "availability_zone": "us-east1-b",
@@ -30,20 +30,25 @@
         "version": "8.8.0"
     },
     "elastic_agent": {
-        "id": "5d3eee86-91a9-4afa-af92-c6b79bd866c0",
-        "snapshot": true,
-        "version": "8.6.0"
+        "id": "5872ddcf-0f11-4ff9-84ce-30e042fe8327",
+        "snapshot": false,
+        "version": "8.7.1"
     },
     "event": {
         "action": "firewall-rule",
         "agent_id_status": "verified",
-        "category": "network",
-        "created": "2023-01-13T15:01:23.807Z",
+        "category": [
+            "network"
+        ],
+        "created": "2023-07-19T18:55:10.718Z",
         "dataset": "gcp.firewall",
         "id": "1f21ciqfpfssuo",
-        "ingested": "2023-01-13T15:01:24Z",
+        "ingested": "2023-07-19T18:55:14Z",
         "kind": "event",
-        "type": "connection"
+        "type": [
+            "allowed",
+            "connection"
+        ]
     },
     "gcp": {
         "destination": {

diff --git a/packages/gcp/data_stream/vpcflow/_dev/test/pipeline/test-vpcflow.log-expected.json b/packages/gcp/data_stream/vpcflow/_dev/test/pipeline/test-vpcflow.log-expected.json
diff --git a/packages/gcp/data_stream/vpcflow/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/vpcflow/elasticsearch/ingest_pipeline/default.yml
@@ -26,10 +26,10 @@ processors:
   - set:
       field: event.kind
       value: event
-  - set:
+  - append:
       field: event.category
       value: network
-  - set:
+  - append:
       field: event.type
       value: connection
   - set: 

diff --git a/packages/gcp/data_stream/vpcflow/sample_event.json b/packages/gcp/data_stream/vpcflow/sample_event.json
@@ -1,45 +1,66 @@
 {
     "@timestamp": "2019-06-14T03:50:10.845Z",
     "agent": {
-        "ephemeral_id": "f4dde373-2ff7-464b-afdb-da94763f219b",
-        "id": "5d3eee86-91a9-4afa-af92-c6b79bd866c0",
+        "ephemeral_id": "a47f1e8b-f681-4e3b-87cd-6b2d54144577",
+        "id": "5872ddcf-0f11-4ff9-84ce-30e042fe8327",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "8.6.0"
+        "version": "8.7.1"
     },
     "cloud": {
-        "provider": "gcp"
+        "availability_zone": "us-east1-b",
+        "project": {
+            "id": "my-sample-project"
+        },
+        "provider": "gcp",
+        "region": "us-east1"
     },
     "data_stream": {
         "dataset": "gcp.vpcflow",
         "namespace": "ep",
         "type": "logs"
     },
     "destination": {
-        "address": "10.87.40.76",
+        "address": "67.43.156.13",
+        "as": {
+            "number": 35908
+        },
         "domain": "kibana",
-        "ip": "10.87.40.76",
-        "port": 5601
+        "geo": {
+            "continent_name": "Asia",
+            "country_iso_code": "BT",
+            "country_name": "Bhutan",
+            "location": {
+                "lat": 27.5,
+                "lon": 90.5
+            }
+        },
+        "ip": "67.43.156.13",
+        "port": 33548
     },
     "ecs": {
         "version": "8.8.0"
     },
     "elastic_agent": {
-        "id": "5d3eee86-91a9-4afa-af92-c6b79bd866c0",
-        "snapshot": true,
-        "version": "8.6.0"
+        "id": "5872ddcf-0f11-4ff9-84ce-30e042fe8327",
+        "snapshot": false,
+        "version": "8.7.1"
     },
     "event": {
         "agent_id_status": "verified",
-        "category": "network",
-        "created": "2023-01-13T15:03:19.118Z",
+        "category": [
+            "network"
+        ],
+        "created": "2023-07-19T18:56:47.758Z",
         "dataset": "gcp.vpcflow",
-        "end": "2019-06-14T03:40:37.048196137Z",
-        "id": "ut8lbrffooxzf",
-        "ingested": "2023-01-13T15:03:20Z",
+        "end": "2019-06-14T03:49:56.393651211Z",
+        "id": "ut8lbrffooxz4",
+        "ingested": "2023-07-19T18:56:51Z",
         "kind": "event",
-        "start": "2019-06-14T03:40:36.895188084Z",
-        "type": "connection"
+        "start": "2019-06-14T03:40:05.147252064Z",
+        "type": [
+            "connection"
+        ]
     },
     "gcp": {
         "destination": {
@@ -54,10 +75,22 @@
                 "vpc_name": "default"
             }
         },
+        "source": {
+            "instance": {
+                "project_id": "my-sample-project",
+                "region": "us-east1",
+                "zone": "us-east1-b"
+            },
+            "vpc": {
+                "project_id": "my-sample-project",
+                "subnetwork_name": "default",
+                "vpc_name": "default"
+            }
+        },
         "vpcflow": {
-            "reporter": "DEST",
+            "reporter": "SRC",
             "rtt": {
-                "ms": 36
+                "ms": 50
             }
         }
     },
@@ -68,33 +101,28 @@
         "logger": "projects/my-sample-project/logs/compute.googleapis.com%2Fvpc_flows"
     },
     "network": {
-        "bytes": 1464,
-        "community_id": "1:++9/JiESSUdwTGGcxwXk4RA0lY8=",
-        "direction": "inbound",
+        "bytes": 159704,
+        "community_id": "1:+S3/6PF+UXU7wlJD68HIrz0Mo6c=",
+        "direction": "internal",
         "iana_number": "6",
-        "packets": 7,
+        "name": "default",
+        "packets": 241,
         "transport": "tcp",
         "type": "ipv4"
     },
     "related": {
         "ip": [
-            "192.168.2.117",
-            "10.87.40.76"
+            "10.139.99.242",
+            "67.43.156.13"
         ]
     },
     "source": {
-        "address": "192.168.2.117",
-        "as": {
-            "number": 15169
-        },
-        "bytes": 1464,
-        "geo": {
-            "continent_name": "America",
-            "country_name": "usa"
-        },
-        "ip": "192.168.2.117",
-        "packets": 7,
-        "port": 50646
+        "address": "10.139.99.242",
+        "bytes": 159704,
+        "domain": "elasticsearch",
+        "ip": "10.139.99.242",
+        "packets": 241,
+        "port": 9200
     },
     "tags": [
         "forwarded",