[ GCP ] Fix IP convert processor in Audit Pipeline (#5597)

* add conditional for "private" in convert processor
elastic · Mar 20, 2023 · 8a6b1af · 8a6b1af
1 parent f7ec174
commit 8a6b1af
Show file tree

Hide file tree

Showing 5 changed files with 77 additions and 2 deletions.
diff --git a/packages/gcp/changelog.yml b/packages/gcp/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.17.2"
+  changes:
+    - description: Fix IP Convert processor in Audit ingest pipeline.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/5597
 - version: "2.17.1"
   changes:
     - description: Added categories and/or subcategories.

diff --git a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-audit.log b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-audit.log
@@ -15,3 +15,4 @@
 {"insertId":"15ciwwfd47gm","logName":"projects/elastic/logs/cloudaudit.googleapis.com%2Fdata_access","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"service-150691754250@container-engine-robot.iam.gserviceaccount.com","principalSubject":"serviceAccount:service-150691754250@container-engine-robot.iam.gserviceaccount.com"},"authorizationInfo":[{"granted":true,"permission":"container.clusters.get","resourceAttributes":{}}],"methodName":"google.container.v1.ClusterManager.GetCluster","policyViolationInfo":{"orgPolicyViolationInfo":{}},"request":{"@type":"type.googleapis.com/google.container.v1alpha1.GetClusterRequest","name":"projects/elastic-product/locations/us-central1-a/clusters/demo-elastic-co"},"requestMetadata":{"callerIp":"192.168.1.1","callerSuppliedUserAgent":"google-api-go-client/0.5 cluster-autoscaler,gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2022-06-01T11:15:10.836131149Z"}},"resourceLocation":{"currentLocations":["us-central1-a"]},"resourceName":"projects/elastic-product/zones/us-central1-a/clusters/demo-elastic-co","serviceName":"container.googleapis.com"},"receiveTimestamp":"2022-06-01T11:15:11.07151757Z","resource":{"labels":{"cluster_name":"demo-elastic-co","location":"us-central1-a","project_id":"elastic-product"},"type":"gke_cluster"},"severity":"INFO","timestamp":"2022-06-01T11:15:10.842495409Z","logging.googleapis.com/timestamp":"2022-06-01T11:15:10.842495409Z"}
 {"insertId":"4pyr6eegiuw1","logName":"projects/elastic/logs/cloudaudit.googleapis.com%2Fdata_access","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"xxx-compute@developer.gserviceaccount.com","serviceAccountDelegationInfo":[{}]},"authorizationInfo":[{"granted":true,"permission":"storage.objects.get","resource":"projects/_/buckets/dataflow-staging-us-central1-xxx/objects/staging/xxx.jar","resourceAttributes":{}}],"methodName":"storage.objects.get","requestMetadata":{"callerSuppliedUserAgent":"BigstoreFile BigstoreIO (cr/xxx) ","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2022-06-01T11:19:08.205760711Z"}},"resourceLocation":{"currentLocations":["us-central1"]},"resourceName":"projects/_/buckets/dataflow-staging-us-central1-xxx/objects/staging/jfxrt-xxx.jar","serviceName":"storage.googleapis.com","status":{}},"receiveTimestamp":"2022-06-01T11:19:08.699785539Z","resource":{"labels":{"bucket_name":"dataflow-staging-us-central1-150691754250","location":"us-central1","project_id":"elastic-product"},"type":"gcs_bucket"},"severity":"INFO","timestamp":"2022-06-01T11:19:08.199407722Z","logging.googleapis.com/timestamp":"2022-06-01T11:19:08.199407722Z"}
 {"insertId":"15ciwwfd47gf","logName":"projects/elastic/logs/cloudaudit.googleapis.com%2Fdata_access","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"service-150691754250@container-engine-robot.iam.gserviceaccount.com","principalSubject":"serviceAccount:service-150691754250@container-engine-robot.iam.gserviceaccount.com"},"authorizationInfo":[{"granted":true,"permission":"container.clusters.get","resourceAttributes":{}}],"methodName":"google.container.v1.ClusterManager.GetCluster","policyViolationInfo":{"orgPolicyViolationInfo":{}},"request":{"@type":"type.googleapis.com/google.container.v1alpha1.GetClusterRequest","name":"projects/elastic-product/locations/us-central1-a/clusters/demo-elastic-co","policy":"scalar-policy"},"requestMetadata":{"callerIp":"192.168.1.1","callerSuppliedUserAgent":"google-api-go-client/0.5 cluster-autoscaler,gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2022-06-01T11:15:10.836131149Z"}},"resourceLocation":{"currentLocations":["us-central1-a"]},"resourceName":"projects/elastic-product/zones/us-central1-a/clusters/demo-elastic-co","serviceName":"container.googleapis.com"},"receiveTimestamp":"2022-06-01T11:15:11.07151757Z","resource":{"labels":{"cluster_name":"demo-elastic-co","location":"us-central1-a","project_id":"elastic-product"},"type":"gke_cluster"},"severity":"INFO","timestamp":"2022-06-01T11:15:10.842495409Z","logging.googleapis.com/timestamp":"2022-06-01T11:15:10.842495409Z"}
+{"insertId":"03adfb9f-71a3-4f41-9701-29b5542f4d23","logName":"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"xxx@xxx.xxx","principalSubject":"sub","serviceAccountKeyName":"//xxx@xxx"},"authorizationInfo":[{"granted":true,"permission":"iam.serviceAccounts.list","resource":"projects/project","resourceAttributes":{}}],"methodName":"google.iam.admin.v1.ListServiceAccounts","request":{"@type":"type.googleapis.com/google.iam.admin.v1.ListServiceAccountsRequest","name":"projects/project","page_token":"cg:FFFFFF"},"requestMetadata":{"callerIp":"private","callerSuppliedUserAgent":"google-api-go-client/0.5,gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2022-02-21T13:57:39.178418578Z"}},"resourceName":"projects/project","serviceName":"iam.googleapis.com","status":{}},"receiveTimestamp":"2022-02-21T13:57:39.341344991Z","resource":{"labels":{"location":"global","method":"google.iam.admin.v1.ListServiceAccounts","project_id":"project","service":"iam.googleapis.com","version":"v1"},"type":"api"},"severity":"INFO","timestamp":"2022-02-21T13:57:39.174555198Z"}
diff --git a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-audit.log-expected.json b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-audit.log-expected.json
@@ -1799,6 +1799,75 @@
                 "name": "Other",
                 "original": "google-api-go-client/0.5 cluster-autoscaler,gzip(gfe)"
             }
+        },
+        {
+            "@timestamp": "2022-02-21T13:57:39.174Z",
+            "client": {
+                "user": {
+                    "email": "xxx@xxx.xxx",
+                    "id": "sub"
+                }
+            },
+            "cloud": {
+                "project": {
+                    "id": "project"
+                },
+                "provider": "gcp"
+            },
+            "ecs": {
+                "version": "8.6.0"
+            },
+            "event": {
+                "action": "google.iam.admin.v1.ListServiceAccounts",
+                "category": [
+                    "network",
+                    "configuration"
+                ],
+                "id": "03adfb9f-71a3-4f41-9701-29b5542f4d23",
+                "kind": "event",
+                "original": "{\"insertId\":\"03adfb9f-71a3-4f41-9701-29b5542f4d23\",\"logName\":\"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access\",\"protoPayload\":{\"@type\":\"type.googleapis.com/google.cloud.audit.AuditLog\",\"authenticationInfo\":{\"principalEmail\":\"xxx@xxx.xxx\",\"principalSubject\":\"sub\",\"serviceAccountKeyName\":\"//xxx@xxx\"},\"authorizationInfo\":[{\"granted\":true,\"permission\":\"iam.serviceAccounts.list\",\"resource\":\"projects/project\",\"resourceAttributes\":{}}],\"methodName\":\"google.iam.admin.v1.ListServiceAccounts\",\"request\":{\"@type\":\"type.googleapis.com/google.iam.admin.v1.ListServiceAccountsRequest\",\"name\":\"projects/project\",\"page_token\":\"cg:FFFFFF\"},\"requestMetadata\":{\"callerIp\":\"private\",\"callerSuppliedUserAgent\":\"google-api-go-client/0.5,gzip(gfe)\",\"destinationAttributes\":{},\"requestAttributes\":{\"auth\":{},\"time\":\"2022-02-21T13:57:39.178418578Z\"}},\"resourceName\":\"projects/project\",\"serviceName\":\"iam.googleapis.com\",\"status\":{}},\"receiveTimestamp\":\"2022-02-21T13:57:39.341344991Z\",\"resource\":{\"labels\":{\"location\":\"global\",\"method\":\"google.iam.admin.v1.ListServiceAccounts\",\"project_id\":\"project\",\"service\":\"iam.googleapis.com\",\"version\":\"v1\"},\"type\":\"api\"},\"severity\":\"INFO\",\"timestamp\":\"2022-02-21T13:57:39.174555198Z\"}",
+                "outcome": "success",
+                "provider": "data_access",
+                "type": [
+                    "access",
+                    "allowed"
+                ]
+            },
+            "gcp": {
+                "audit": {
+                    "authorization_info": [
+                        {
+                            "granted": true,
+                            "permission": "iam.serviceAccounts.list",
+                            "resource": "projects/project"
+                        }
+                    ],
+                    "request": {
+                        "@type": "type.googleapis.com/google.iam.admin.v1.ListServiceAccountsRequest",
+                        "name": "projects/project",
+                        "page_token": "cg:FFFFFF"
+                    },
+                    "resource_name": "projects/project",
+                    "type": "type.googleapis.com/google.cloud.audit.AuditLog"
+                }
+            },
+            "log": {
+                "level": "INFO",
+                "logger": "projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access"
+            },
+            "service": {
+                "name": "iam.googleapis.com"
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "user_agent": {
+                "device": {
+                    "name": "Other"
+                },
+                "name": "Other",
+                "original": "google-api-go-client/0.5,gzip(gfe)"
+            }
         }
     ]
 }
diff --git a/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml
@@ -178,7 +178,7 @@ processors:
 # https://cloud.google.com/logging/docs/reference/audit/auditlog/rest/Shared.Types/AuditLog#requestmetadata
 ##
   - convert:
-      if: ctx.json?.protoPayload?.requestMetadata?.callerIp != null && ctx.json?.protoPayload?.requestMetadata?.callerIp != "gce-internal-ip" 
+      if: ctx.json?.protoPayload?.requestMetadata?.callerIp != null && ctx.json?.protoPayload?.requestMetadata?.callerIp != "gce-internal-ip"  && ctx.json?.protoPayload?.requestMetadata?.callerIp != "private"
       type: ip
       field: json.protoPayload.requestMetadata.callerIp
       target_field: source.ip

diff --git a/packages/gcp/manifest.yml b/packages/gcp/manifest.yml
@@ -1,6 +1,6 @@
 name: gcp
 title: Google Cloud Platform
-version: "2.17.1"
+version: "2.17.2"
 release: ga
 description: Collect logs and metrics from Google Cloud Platform with Elastic Agent.
 type: integration