diff --git a/packages/azure/changelog.yml b/packages/azure/changelog.yml index c2cdcbebcc5..3078f891ffa 100644 --- a/packages/azure/changelog.yml +++ b/packages/azure/changelog.yml @@ -1,3 +1,8 @@ +- version: "1.29.0" + changes: + - description: Parse responseBody and requestBody json in activitylogs. + type: enhancement + link: https://github.com/elastic/integrations/pull/15690 - version: "1.28.7" changes: - description: Interim fix to support non-standard log events. diff --git a/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log b/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log index d5c8c1930cd..7e69b255cb5 100644 --- a/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log +++ b/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log @@ -1 +1,3 @@ -{"callerIpAddress":"81.2.69.144","category":"Action","correlationId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","durationMs":0,"identity":{"authorization":{"action":"Microsoft.EventHub/namespaces/authorizationRules/listKeys/action","evidence":{"principalId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","principalType":"ServicePrincipal","role":"Azure EventGrid Service BuiltIn Role","roleAssignmentId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","roleAssignmentScope":"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53","roleDefinitionId":"8a4de8b5-095c-47d0-a96f-a75130c61d53"},"scope":"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53/resourceGroups/sa-hem/providers/Microsoft.EventHub/namespaces/azurelsevents/authorizationRules/RootManageSharedAccessKey"},"claims":{"aio":"8a4de8b5-095c-47d0-a96f-a75130c61d53","appid":"8a4de8b5-095c-47d0-a96f-a75130c61d53","appidacr":"2","aud":"https://management.core.windows.net/","exp":"1571904826","http://schemas.microsoft.com/identity/claims/identityprovider":"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/","http://schemas.microsoft.com/identity/claims/objectidentifier":"8a4de8b5-095c-47d0-a96f-a75130c61d53","http://schemas.microsoft.com/identity/claims/tenantid":"8a4de8b5-095c-47d0-a96f-a75130c61d53","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier":"8a4de8b5-095c-47d0-a96f-a75130c61d53","iat":"1571875726","iss":"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/","nbf":"1571875726","uti":"8a4de8b5-095c-47d0-a96f-a75130c61d53","ver":"1.0"}},"level":"Information","location":"global","operationName":"MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION","resourceId":"/SUBSCRIPTIONS/8a4de8b5-095c-47d0-a96f-a75130c61d53/RESOURCEGROUPS/SA-HEMA/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY","resultSignature":"Started.","resultType":"Start","time":"2019-10-24T00:13:46.3554259Z"} \ No newline at end of file +{"callerIpAddress":"81.2.69.144","category":"Action","correlationId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","durationMs":0,"identity":{"authorization":{"action":"Microsoft.EventHub/namespaces/authorizationRules/listKeys/action","evidence":{"principalId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","principalType":"ServicePrincipal","role":"Azure EventGrid Service BuiltIn Role","roleAssignmentId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","roleAssignmentScope":"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53","roleDefinitionId":"8a4de8b5-095c-47d0-a96f-a75130c61d53"},"scope":"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53/resourceGroups/sa-hem/providers/Microsoft.EventHub/namespaces/azurelsevents/authorizationRules/RootManageSharedAccessKey"},"claims":{"aio":"8a4de8b5-095c-47d0-a96f-a75130c61d53","appid":"8a4de8b5-095c-47d0-a96f-a75130c61d53","appidacr":"2","aud":"https://management.core.windows.net/","exp":"1571904826","http://schemas.microsoft.com/identity/claims/identityprovider":"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/","http://schemas.microsoft.com/identity/claims/objectidentifier":"8a4de8b5-095c-47d0-a96f-a75130c61d53","http://schemas.microsoft.com/identity/claims/tenantid":"8a4de8b5-095c-47d0-a96f-a75130c61d53","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier":"8a4de8b5-095c-47d0-a96f-a75130c61d53","iat":"1571875726","iss":"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/","nbf":"1571875726","uti":"8a4de8b5-095c-47d0-a96f-a75130c61d53","ver":"1.0"}},"level":"Information","location":"global","operationName":"MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION","resourceId":"/SUBSCRIPTIONS/8a4de8b5-095c-47d0-a96f-a75130c61d53/RESOURCEGROUPS/SA-HEMA/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY","resultSignature":"Started.","resultType":"Start","time":"2019-10-24T00:13:46.3554259Z"} +{"category":"ResourceHealth","correlationId":"1c867fe2-050c-4a74-bb1c-a83b15246fdd","level":"Information","operationName":"Microsoft.Resourcehealth/healthevent/Updated/action","properties":{"responseBody": "{\"sku\":{\"name\":\"Standard_LRS\",\"tier\":\"Standard\"},\"kind\":\"StorageV2\",\"id\":\"/subscriptions/abc-123-your-sub-id/resourceGroups/my-resource-group/providers/Microsoft.Storage/storageAccounts/mystorageacct123\",\"location\":\"eastus\"}", "requestBody": "{\"id\":\"/subscriptions/abc-123-your-sub-id/resourceGroups/my-resource-group/providers/Microsoft.Storage/storageAccounts/mystorageacct123\",\"kind\":\"StorageV2\",\"location\":\"eastus\",\"name\":\"mystorageacct123\",\"properties\":{\"creationTime\":\"2025-01-15T14:20:00.1234567Z\",\"primaryEndpoints\":{\"blob\":\"https://mystorageacct123.blob.core.windows.net/\",\"file\":\"https://mystorageacct123.file.core.windows.net/\"},\"provisioningState\":\"Succeeded\",\"publicNetworkAccess\":\"Enabled\"},\"sku\":{\"name\":\"Standard_GRS\",\"tier\":\"Standard\"},\"type\":\"Microsoft.Storage/storageAccounts\"}", "eventCategory":"ResourceHealth","eventProperties":{"cause":"PlatformInitiated"}},"resourceId":"/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration","resultType":"Updated","time":"2025-10-17T11:50:07.22Z"} +{"category":"ResourceHealth","correlationId":"1c867fe2-050c-4a74-bb1c-a83b15246fdd","level":"Information","operationName":"Microsoft.Resourcehealth/healthevent/Updated/action","properties":{"responseBody": {"id":"\/subscriptions\/abc-123-your-sub-id\/resourceGroups\/my-resource-group\/providers\/Microsoft.Storage\/storageAccounts\/mystorageacct123","kind":"StorageV2","location":"eastus","sku":{"name":"Standard_LRS","tier":"Standard"}}, "requestBody": {"id":"\/subscriptions\/abc-123-your-sub-id\/resourceGroups\/my-resource-group\/providers\/Microsoft.Storage\/storageAccounts\/mystorageacct123","kind":"StorageV2","location":"eastus","name":"mystorageacct123","properties":{"creationTime":"2025-01-15T14:20:00.1234567Z","primaryEndpoints":{"blob":"https:\/\/mystorageacct123.blob.core.windows.net\/","file":"https:\/\/mystorageacct123.file.core.windows.net\/"},"provisioningState":"Succeeded","publicNetworkAccess":"Enabled"},"sku":{"name":"Standard_GRS","tier":"Standard"},"type":"Microsoft.Storage\/storageAccounts"}, "eventCategory":"ResourceHealth","eventProperties":{"cause":"PlatformInitiated"}},"resourceId":"/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration","resultType":"Updated","time":"2025-10-17T11:50:07.22Z"} \ No newline at end of file diff --git a/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json b/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json index 685772b74f4..9ac8aa57e3a 100644 --- a/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json +++ b/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json @@ -110,6 +110,152 @@ "tags": [ "preserve_original_event" ] + }, + { + "@timestamp": "2025-10-17T11:50:07.220Z", + "azure": { + "activitylogs": { + "category": "ResourceHealth", + "event_category": "ResourceHealth", + "operation_name": "Microsoft.Resourcehealth/healthevent/Updated/action", + "properties": { + "eventProperties": { + "cause": "PlatformInitiated" + }, + "requestBody": { + "id": "/subscriptions/abc-123-your-sub-id/resourceGroups/my-resource-group/providers/Microsoft.Storage/storageAccounts/mystorageacct123", + "kind": "StorageV2", + "location": "eastus", + "name": "mystorageacct123", + "properties": { + "creationTime": "2025-01-15T14:20:00.1234567Z", + "primaryEndpoints": { + "blob": "https://mystorageacct123.blob.core.windows.net/", + "file": "https://mystorageacct123.file.core.windows.net/" + }, + "provisioningState": "Succeeded", + "publicNetworkAccess": "Enabled" + }, + "sku": { + "name": "Standard_GRS", + "tier": "Standard" + }, + "type": "Microsoft.Storage/storageAccounts" + }, + "responseBody": { + "id": "/subscriptions/abc-123-your-sub-id/resourceGroups/my-resource-group/providers/Microsoft.Storage/storageAccounts/mystorageacct123", + "kind": "StorageV2", + "location": "eastus", + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + } + } + }, + "result_type": "Updated" + }, + "correlation_id": "1c867fe2-050c-4a74-bb1c-a83b15246fdd", + "resource": { + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration", + "provider": "Microsoft.domainRegistration" + }, + "subscription_id": "00000000-0000-0000-0000-000000000000" + }, + "cloud": { + "provider": "azure" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "Microsoft.Resourcehealth/healthevent/Updated/action", + "kind": "event", + "original": "{\"category\":\"ResourceHealth\",\"correlationId\":\"1c867fe2-050c-4a74-bb1c-a83b15246fdd\",\"level\":\"Information\",\"operationName\":\"Microsoft.Resourcehealth/healthevent/Updated/action\",\"properties\":{\"responseBody\": \"{\\\"sku\\\":{\\\"name\\\":\\\"Standard_LRS\\\",\\\"tier\\\":\\\"Standard\\\"},\\\"kind\\\":\\\"StorageV2\\\",\\\"id\\\":\\\"/subscriptions/abc-123-your-sub-id/resourceGroups/my-resource-group/providers/Microsoft.Storage/storageAccounts/mystorageacct123\\\",\\\"location\\\":\\\"eastus\\\"}\", \"requestBody\": \"{\\\"id\\\":\\\"/subscriptions/abc-123-your-sub-id/resourceGroups/my-resource-group/providers/Microsoft.Storage/storageAccounts/mystorageacct123\\\",\\\"kind\\\":\\\"StorageV2\\\",\\\"location\\\":\\\"eastus\\\",\\\"name\\\":\\\"mystorageacct123\\\",\\\"properties\\\":{\\\"creationTime\\\":\\\"2025-01-15T14:20:00.1234567Z\\\",\\\"primaryEndpoints\\\":{\\\"blob\\\":\\\"https://mystorageacct123.blob.core.windows.net/\\\",\\\"file\\\":\\\"https://mystorageacct123.file.core.windows.net/\\\"},\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"publicNetworkAccess\\\":\\\"Enabled\\\"},\\\"sku\\\":{\\\"name\\\":\\\"Standard_GRS\\\",\\\"tier\\\":\\\"Standard\\\"},\\\"type\\\":\\\"Microsoft.Storage/storageAccounts\\\"}\", \"eventCategory\":\"ResourceHealth\",\"eventProperties\":{\"cause\":\"PlatformInitiated\"}},\"resourceId\":\"/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration\",\"resultType\":\"Updated\",\"time\":\"2025-10-17T11:50:07.22Z\"}" + }, + "log": { + "level": "Information" + }, + "related": { + "entity": [ + "/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration" + ] + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2025-10-17T11:50:07.220Z", + "azure": { + "activitylogs": { + "category": "ResourceHealth", + "event_category": "ResourceHealth", + "operation_name": "Microsoft.Resourcehealth/healthevent/Updated/action", + "properties": { + "eventProperties": { + "cause": "PlatformInitiated" + }, + "requestBody": { + "id": "/subscriptions/abc-123-your-sub-id/resourceGroups/my-resource-group/providers/Microsoft.Storage/storageAccounts/mystorageacct123", + "kind": "StorageV2", + "location": "eastus", + "name": "mystorageacct123", + "properties": { + "creationTime": "2025-01-15T14:20:00.1234567Z", + "primaryEndpoints": { + "blob": "https://mystorageacct123.blob.core.windows.net/", + "file": "https://mystorageacct123.file.core.windows.net/" + }, + "provisioningState": "Succeeded", + "publicNetworkAccess": "Enabled" + }, + "sku": { + "name": "Standard_GRS", + "tier": "Standard" + }, + "type": "Microsoft.Storage/storageAccounts" + }, + "responseBody": { + "id": "/subscriptions/abc-123-your-sub-id/resourceGroups/my-resource-group/providers/Microsoft.Storage/storageAccounts/mystorageacct123", + "kind": "StorageV2", + "location": "eastus", + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + } + } + }, + "result_type": "Updated" + }, + "correlation_id": "1c867fe2-050c-4a74-bb1c-a83b15246fdd", + "resource": { + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration", + "provider": "Microsoft.domainRegistration" + }, + "subscription_id": "00000000-0000-0000-0000-000000000000" + }, + "cloud": { + "provider": "azure" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "Microsoft.Resourcehealth/healthevent/Updated/action", + "kind": "event", + "original": "{\"category\":\"ResourceHealth\",\"correlationId\":\"1c867fe2-050c-4a74-bb1c-a83b15246fdd\",\"level\":\"Information\",\"operationName\":\"Microsoft.Resourcehealth/healthevent/Updated/action\",\"properties\":{\"responseBody\": {\"id\":\"\\/subscriptions\\/abc-123-your-sub-id\\/resourceGroups\\/my-resource-group\\/providers\\/Microsoft.Storage\\/storageAccounts\\/mystorageacct123\",\"kind\":\"StorageV2\",\"location\":\"eastus\",\"sku\":{\"name\":\"Standard_LRS\",\"tier\":\"Standard\"}}, \"requestBody\": {\"id\":\"\\/subscriptions\\/abc-123-your-sub-id\\/resourceGroups\\/my-resource-group\\/providers\\/Microsoft.Storage\\/storageAccounts\\/mystorageacct123\",\"kind\":\"StorageV2\",\"location\":\"eastus\",\"name\":\"mystorageacct123\",\"properties\":{\"creationTime\":\"2025-01-15T14:20:00.1234567Z\",\"primaryEndpoints\":{\"blob\":\"https:\\/\\/mystorageacct123.blob.core.windows.net\\/\",\"file\":\"https:\\/\\/mystorageacct123.file.core.windows.net\\/\"},\"provisioningState\":\"Succeeded\",\"publicNetworkAccess\":\"Enabled\"},\"sku\":{\"name\":\"Standard_GRS\",\"tier\":\"Standard\"},\"type\":\"Microsoft.Storage\\/storageAccounts\"}, \"eventCategory\":\"ResourceHealth\",\"eventProperties\":{\"cause\":\"PlatformInitiated\"}},\"resourceId\":\"/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration\",\"resultType\":\"Updated\",\"time\":\"2025-10-17T11:50:07.22Z\"}" + }, + "log": { + "level": "Information" + }, + "related": { + "entity": [ + "/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration" + ] + }, + "tags": [ + "preserve_original_event" + ] } ] -} \ No newline at end of file +} diff --git a/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml index 85b1c26dc8b..4db8a8d8230 100644 --- a/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml @@ -96,6 +96,14 @@ processors: field: azure.activitylogs.properties if: "ctx.azure?.activitylogs?.properties instanceof String" ignore_failure: true + - json: + field: azure.activitylogs.properties.responseBody + if: "ctx.azure?.activitylogs?.properties?.responseBody instanceof String" + ignore_failure: true + - json: + field: azure.activitylogs.properties.requestBody + if: "ctx.azure?.activitylogs?.properties?.requestBody instanceof String" + ignore_failure: true - script: lang: painless source: >- diff --git a/packages/azure/data_stream/application_gateway/_dev/test/pipeline/test-application-gateway-raw.log-expected.json b/packages/azure/data_stream/application_gateway/_dev/test/pipeline/test-application-gateway-raw.log-expected.json index 38eb4bdbf94..56550e5aeb3 100644 --- a/packages/azure/data_stream/application_gateway/_dev/test/pipeline/test-application-gateway-raw.log-expected.json +++ b/packages/azure/data_stream/application_gateway/_dev/test/pipeline/test-application-gateway-raw.log-expected.json @@ -452,4 +452,4 @@ } } ] -} \ No newline at end of file +} diff --git a/packages/azure/manifest.yml b/packages/azure/manifest.yml index 892f9aea996..73345378ce8 100644 --- a/packages/azure/manifest.yml +++ b/packages/azure/manifest.yml @@ -1,6 +1,6 @@ name: azure title: Azure Logs -version: "1.28.7" +version: "1.29.0" description: This Elastic integration collects logs from Azure type: integration icons: