elastic · kcreddy · May 14, 2024 · Apr 23, 2024 · Apr 23, 2024 · Apr 23, 2024
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -95,6 +95,7 @@
 /packages/azure_metrics/data_stream/database_account @elastic/obs-ds-hosted-services
 /packages/azure_metrics/data_stream/monitor @elastic/obs-ds-hosted-services
 /packages/azure_metrics/data_stream/storage_account @elastic/obs-ds-hosted-services
+/packages/azure_network_watcher_vnet @elastic/security-service-integrations
 /packages/barracuda @elastic/security-service-integrations
 /packages/barracuda_cloudgen_firewall @elastic/security-service-integrations
 /packages/beaconing @elastic/ml-ui @elastic/sec-applied-ml

@@ -0,0 +1,4 @@
+dependencies:
+  ecs:
+    reference: "git@v8.11.0"
+    import_mappings: true
@@ -0,0 +1,69 @@
+# Azure Network Watcher VNet
+
+[VNet](https://learn.microsoft.com/en-us/azure/virtual-network/virtual-networks-overview) flow logs in Azure Network Watcher track IP traffic in virtual networks, sending data to Azure Storage for analysis. Unlike NSG flow logs, VNet flow logs offer enhanced monitoring capabilities. They are crucial for understanding network activity, identifying connections, and monitoring open ports. Flow logs serve as the primary source for optimizing resources, ensuring compliance, and detecting intrusions in cloud environments, catering to both startups and enterprises.
+
+## Data streams
+
+This integration supports ingestion of logs from Azure Network Watcher VNet, via [Azure Blob Storage](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html) input.
+
+**Log** is used to retrieve VNet Flow data. See more details in the documentation [here](https://learn.microsoft.com/en-us/azure/network-watcher/vnet-flow-logs-overview).
+
+## Requirements
+
+Elastic Agent must be installed. For more information, refer to the link [here](https://www.elastic.co/guide/en/fleet/current/elastic-agent-installation.html).
+
+### Installing and managing an Elastic Agent:
+
+You have a few options for installing and managing an Elastic Agent:
+
+### Install a Fleet-managed Elastic Agent (recommended):
+
+With this approach, you install Elastic Agent and use Fleet in Kibana to define, configure, and manage your agents in a central location. We recommend using Fleet management because it makes the management and upgrade of your agents considerably easier.
+
+### Install Elastic Agent in standalone mode (advanced users):
+
+With this approach, you install Elastic Agent and manually configure the agent locally on the system where it’s installed. You are responsible for managing and upgrading the agents. This approach is reserved for advanced users only.
+
+### Install Elastic Agent in a containerized environment:
+
+You can run Elastic Agent inside a container, either with Fleet Server or standalone. Docker images for all versions of Elastic Agent are available from the Elastic Docker registry, and we provide deployment manifests for running on Kubernetes.
+
+There are some minimum requirements for running Elastic Agent and for more information, refer to the link [here](https://www.elastic.co/guide/en/fleet/current/elastic-agent-installation.html#_minimum_requirements).
+
+The minimum **Kibana version** required is **8.12.0**.
+
+## Setup
+
+### To collect data from Azure Network Watcher VNet follow the below steps:
+
+1. In the [Azure portal](https://portal.azure.com/), go to your **storage account**.
+2. Under **Security + networking**, Click On **Access keys**. Your account access keys appear, as well as the complete connection string for each key.
+3. Click On **Show** keys to show your **access keys** and **connection strings** and to enable buttons to copy the values.
+4. Under key1, find the Key value. Click On the Copy button to copy the **account key**. Same way you can copy the **storage account name** shown above keys.
+5. Go to **Containers** under **Data storage** in your storage account to copy the **container name**.
+
+**Note**:  Enable virtual network flow logs using the steps provided in [reference](https://learn.microsoft.com/en-us/azure/network-watcher/vnet-flow-logs-portal).
+
+### Enabling the integration in Elastic:
+
+1. In Kibana go to Management > Integrations.
+2. In "Search for integrations" search bar, type Azure Network Watcher VNet.
+3. Click on the "Azure Network Watcher VNet" integration from the search results.
+4. Click on the "Add Azure Network Watcher VNet" button to add the integration.
+5. While adding the integration, to collect logs via Azure Blob Storage, keep **Collect VNet logs via Azure Blob Storage** toggle on and then configure following parameters:
+   - account name
+   - containers
+   - service account key/service account uri
+6. Save the integration.
+
+## Logs Reference
+
+### Log
+
+This is the `Log` dataset.
+
+#### Example
+
+{{event "log"}}
+
+{{fields "log"}}
@@ -0,0 +1,21 @@
+version: '2.3'
+services:
+  azure-network-watcher-vnet-log:
+    image: mcr.microsoft.com/azure-storage/azurite
+    command: azurite-blob --blobHost 0.0.0.0 --blobPort 10000
+    ports:
+      - "10000/tcp"
+  azure-blob-storage:
+    image: docker.elastic.co/observability/stream:v0.10.0
+    volumes:
+      - ./sample_logs:/sample_logs:ro
+    command:
+      - log
+      - --retry=30
+      - --addr=elastic-package-service-azure-network-watcher-vnet-log-1
+      - -p=azureblobstorage
+      - --azure-blob-storage-port=10000
+      - --azure-blob-storage-container=azure-container1
+      - /sample_logs/testdata.log
+    depends_on:
+      - azure-network-watcher-vnet-log
@@ -0,0 +1 @@
+{"records":[{"time":"2022-09-14T09:00:52.5625085Z","flowLogVersion":4,"flowLogGUID":"abcdef01-2345-6789-0abc-def012345678","macAddress":"00224871C205","category":"FlowLogFlowEvent","flowLogResourceID":"/SUBSCRIPTIONS/00000000-0000-0000-0000-000000000000/RESOURCEGROUPS/NETWORKWATCHERRG/PROVIDERS/MICROSOFT.NETWORK/NETWORKWATCHERS/NETWORKWATCHER_EASTUS2EUAP/FLOWLOGS/VNETFLOWLOG","targetResourceID":"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myResourceGroup/providers/Microsoft.Network/virtualNetworks/myVNet","operationName":"FlowLogFlowEvent","flowRecords":{"flows":[{"aclID":"00000000-1234-abcd-ef00-c1c2c3c4c5c6","flowGroups":[{"rule":"DefaultRule_AllowInternetOutBound","flowTuples":["1663146003599,10.0.0.6,52.239.184.180,23956,443,6,O,B,NX,0,0,0,0","1663146003606,10.0.0.6,52.239.184.180,23956,443,6,O,E,NX,3,767,2,1580","1663146003637,10.0.0.6,40.74.146.17,22730,443,6,O,B,NX,0,0,0,0","1663146003640,10.0.0.6,40.74.146.17,22730,443,6,O,E,NX,3,705,4,4569","1663146004251,10.0.0.6,40.74.146.17,22732,443,6,O,B,NX,0,0,0,0","1663146004251,10.0.0.6,40.74.146.17,22732,443,6,O,E,NX,3,705,4,4569","1663146004622,10.0.0.6,40.74.146.17,22734,443,6,O,B,NX,0,0,0,0","1663146004622,10.0.0.6,40.74.146.17,22734,443,6,O,E,NX,2,134,1,108","1663146017343,10.0.0.6,104.16.218.84,36776,443,6,O,B,NX,0,0,0,0","1663146022793,10.0.0.6,104.16.218.84,36776,443,6,O,E,NX,22,2217,33,32466"]}]},{"aclID":"01020304-abcd-ef00-1234-102030405060","flowGroups":[{"rule":"BlockHighRiskTCPPortsFromInternet","flowTuples":["1663145998065,101.33.218.153,10.0.0.6,55188,22,6,I,D,NX,0,0,0,0","1663146005503,192.241.200.164,10.0.0.6,35276,119,6,I,D,NX,0,0,0,0"]},{"rule":"Internet","flowTuples":["1663145989563,20.106.221.10,10.0.0.6,50557,44357,6,I,D,NX,0,0,0,0","1663145989679,20.55.117.81,10.0.0.6,62797,35945,6,I,D,NX,0,0,0,0","1663145989709,20.55.113.5,10.0.0.6,51961,65515,6,I,D,NX,0,0,0,0","1663145990049,13.65.224.51,10.0.0.6,40497,40129,6,I,D,NX,0,0,0,0","1663145990145,20.55.117.81,10.0.0.6,62797,30472,6,I,D,NX,0,0,0,0","1663145990175,20.55.113.5,10.0.0.6,51961,28184,6,I,D,NX,0,0,0,0","1663146015545,20.106.221.10,10.0.0.6,50557,31244,6,I,D,NX,0,0,0,0"]}]}]}}]}
@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: 0.1.0
+  changes:
+    - description: Initial release.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/9680
@@ -0,0 +1,3 @@
+fields:
+  tags:
+    - preserve_original_event
@@ -0,0 +1 @@
+{"time":"2022-09-14T09:00:52.5625085Z","flowLogVersion":4,"flowLogGUID":"abcdef01-2345-6789-0abc-def012345678","macAddress":"0022487bC205","category":"FlowLogFlowEvent","flowLogResourceID":"/SUBSCRIPTIONS/00000000-0000-0000-0000-000000000000/RESOURCEGROUPS/NETWORKWATCHERRG/PROVIDERS/MICROSOFT.NETWORK/NETWORKWATCHERS/NETWORKWATCHER_EASTUS2EUAP/FLOWLOGS/VNETFLOWLOG","targetResourceID":"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myResourceGroup/providers/Microsoft.Network/virtualNetworks/myVNet","operationName":"FlowLogFlowEvent","flowRecords":{"flows":[{"aclID":"00000000-1234-abcd-ef00-c1c2c3c4c5c6","flowGroups":[{"rule":"DefaultRule_AllowInternetOutBound","flowTuples":["1663146003599,1.128.0.0,175.16.199.0,23956,443,6,O,B,NX,0,0,0,0","1663146003606,216.160.83.56,81.2.69.142,36776,443,6,O,E,NX,22,2217,33,0"]}]}]}}
@@ -0,0 +1,169 @@
+{
+    "expected": [
+        {
+            "@timestamp": "2022-09-14T09:00:52.562Z",
+            "azure": {
+                "resource": {
+                    "group": "NETWORKWATCHERRG",
+                    "id": "/SUBSCRIPTIONS/00000000-0000-0000-0000-000000000000/RESOURCEGROUPS/NETWORKWATCHERRG/PROVIDERS/MICROSOFT.NETWORK/NETWORKWATCHERS/NETWORKWATCHER_EASTUS2EUAP/FLOWLOGS/VNETFLOWLOG",
+                    "name": "NETWORKWATCHER_EASTUS2EUAP/FLOWLOGS/VNETFLOWLOG",
+                    "provider": "MICROSOFT.NETWORK/NETWORKWATCHERS"
+                },
+                "subscription_id": "00000000-0000-0000-0000-000000000000"
+            },
+            "azure_network_watcher_vnet": {
+                "log": {
+                    "category": "FlowLogFlowEvent",
+                    "flow_log": {
+                        "guid": "abcdef01-2345-6789-0abc-def012345678",
+                        "resource_id": "/SUBSCRIPTIONS/00000000-0000-0000-0000-000000000000/RESOURCEGROUPS/NETWORKWATCHERRG/PROVIDERS/MICROSOFT.NETWORK/NETWORKWATCHERS/NETWORKWATCHER_EASTUS2EUAP/FLOWLOGS/VNETFLOWLOG",
+                        "version": "4"
+                    },
+                    "mac_address": "00-22-48-7B-C2-05",
+                    "operation_name": "FlowLogFlowEvent",
+                    "records": {
+                        "flows": [
+                            {
+                                "acl_id": "00000000-1234-abcd-ef00-c1c2c3c4c5c6",
+                                "groups": [
+                                    {
+                                        "rule": "DefaultRule_AllowInternetOutBound",
+                                        "tuples": [
+                                            {
+                                                "bytes": {
+                                                    "received": 0,
+                                                    "sent": 0
+                                                },
+                                                "destination": {
+                                                    "ip": "175.16.199.0",
+                                                    "port": 443
+                                                },
+                                                "flow": {
+                                                    "direction": "Outbound",
+                                                    "encryption": "NX",
+                                                    "state": "Begin"
+                                                },
+                                                "packets": {
+                                                    "received": 0,
+                                                    "sent": 0
+                                                },
+                                                "protocol": "6",
+                                                "source": {
+                                                    "ip": "1.128.0.0",
+                                                    "port": 23956
+                                                },
+                                                "timestamp": "2022-09-14T09:00:03.599Z"
+                                            },
+                                            {
+                                                "bytes": {
+                                                    "received": 0,
+                                                    "sent": 2217
+                                                },
+                                                "destination": {
+                                                    "ip": "81.2.69.142",
+                                                    "port": 443
+                                                },
+                                                "flow": {
+                                                    "direction": "Outbound",
+                                                    "encryption": "NX",
+                                                    "state": "End"
+                                                },
+                                                "packets": {
+                                                    "received": 33,
+                                                    "sent": 22
+                                                },
+                                                "protocol": "6",
+                                                "source": {
+                                                    "ip": "216.160.83.56",
+                                                    "port": 36776
+                                                },
+                                                "timestamp": "2022-09-14T09:00:03.606Z"
+                                            }
+                                        ]
+                                    }
+                                ]
+                            }
+                        ]
+                    },
+                    "target_resource_id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myResourceGroup/providers/Microsoft.Network/virtualNetworks/myVNet",
+                    "time": "2022-09-14T09:00:52.562Z"
+                }
+            },
+            "cloud": {
+                "provider": "azure"
+            },
+            "destination": {
+                "bytes": [
+                    0
+                ],
+                "ip": [
+                    "175.16.199.0",
+                    "81.2.69.142"
+                ],
+                "packets": [
+                    0,
+                    33
+                ],
+                "port": [
+                    443
+                ]
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "category": [
+                    "network"
+                ],
+                "kind": "event",
+                "original": "{\"time\":\"2022-09-14T09:00:52.5625085Z\",\"flowLogVersion\":4,\"flowLogGUID\":\"abcdef01-2345-6789-0abc-def012345678\",\"macAddress\":\"0022487bC205\",\"category\":\"FlowLogFlowEvent\",\"flowLogResourceID\":\"/SUBSCRIPTIONS/00000000-0000-0000-0000-000000000000/RESOURCEGROUPS/NETWORKWATCHERRG/PROVIDERS/MICROSOFT.NETWORK/NETWORKWATCHERS/NETWORKWATCHER_EASTUS2EUAP/FLOWLOGS/VNETFLOWLOG\",\"targetResourceID\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myResourceGroup/providers/Microsoft.Network/virtualNetworks/myVNet\",\"operationName\":\"FlowLogFlowEvent\",\"flowRecords\":{\"flows\":[{\"aclID\":\"00000000-1234-abcd-ef00-c1c2c3c4c5c6\",\"flowGroups\":[{\"rule\":\"DefaultRule_AllowInternetOutBound\",\"flowTuples\":[\"1663146003599,1.128.0.0,175.16.199.0,23956,443,6,O,B,NX,0,0,0,0\",\"1663146003606,216.160.83.56,81.2.69.142,36776,443,6,O,E,NX,22,2217,33,0\"]}]}]}}",
+                "type": [
+                    "info"
+                ]
+            },
+            "network": {
+                "direction": [
+                    "outbound"
+                ],
+                "iana_number": [
+                    "6"
+                ]
+            },
+            "related": {
+                "ip": [
+                    "175.16.199.0",
+                    "81.2.69.142",
+                    "1.128.0.0",
+                    "216.160.83.56"
+                ]
+            },
+            "rule": {
+                "name": [
+                    "DefaultRule_AllowInternetOutBound"
+                ]
+            },
+            "source": {
+                "bytes": [
+                    0,
+                    2217
+                ],
+                "ip": [
+                    "1.128.0.0",
+                    "216.160.83.56"
+                ],
+                "mac": "00-22-48-7B-C2-05",
+                "packets": [
+                    0,
+                    22
+                ],
+                "port": [
+                    23956,
+                    36776
+                ]
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
+        }
+    ]
+}
@@ -0,0 +1,11 @@
+service: azure-network-watcher-vnet-log
+input: azure-blob-storage
+data_stream:
+  vars:
+    storage_url: "http://{{Hostname}}:{{Port}}/devstoreaccount1/"
+    account_name: devstoreaccount1
+    service_account_key: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+    containers: |
+      - name: azure-container1
+assert:
+  hit_count: 1
@@ -0,0 +1,47 @@
+{{#if account_name}}
+account_name: {{account_name}}
+{{/if}}
+{{#if service_account_key}}
+auth.shared_credentials.account_key: {{service_account_key}}
+{{/if}}
+{{#if service_account_uri}}
+auth.connection_string.uri: {{service_account_uri}}
+{{/if}}
+{{#if storage_url}}
+storage_url: {{storage_url}} 
+{{/if}}
+{{#if number_of_workers}}
+max_workers: {{number_of_workers}}
+{{/if}}
+{{#if poll}} 
+poll: {{poll}}
+{{/if}}
+{{#if poll_interval}}
+poll_interval: {{poll_interval}}
+{{/if}}
+{{#if containers}}
+containers:
+{{containers}}
+{{/if}}
+{{#if file_selectors}}
+file_selectors:
+{{file_selectors}}
+{{/if}}
+{{#if timestamp_epoch}}
+timestamp_epoch: {{timestamp_epoch}}
+{{/if}}
+expand_event_list_from_field: records
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#each tags as |tag|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}