From 18ba9dd019f9c4f3eada2eef79b82e2c6b2a8864 Mon Sep 17 00:00:00 2001 From: sumoanema Date: Thu, 24 Oct 2024 13:26:51 +0530 Subject: [PATCH 1/5] Doc for masking for windows source template --- .../processing-rules/mask-rules-windows.md | 158 ++++++++++++++++++ .../processing-rules/mask-rules.md | 4 + 2 files changed, 162 insertions(+) create mode 100644 docs/send-data/opentelemetry-collector/remote-management/processing-rules/mask-rules-windows.md diff --git a/docs/send-data/opentelemetry-collector/remote-management/processing-rules/mask-rules-windows.md b/docs/send-data/opentelemetry-collector/remote-management/processing-rules/mask-rules-windows.md new file mode 100644 index 0000000000..f5e1171f0c --- /dev/null +++ b/docs/send-data/opentelemetry-collector/remote-management/processing-rules/mask-rules-windows.md @@ -0,0 +1,158 @@ +--- +id: mask-rules-windows +title: Mask Rules for Windows Source Template (Beta) +sidebar_label: Mask Rules for Windows +description: Create a mask rule to replace an expression with a mask string. +--- + + + + +

Beta

+ +:::note +This doc is specific for masking with Windows Source Template. For masking logs for any other source template please refer [this](mask-rules.md) doc. +::: + +A mask rule is a type of processing rule that hides irrelevant or sensitive information from logs before ingestion. When you create a mask rule, whatever key you choose to mask, value for that key will be matched against a regex and replaced with a mask string before it is sent to Sumo Logic. You can provide a mask string, or use the default `"#####"`. + +Ingestion volume is calculated after applying the mask filter. If the mask reduces the size of the log, the smaller size will be measured against ingestion limits. Masking is a good method for reducing overall ingestion volume. + +Following inputs are required from user to mask specific field in Windows Event Log : +- **Key** - This should point to the key in windows event log, value for which needs to get masked. This key can be nested as well and each level can be seperated by dot(.) for example `provider.guid` +- **Regex** - This is to identify the part of string value, which needs to get masked. +- **Replacement** - This is to get the string which will be substituted in place of the string selected through regex expression. + + +For example, to mask numbers inside `guid` under `provider` field from this log: + +``` +{ + "record_id": 163054, + "channel": "Security", + "event_data": { + "TargetDomainName": "EC2AMAZ-V57A85N", + "SubjectUserSid": "S-1-5-21-2435622068-2303779566-2814161656-500", + "CallerProcessId": "0x1768", + "TargetUserName": "Guest", + "CallerProcessName": "C:\\Windows\\ImmersiveControlPanel\\SystemSettings.exe", + "SubjectUserName": "Administrator", + "TargetSid": "S-1-5-21-2435622068-2303779566-2814161656-501", + "SubjectLogonId": "0x71aef", + "SubjectDomainName": "EC2AMAZ-V57A85N" + }, + "task": "User Account Management", + "provider": { + "name": "Microsoft-Windows-Security-Auditing", + "guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "event_source": "" + }, + "system_time": "2023-07-14T07:58:59.9575956Z", + "computer": "EC2AMAZ-V57A85N", + "opcode": "Info", + "keywords": [ + "Audit Success" + ], + "details": { + "Subject": { + "Security ID": "S-1-5-21-2435622068-2303779566-2814161656-500", + "Account Name": "Administrator", + "Account Domain": "EC2AMAZ-V57A85N", + "Logon ID": "0x71AEF" + }, + "User": { + "Security ID": "S-1-5-21-2435622068-2303779566-2814161656-501", + "Account Name": "Guest", + "Account Domain": "EC2AMAZ-V57A85N" + }, + "Process Information": { + "Process ID": "0x1768", + "Process Name": "C:\\Windows\\ImmersiveControlPanel\\SystemSettings.exe" + } + }, + "message": "A user's local group membership was enumerated.", + "event_id": { + "qualifiers": 0, + "id": 4798 + }, + "level": "Information" +} +``` + +You could use the following masking expression input: +1. Key as `provider.guid` +2. Regex as `[-a-z0-9]+` +3. Replacement as `######` + +Using the above masking options would provide the following result: + +``` +{ + "record_id": 163054, + "channel": "Security", + "event_data": { + "TargetDomainName": "EC2AMAZ-V57A85N", + "SubjectUserSid": "S-1-5-21-2435622068-2303779566-2814161656-500", + "CallerProcessId": "0x1768", + "TargetUserName": "Guest", + "CallerProcessName": "C:\\Windows\\ImmersiveControlPanel\\SystemSettings.exe", + "SubjectUserName": "Administrator", + "TargetSid": "S-1-5-21-2435622068-2303779566-2814161656-501", + "SubjectLogonId": "0x71aef", + "SubjectDomainName": "EC2AMAZ-V57A85N" + }, + "task": "User Account Management", + "provider": { + "name": "Microsoft-Windows-Security-Auditing", + "guid": "{######}", + "event_source": "" + }, + "system_time": "2023-07-14T07:58:59.9575956Z", + "computer": "EC2AMAZ-V57A85N", + "opcode": "Info", + "keywords": [ + "Audit Success" + ], + "details": { + "Subject": { + "Security ID": "S-1-5-21-2435622068-2303779566-2814161656-500", + "Account Name": "Administrator", + "Account Domain": "EC2AMAZ-V57A85N", + "Logon ID": "0x71AEF" + }, + "User": { + "Security ID": "S-1-5-21-2435622068-2303779566-2814161656-501", + "Account Name": "Guest", + "Account Domain": "EC2AMAZ-V57A85N" + }, + "Process Information": { + "Process ID": "0x1768", + "Process Name": "C:\\Windows\\ImmersiveControlPanel\\SystemSettings.exe" + } + }, + "message": "A user's local group membership was enumerated.", + "event_id": { + "qualifiers": 0, + "id": 4798 + }, + "level": "Information" +} +``` + +## Limitations +- Users can only mask the data which is a string in the windows event log json. Any value which is not a string like integer cannot be masked. This is because internally we will be using [replace_pattern](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/pkg/ottl/ottlfuncs/README.md#replace_pattern) ottl function which can work on strings only. +- Limtation is also applicable for masking a value which is nested inside some array. + + +:::note +- For masking, we use the [replace_pattern](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/pkg/ottl/ottlfuncs/README.md#replace_pattern) OTTL function. In this function: + - $ must be escaped as $$ to bypass environment variable substitution logic. + - To input a literal $, use $$$. +- When masking strings containing special characters like double quotes (`"`) and backslashes (`\`), these characters will be escaped by a backslash when masking the logs. +::: + + +:::important +Any masking expression should be tested and verified with a sample source file before applying it to your production logs. +::: + diff --git a/docs/send-data/opentelemetry-collector/remote-management/processing-rules/mask-rules.md b/docs/send-data/opentelemetry-collector/remote-management/processing-rules/mask-rules.md index 00cfab2c04..45e9188692 100644 --- a/docs/send-data/opentelemetry-collector/remote-management/processing-rules/mask-rules.md +++ b/docs/send-data/opentelemetry-collector/remote-management/processing-rules/mask-rules.md @@ -10,6 +10,10 @@ description: Create a mask rule to replace an expression with a mask string.

Beta

+:::note +For masking logs for Windows Source Template please refer [this](mask-rules-windows.md) doc. +::: + A mask rule is a type of processing rule that hides irrelevant or sensitive information from logs before ingestion. When you create a mask rule, whatever expression you choose to mask will be replaced with a mask string before it is sent to Sumo Logic. You can provide a mask string, or use the default `"#####"`. Ingestion volume is calculated after applying the mask filter. If the mask reduces the size of the log, the smaller size will be measured against ingestion limits. Masking is a good method for reducing overall ingestion volume. From d43707df54a5c512867d6cf7670dbb522a81b431 Mon Sep 17 00:00:00 2001 From: sumoanema Date: Thu, 24 Oct 2024 13:30:26 +0530 Subject: [PATCH 2/5] Pointing to windows masking doc from windows Source template doc page --- .../remote-management/source-templates/windows/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/send-data/opentelemetry-collector/remote-management/source-templates/windows/index.md b/docs/send-data/opentelemetry-collector/remote-management/source-templates/windows/index.md index 6fbfe9cef6..250b137e6d 100644 --- a/docs/send-data/opentelemetry-collector/remote-management/source-templates/windows/index.md +++ b/docs/send-data/opentelemetry-collector/remote-management/source-templates/windows/index.md @@ -65,7 +65,7 @@ import ProcMetrics from '../../../../../reuse/apps/opentelemetry/process-metric- - **Scan Interval**. The frequency at which the source is scanned. -- **Processing Rules**. You can add processing rules for logs/metrics collected. To learn more, refer to [Processing Rules](../../processing-rules/index.md). +- **Processing Rules**. You can add processing rules for logs/metrics collected. To learn more, refer to [Processing Rules](../../processing-rules/index.md). For masking windows event logs please refer to [this](../../processing-rules/mask-rules-windows.md) doc ### Step 3: Push the source template to the desired remotely managed collectors From 1d91575c151cb025bce5d0fefdb2d39840d2a8f3 Mon Sep 17 00:00:00 2001 From: Jagadisha V <129049263+JV0812@users.noreply.github.com> Date: Fri, 25 Oct 2024 13:10:24 +0530 Subject: [PATCH 3/5] Update mask-rules-windows.md --- .../processing-rules/mask-rules-windows.md | 35 +++++++++---------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/docs/send-data/opentelemetry-collector/remote-management/processing-rules/mask-rules-windows.md b/docs/send-data/opentelemetry-collector/remote-management/processing-rules/mask-rules-windows.md index f5e1171f0c..f65d060163 100644 --- a/docs/send-data/opentelemetry-collector/remote-management/processing-rules/mask-rules-windows.md +++ b/docs/send-data/opentelemetry-collector/remote-management/processing-rules/mask-rules-windows.md @@ -11,18 +11,21 @@ description: Create a mask rule to replace an expression with a mask string.

Beta

:::note -This doc is specific for masking with Windows Source Template. For masking logs for any other source template please refer [this](mask-rules.md) doc. +This document only support masking logs for Windows source template. Refer to [Mask Rules](mask-rules.md) to mask logs for other source template. ::: -A mask rule is a type of processing rule that hides irrelevant or sensitive information from logs before ingestion. When you create a mask rule, whatever key you choose to mask, value for that key will be matched against a regex and replaced with a mask string before it is sent to Sumo Logic. You can provide a mask string, or use the default `"#####"`. +A mask rule is a type of processing rule that hides irrelevant or sensitive information from logs before they are ingested. When you create a mask rule, the selected key will have its value matched against a regex pattern, which will then be replaced with a mask string before being sent to Sumo Logic. You can provide a custom mask string or use the default string, `"#####"`. -Ingestion volume is calculated after applying the mask filter. If the mask reduces the size of the log, the smaller size will be measured against ingestion limits. Masking is a good method for reducing overall ingestion volume. +Ingestion volume is calculated after applying the mask filter. If masking reduces the log size, the smaller size will be considered against the ingestion limits. Masking is an effective method for reducing overall ingestion volume. -Following inputs are required from user to mask specific field in Windows Event Log : -- **Key** - This should point to the key in windows event log, value for which needs to get masked. This key can be nested as well and each level can be seperated by dot(.) for example `provider.guid` -- **Regex** - This is to identify the part of string value, which needs to get masked. -- **Replacement** - This is to get the string which will be substituted in place of the string selected through regex expression. +To mask specific fields in the Windows Event Log, the following inputs are required: +- **Key**. This should point to the key in the Windows Event Log for which the value needs to be masked. This key can be nested, with each level separated by a dot(.). For example, `provider.guid`. +- **Regex**. This identifies the part of the string value that needs to be masked. +- ** Replacement **. This is to get the string that will be substituted in place of the string that was selected through the regex expression. +:::important +Any masking expression should be tested and verified with a sample source file before applying it to your production logs. +::: For example, to mask numbers inside `guid` under `provider` field from this log: @@ -80,9 +83,9 @@ For example, to mask numbers inside `guid` under `provider` field from this log: ``` You could use the following masking expression input: -1. Key as `provider.guid` -2. Regex as `[-a-z0-9]+` -3. Replacement as `######` +1. Key as `provider.guid`. +1. Regex as `[-a-z0-9]+`. +1. Replacement as `######`. Using the above masking options would provide the following result: @@ -139,11 +142,6 @@ Using the above masking options would provide the following result: } ``` -## Limitations -- Users can only mask the data which is a string in the windows event log json. Any value which is not a string like integer cannot be masked. This is because internally we will be using [replace_pattern](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/pkg/ottl/ottlfuncs/README.md#replace_pattern) ottl function which can work on strings only. -- Limtation is also applicable for masking a value which is nested inside some array. - - :::note - For masking, we use the [replace_pattern](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/pkg/ottl/ottlfuncs/README.md#replace_pattern) OTTL function. In this function: - $ must be escaped as $$ to bypass environment variable substitution logic. @@ -151,8 +149,7 @@ Using the above masking options would provide the following result: - When masking strings containing special characters like double quotes (`"`) and backslashes (`\`), these characters will be escaped by a backslash when masking the logs. ::: +## Limitations -:::important -Any masking expression should be tested and verified with a sample source file before applying it to your production logs. -::: - +- You can *only* mask the data which is a string in the Windows event log JSON. +- You cannot mask a value which is nested inside any array. From 34be70135422cd1480f712636c119564a4bc17bc Mon Sep 17 00:00:00 2001 From: Alekh Nema <91047769+sumoanema@users.noreply.github.com> Date: Fri, 25 Oct 2024 13:12:14 +0530 Subject: [PATCH 4/5] Apply suggestions from code review Co-authored-by: Jagadisha V <129049263+JV0812@users.noreply.github.com> --- .../remote-management/processing-rules/mask-rules.md | 2 +- .../remote-management/source-templates/windows/index.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/send-data/opentelemetry-collector/remote-management/processing-rules/mask-rules.md b/docs/send-data/opentelemetry-collector/remote-management/processing-rules/mask-rules.md index 45e9188692..d3d6b53544 100644 --- a/docs/send-data/opentelemetry-collector/remote-management/processing-rules/mask-rules.md +++ b/docs/send-data/opentelemetry-collector/remote-management/processing-rules/mask-rules.md @@ -11,7 +11,7 @@ description: Create a mask rule to replace an expression with a mask string.

Beta

:::note -For masking logs for Windows Source Template please refer [this](mask-rules-windows.md) doc. +This document do not support masking logs for Windows source template. Refer to [Mask Rules for Windows Source Template](mask-rules-windows.md) to mask logs for Windows source template. ::: A mask rule is a type of processing rule that hides irrelevant or sensitive information from logs before ingestion. When you create a mask rule, whatever expression you choose to mask will be replaced with a mask string before it is sent to Sumo Logic. You can provide a mask string, or use the default `"#####"`. diff --git a/docs/send-data/opentelemetry-collector/remote-management/source-templates/windows/index.md b/docs/send-data/opentelemetry-collector/remote-management/source-templates/windows/index.md index 250b137e6d..9711fdd14d 100644 --- a/docs/send-data/opentelemetry-collector/remote-management/source-templates/windows/index.md +++ b/docs/send-data/opentelemetry-collector/remote-management/source-templates/windows/index.md @@ -65,7 +65,7 @@ import ProcMetrics from '../../../../../reuse/apps/opentelemetry/process-metric- - **Scan Interval**. The frequency at which the source is scanned. -- **Processing Rules**. You can add processing rules for logs/metrics collected. To learn more, refer to [Processing Rules](../../processing-rules/index.md). For masking windows event logs please refer to [this](../../processing-rules/mask-rules-windows.md) doc +- **Processing Rules**. You can add processing rules for logs/metrics collected. To learn more, refer to [Processing Rules](../../processing-rules/index.md). For masking windows event logs, refer to [Mask Rules for Windows Source Template](mask-rules-windows.md). ### Step 3: Push the source template to the desired remotely managed collectors From 40e4d21a4a1aa73fa31d949edc3b9352f7ec9b7c Mon Sep 17 00:00:00 2001 From: John Pipkin Date: Fri, 25 Oct 2024 09:49:23 -0500 Subject: [PATCH 5/5] Updates from review --- .../source-templates/windows/index.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/send-data/opentelemetry-collector/remote-management/source-templates/windows/index.md b/docs/send-data/opentelemetry-collector/remote-management/source-templates/windows/index.md index 9711fdd14d..1edc9d5a66 100644 --- a/docs/send-data/opentelemetry-collector/remote-management/source-templates/windows/index.md +++ b/docs/send-data/opentelemetry-collector/remote-management/source-templates/windows/index.md @@ -17,14 +17,14 @@ import TabItem from '@theme/TabItem'; Thumbnail iconThumbnail icon -The Windows source template creates an OpenTelemetry configuration that can be pushed to a remotely managed OpenTelemetry collector (abbreviated as otelcol). By creating this source template and pushing the config to the appropriate OpenTelemetry agent, you can ensure collection of windows event log and metrics of Windows to Sumo Logic. +The Windows source template creates an OpenTelemetry configuration that can be pushed to a remotely managed OpenTelemetry collector (abbreviated as otelcol). By creating this source template and pushing the config to the appropriate OpenTelemetry agent, you can ensure collection of Windows event log and metrics of Windows to Sumo Logic. ## Fields creation in Sumo Logic for Windows -If not already present, the following [Fields](/docs/manage/fields/) are created as part of source template creation. +If not already present, the following [fields](/docs/manage/fields/) are created as part of source template creation. - **`sumo.datasource`**. Fixed value of **windows**. -- **`deployment.environment`** User configured field at the time of collector installation. This identifies the environment where the Windows system resides. For example: `dev`, `prod`, or `qa`. +- **`deployment.environment`**. User configured field at the time of collector installation. This identifies the environment where the Windows system resides. For example: `dev`, `prod`, or `qa`. - **`host.group`**. This is a collector-level field that is user configured at the time of collector installation. It identifies the Windows host group. - **`host.name`**. This is tagged through the resourcedetection processor. It holds the value of the host name where the OTel collector is installed. @@ -45,27 +45,27 @@ import CollectorInstallation from '../../../../../reuse/apps/opentelemetry/colle ### Step 2: Configure the source template -In this step, you will configure the yaml required for Windows Collection. Below are the inputs required for configuration: +In this step, you will configure the YAML required for Windows collection. Below are the inputs required for configuration: - **Name**. Name of the source template. - **Description**. Description for the source template. #### Logs Collection -- **Fields/Metadata**. You can provide any customer fields to be tagged with the data collected. By default, sumo tags `_sourceCategory` with the value otel/windows. -- **Windows Event**. In this section you can select choose among the most widely used windows event channel for which windows event log collection will be enabled. You can also provide **Custom Event Channels** providing any customer event channel for which event logs are to be collected. +- **Fields/Metadata**. You can provide any customer fields to be tagged with the data collected. By default, Sumo Logic tags `_sourceCategory` with the value `otel/windows`. +- **Windows Event**. In this section you can select choose among the most widely used Windows event channel for which Windows event log collection will be enabled. You can also provide **Custom Event Channels** providing any customer event channel for which event logs are to be collected. - **Forward to SIEM**. Check the checkbox to forward your data to [Cloud SIEM](/docs/cse). #### Metrics Collection - **Metrics**. Select the metric scrappers you want to enable. By default, metric collection for CPU, memory, disk, load, file system, network and paging are enabled, and process metric collection is disabled. -##### Enable process metric collection (Optional) +##### Enable process metric collection (optional) import ProcMetrics from '../../../../../reuse/apps/opentelemetry/process-metric-collection.md'; - **Scan Interval**. The frequency at which the source is scanned. -- **Processing Rules**. You can add processing rules for logs/metrics collected. To learn more, refer to [Processing Rules](../../processing-rules/index.md). For masking windows event logs, refer to [Mask Rules for Windows Source Template](mask-rules-windows.md). +- **Processing Rules**. You can add processing rules for logs/metrics collected. To learn more, refer to [Processing Rules](/docs/send-data/opentelemetry-collector/remote-management/processing-rules/). For masking windows event logs, refer to [Mask Rules for Windows Source Template](/docs/send-data/opentelemetry-collector/remote-management/processing-rules/mask-rules-windows). ### Step 3: Push the source template to the desired remotely managed collectors @@ -74,5 +74,5 @@ import DataConfiguration from '../../../../../reuse/apps/opentelemetry/data-conf :::info -Refer to the [changelog](changelog.md) for information on periodic updates to this source template. +Refer to the [changelog](/docs/send-data/opentelemetry-collector/remote-management/source-templates/windows/changelog/) for information on periodic updates to this source template. :::