[BugFix] Remove script_block_signature - Performance Problems

packages/windows/changelog.yml

@@ -1,4 +1,10 @@
 # newer versions go on top
+- version: "3.2.2"
+  changes:
+    - description: |
+        Remove the `script_block_signature` field to improve pipeline performance.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/15907
 - version: "3.2.1"
   changes:
     - description: |

packages/windows/data_stream/forwarded/_dev/test/pipeline/test-powershell-operational-events.json-expected.json

@@ -308,7 +308,6 @@
                     "script_block_hash": "r0sdjfD0qsH7ckPwQpUfLLA0Slo=",
                     "script_block_id": "50d2dbda-7361-4926-a94d-d9eadfdb43fa",
                     "script_block_length": 3350,
-                    "script_block_signature": "MIInoQYJKoZIhvcNAQcCoIInkjCCJ44CAQExDzANBglghkgBZQMEAgEFADB5BgorBgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLGKX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCANw97w1D+bi5LY8ZEuubcA0tI0Z0h+CImFRYop+IIqQaCCDYEwggX/MIID56ADAgECAhMzAAACUoszqviV8znbAAAAAAJSMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD+nC4D7IMA1+6smM7fbSJa7o4BHfyje8PHB3w9GF223mZTG0EhBlultQkMSpV/c889hsbwx16Cr5sY9M/lSRt4oC3qzSuTmYd6VYJ/ILt9ptrpOkaYCiXXRx8Cfz7w53wAu/J8xJjNWvrKxkcc8XiUXPfGGTXujyiS2MqvztBkg6wCduFKqogmvOtQiiwQQxEG6lU/rss27omoTUc41EawOr1km5y+fUS9aoYX9K8NNhFH6TSni3dp/+Hiyif1T7Xg0cBy4yHuYxMmRrFcmGeplW3KhXHfkJjbHaVs1QgnRfkgFuypwF5YoFWrW7Xgj+aZCDKSoYq45E4v0ryIvyu0shBoHQXREAzpBv3L9h5A9vEFQG4alCI57oSbdqJ1YIaggkTQHR2CWdB7FnQilCqqZjSnAtXYZh/RD+PX6fg1UyUUQf5ohnw951pQeKYTYHmFwut+RibzdbHEF/kLZr6SZsDupCv",
                     "script_block_surprisal_stdev": 1.760352963786286,
                     "script_block_text": "###\n# ==++==\n#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n###\n@{\n    GUID = \"4ae9fd46-338a-459c-8186-07f910774cb8\"\n    Author = \"Microsoft Corporation\"\n    CompanyName = \"Microsoft Corporation\"\n    Copyright = \"(C) Microsoft Corporation. All rights reserved.\"\n    HelpInfoUri = \"https://go.microsoft.com/fwlink/?linkid=2113634\"\n    ModuleVersion = \"1.4.8.1\"\n    PowerShellVersion = \"3.0\"\n    ClrVersion = \"4.0\"\n    RootModule = \"PackageManagement.psm1\"\n\tDescription = 'PackageManagement (a.k.a. OneGet) is a new way to discover and install software packages from around the web.\n It is a manager or multiplexor of existing package managers (also called package providers) that unifies Windows package management with a single Windows PowerShell interface. With PackageManagement, you can do the following.\n  - Manage a list of software repositories in which packages can be searched, acquired and installed\n  - Discover software packages\n  - Seamlessly install, uninstall, and inventory packages from one or more software repositories'\n\n    CmdletsToExport = @(\n        'Find-Package',\n        'Get-Package',\n        'Get-PackageProvider',\n        'Get-PackageSource',\n        'Install-Package',\n        'Import-PackageProvider'\n        'Find-PackageProvider'\n        'Install-PackageProvider'\n        'Register-PackageSource',\n        'Set-PackageSource',\n        'Unregister-PackageSource',\n        'Uninstall-Package'\n        'Save-Package'\n\t)\n\n\tFormatsToProcess  = @('PackageManagement.format.ps1xml')\n\n\tPrivateData = @{\n        PSData = @{\n            Tags = @('PackageManagement', 'PSEdition_Core', 'PSEdition_Desktop', 'Linux', 'Mac')\n            ProjectUri = 'https://oneget.org'\n            ReleaseNotes = @'\n## 1.4.8.1\n- Update PackageManagement's strong name signing\n\n## 1.4.8\n- Add NuGet as a source when generating nuget.config file for user in the NuGet Provider\n\n## 1.4.7\n- Update security protocol to use TLS 1.2\n- Remove catalog file\n\n## 1.4.6\n- Update `HelpInfoUri` to point to the latest content\n\n## 1.4.5\n- Bug fix for deadlock when getting parameters in an event\n\n## 1.4.4\n- Bug fix when installing modules from private feeds\n\n ## 1.4.3\n- Another bug fix when registering repositories with PowerShellGet\n\n## 1.4.2\n- Bug fix for passing credentials from PowerShellGet when registering repositories\n\n## 1.4.1\n- Bug fix for using credential provider installed in Visual Studio\n\n## 1.4\n- Allow credential persistance for registering private repositories and finding or installing packages from those repositories\n\n## 1.3.2\n- Enable bootstrap on PSCore\n- Bug fix to run on .NET Core 3.0\n\n## 1.3.1\n- Targets net452 and netstandard2.0 instead of net451, netcoreapp2.0, and netstandard1.6\n            \n## Previous releases are not included in this Changelog\n'@\n        }\n    }\n}\n\n# SIG # Begin signature block\n# MIInoQYJKoZIhvcNAQcCoIInkjCCJ44CAQExDzANBglghkgBZQMEAgEFADB5Bgor\n# BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG\n# KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCANw97w1D+bi5LY\n# 8ZEuubcA0tI0Z0h+CImFRYop+IIqQaCCDYEwggX/MIID56ADAgECAhMzAAACUosz\n# qviV8znbAAAAAAJSMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD\n# +nC4D7IMA1+6smM7fbSJa7o4BHfyje8PHB3w9GF223mZTG0EhBlultQkMSpV/c88\n# 9hsbwx16Cr5sY9M/lSRt4oC3qzSuTmYd6VYJ/ILt9ptrpOkaYCiXXRx8Cfz7w53w\n# Au/J8xJjNWvrKxkcc8XiUXPfGGTXujyiS2MqvztBkg6wCduFKqogmvOtQiiwQQxE\n# G6lU/rss27omoTUc41EawOr1km5y+fUS9aoYX9K8NNhFH6TSni3dp/+Hiyif1T7X\n# g0cBy4yHuYxMmRrFcmGeplW3KhXHfkJjbHaVs1QgnRfkgFuypwF5YoFWrW7Xgj+a\n# ZCDKSoYq45E4v0ryIvyu0shBoHQXREAzpBv3L9h5A9vEFQG4alCI57oSbdqJ1YIa\n# ggkTQHR2CWdB7FnQilCqqZjSnAtXYZh/RD+PX6fg1UyUUQf5ohnw951pQeKYTYHm\n# Fwut+RibzdbHEF/kLZr6SZsDupCv\n# SIG # End signature block",
                     "script_block_unique_symbols": 79

packages/windows/data_stream/forwarded/elasticsearch/ingest_pipeline/powershell_operational.yml

@@ -302,22 +302,6 @@ processors:
   - trim:
       field: powershell.file.script_block_text
       ignore_missing: true
-  - gsub:
-      field: powershell.file.script_block_text
-      target_field: _temp.script_block_signature
-      pattern: "(?s).+# SIG # Begin signature block"
-      replacement: "# SIG # Begin signature block"
-      ignore_missing: true
-  - dissect:
-      field: _temp.script_block_signature
-      pattern: "# SIG # Begin signature block%{powershell.file.script_block_signature}# SIG # End signature block"
-      ignore_missing: true
-      ignore_failure: true
-  - gsub:
-      field: powershell.file.script_block_signature
-      pattern: "\\n# |\\n"
-      replacement: ""
-      ignore_missing: true
   - gsub:
       field: powershell.file.script_block_text
       target_field: _temp.script_block_no_space

packages/windows/data_stream/forwarded/fields/fields.yml

@@ -141,10 +141,6 @@
         Text of the executed script block.
 
       example: ".\\a_script.ps1"
-    - name: script_block_signature
-      type: keyword
-      description: >
-        If present in the script, the script signature.
 
     - name: script_block_hash
       type: keyword

packages/windows/data_stream/powershell_operational/_dev/test/pipeline/test-events.json-expected.json

@@ -292,7 +292,6 @@
                     "script_block_hash": "r0sdjfD0qsH7ckPwQpUfLLA0Slo=",
                     "script_block_id": "50d2dbda-7361-4926-a94d-d9eadfdb43fa",
                     "script_block_length": 3350,
-                    "script_block_signature": "MIInoQYJKoZIhvcNAQcCoIInkjCCJ44CAQExDzANBglghkgBZQMEAgEFADB5BgorBgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLGKX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCANw97w1D+bi5LY8ZEuubcA0tI0Z0h+CImFRYop+IIqQaCCDYEwggX/MIID56ADAgECAhMzAAACUoszqviV8znbAAAAAAJSMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD+nC4D7IMA1+6smM7fbSJa7o4BHfyje8PHB3w9GF223mZTG0EhBlultQkMSpV/c889hsbwx16Cr5sY9M/lSRt4oC3qzSuTmYd6VYJ/ILt9ptrpOkaYCiXXRx8Cfz7w53wAu/J8xJjNWvrKxkcc8XiUXPfGGTXujyiS2MqvztBkg6wCduFKqogmvOtQiiwQQxEG6lU/rss27omoTUc41EawOr1km5y+fUS9aoYX9K8NNhFH6TSni3dp/+Hiyif1T7Xg0cBy4yHuYxMmRrFcmGeplW3KhXHfkJjbHaVs1QgnRfkgFuypwF5YoFWrW7Xgj+aZCDKSoYq45E4v0ryIvyu0shBoHQXREAzpBv3L9h5A9vEFQG4alCI57oSbdqJ1YIaggkTQHR2CWdB7FnQilCqqZjSnAtXYZh/RD+PX6fg1UyUUQf5ohnw951pQeKYTYHmFwut+RibzdbHEF/kLZr6SZsDupCv",
                     "script_block_surprisal_stdev": 1.760352963786286,
                     "script_block_text": "###\n# ==++==\n#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n###\n@{\n    GUID = \"4ae9fd46-338a-459c-8186-07f910774cb8\"\n    Author = \"Microsoft Corporation\"\n    CompanyName = \"Microsoft Corporation\"\n    Copyright = \"(C) Microsoft Corporation. All rights reserved.\"\n    HelpInfoUri = \"https://go.microsoft.com/fwlink/?linkid=2113634\"\n    ModuleVersion = \"1.4.8.1\"\n    PowerShellVersion = \"3.0\"\n    ClrVersion = \"4.0\"\n    RootModule = \"PackageManagement.psm1\"\n\tDescription = 'PackageManagement (a.k.a. OneGet) is a new way to discover and install software packages from around the web.\n It is a manager or multiplexor of existing package managers (also called package providers) that unifies Windows package management with a single Windows PowerShell interface. With PackageManagement, you can do the following.\n  - Manage a list of software repositories in which packages can be searched, acquired and installed\n  - Discover software packages\n  - Seamlessly install, uninstall, and inventory packages from one or more software repositories'\n\n    CmdletsToExport = @(\n        'Find-Package',\n        'Get-Package',\n        'Get-PackageProvider',\n        'Get-PackageSource',\n        'Install-Package',\n        'Import-PackageProvider'\n        'Find-PackageProvider'\n        'Install-PackageProvider'\n        'Register-PackageSource',\n        'Set-PackageSource',\n        'Unregister-PackageSource',\n        'Uninstall-Package'\n        'Save-Package'\n\t)\n\n\tFormatsToProcess  = @('PackageManagement.format.ps1xml')\n\n\tPrivateData = @{\n        PSData = @{\n            Tags = @('PackageManagement', 'PSEdition_Core', 'PSEdition_Desktop', 'Linux', 'Mac')\n            ProjectUri = 'https://oneget.org'\n            ReleaseNotes = @'\n## 1.4.8.1\n- Update PackageManagement's strong name signing\n\n## 1.4.8\n- Add NuGet as a source when generating nuget.config file for user in the NuGet Provider\n\n## 1.4.7\n- Update security protocol to use TLS 1.2\n- Remove catalog file\n\n## 1.4.6\n- Update `HelpInfoUri` to point to the latest content\n\n## 1.4.5\n- Bug fix for deadlock when getting parameters in an event\n\n## 1.4.4\n- Bug fix when installing modules from private feeds\n\n ## 1.4.3\n- Another bug fix when registering repositories with PowerShellGet\n\n## 1.4.2\n- Bug fix for passing credentials from PowerShellGet when registering repositories\n\n## 1.4.1\n- Bug fix for using credential provider installed in Visual Studio\n\n## 1.4\n- Allow credential persistance for registering private repositories and finding or installing packages from those repositories\n\n## 1.3.2\n- Enable bootstrap on PSCore\n- Bug fix to run on .NET Core 3.0\n\n## 1.3.1\n- Targets net452 and netstandard2.0 instead of net451, netcoreapp2.0, and netstandard1.6\n            \n## Previous releases are not included in this Changelog\n'@\n        }\n    }\n}\n\n# SIG # Begin signature block\n# MIInoQYJKoZIhvcNAQcCoIInkjCCJ44CAQExDzANBglghkgBZQMEAgEFADB5Bgor\n# BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG\n# KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCANw97w1D+bi5LY\n# 8ZEuubcA0tI0Z0h+CImFRYop+IIqQaCCDYEwggX/MIID56ADAgECAhMzAAACUosz\n# qviV8znbAAAAAAJSMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD\n# +nC4D7IMA1+6smM7fbSJa7o4BHfyje8PHB3w9GF223mZTG0EhBlultQkMSpV/c88\n# 9hsbwx16Cr5sY9M/lSRt4oC3qzSuTmYd6VYJ/ILt9ptrpOkaYCiXXRx8Cfz7w53w\n# Au/J8xJjNWvrKxkcc8XiUXPfGGTXujyiS2MqvztBkg6wCduFKqogmvOtQiiwQQxE\n# G6lU/rss27omoTUc41EawOr1km5y+fUS9aoYX9K8NNhFH6TSni3dp/+Hiyif1T7X\n# g0cBy4yHuYxMmRrFcmGeplW3KhXHfkJjbHaVs1QgnRfkgFuypwF5YoFWrW7Xgj+a\n# ZCDKSoYq45E4v0ryIvyu0shBoHQXREAzpBv3L9h5A9vEFQG4alCI57oSbdqJ1YIa\n# ggkTQHR2CWdB7FnQilCqqZjSnAtXYZh/RD+PX6fg1UyUUQf5ohnw951pQeKYTYHm\n# Fwut+RibzdbHEF/kLZr6SZsDupCv\n# SIG # End signature block",
                     "script_block_unique_symbols": 79

packages/windows/data_stream/powershell_operational/elasticsearch/ingest_pipeline/default.yml

@@ -302,22 +302,6 @@ processors:
   - trim:
       field: powershell.file.script_block_text
       ignore_missing: true
-  - gsub:
-      field: powershell.file.script_block_text
-      target_field: _temp.script_block_signature
-      pattern: "(?s).+# SIG # Begin signature block"
-      replacement: "# SIG # Begin signature block"
-      ignore_missing: true
-  - dissect:
-      field: _temp.script_block_signature
-      pattern: "# SIG # Begin signature block%{powershell.file.script_block_signature}# SIG # End signature block"
-      ignore_missing: true
-      ignore_failure: true
-  - gsub:
-      field: powershell.file.script_block_signature
-      pattern: "\\n# |\\n"
-      replacement: ""
-      ignore_missing: true
   - gsub:
       field: powershell.file.script_block_text
       target_field: _temp.script_block_no_space

packages/windows/data_stream/powershell_operational/fields/fields.yml

@@ -102,10 +102,6 @@
         Text of the executed script block.
 
       example: ".\\a_script.ps1"
-    - name: script_block_signature
-      type: keyword
-      description: >
-        If present in the script, the script signature.
 
     - name: script_block_hash
       type: keyword

packages/windows/docs/README.md

@@ -2156,7 +2156,6 @@ An example event for `powershell_operational` looks as following:
 | powershell.file.script_block_hash | A hash of the script to be used in rules. | keyword |
 | powershell.file.script_block_id | Id of the executed script block. | keyword |
 | powershell.file.script_block_length | Total number of characters in the script. | long |
-| powershell.file.script_block_signature | If present in the script, the script signature. | keyword |
 | powershell.file.script_block_surprisal_stdev | Consistency of randomness distribution across the script. Low values indicate uniform randomness. High values indicate mixed patterns with variability. | float |
 | powershell.file.script_block_text | Text of the executed script block. | text |
 | powershell.file.script_block_unique_symbols | Number of distinct characters used in the script. | long |

packages/windows/manifest.yml

@@ -1,6 +1,6 @@
 name: windows
 title: Windows
-version: 3.2.1
+version: 3.2.2
 description: Collect logs and metrics from Windows OS and services with Elastic Agent.
 type: integration
 categories: