[Backport 2.x] PRs for correlation engine and minor UX improvements

.github/workflows/cypress-workflow.yml

@@ -7,7 +7,7 @@ on:
     branches:
       - "*"
 env:
-  OPENSEARCH_DASHBOARDS_VERSION: '2.x'
+  OPENSEARCH_DASHBOARDS_VERSION: '2.7.0'
   OPENSEARCH_VERSION: '2.7.0-SNAPSHOT'
   SECURITY_ANALYTICS_BRANCH: '2.x'
 jobs:
@@ -94,12 +94,12 @@ jobs:
       # Window is slow so wait longer
       - name: Sleep until OSD server starts - windows
         if: ${{ matrix.os == 'windows-latest' }}
-        run: Start-Sleep -s 400
+        run: Start-Sleep -s 450
         shell: powershell
 
       - name: Sleep until OSD server starts - non-windows
         if: ${{ matrix.os != 'windows-latest' }}
-        run: sleep 300
+        run: sleep 450
         shell: bash
 
       - name: Install Cypress
@@ -132,6 +132,7 @@ jobs:
           working-directory: OpenSearch-Dashboards/plugins/security-analytics-dashboards-plugin
           command: yarn run cypress run
           wait-on: 'http://localhost:5601'
+          wait-on-timeout: 300
           browser: chrome
         env:
           CYPRESS_CACHE_FOLDER: ${{ matrix.cypress_cache_folder }}

cypress.json

@@ -5,6 +5,7 @@
   "requestTimeout": 300000,
   "responseTimeout": 300000,
   "baseUrl": "http://localhost:5601",
+  "retries": 1,
   "env": {
     "opensearch_url": "localhost:9200",
     "opensearch_dashboards": "http://localhost:5601",

cypress/fixtures/integration_tests/detector/create_dns_detector_data.json

@@ -27,19 +27,19 @@
   "triggers": [
     {
       "name": "DNS name alert",
-      "sev_levels": ["low"],
-      "tags": ["dns.low"],
+      "sev_levels": ["high"],
+      "tags": ["dns.high"],
       "actions": [
         {
           "id": "",
-          "name": "Triggered alert condition:  - Severity: 1 (Highest) - Threat detector: Cypress DNS Detector",
+          "name": "Triggered alert condition:  - Severity: 1 (Highest)  - Threat detector: Cypress DNS Detector",
           "destination_id": "",
           "subject_template": {
-            "source": "Triggered alert condition:  - Severity: 1 (Highest) - Threat detector: Cypress DNS Detector",
+            "source": "Triggered alert condition:  - Severity: 1 (Highest)  - Threat detector: Cypress DNS Detector",
             "lang": "mustache"
           },
           "message_template": {
-            "source": "Triggered alert condition: \nSeverity: 1 (Highest)\nThreat detector: Cypress DNS Detector\nDescription: Detects DNS names.\nDetector data sources:\n\tdns",
+            "source": "Triggered alert condition: \nSeverity: 1 (Highest) \nThreat detector: Cypress DNS Detector\nDescription: Detects DNS names.\nDetector data sources:\n\tdns",
             "lang": "mustache"
           },
           "throttle_enabled": false,

cypress/fixtures/integration_tests/detector/create_dns_detector_mappings_data.json

@@ -2,15 +2,15 @@
   "properties": {
     "dns-answers-type": {
       "type": "alias",
-      "path": "DnsAnswerType"
+      "path": "dns.answers.type"
     },
     "dns-question-name": {
       "type": "alias",
-      "path": "DnsQuestionName"
+      "path": "dns.question.name"
     },
     "dns-question-registered_domain": {
       "type": "alias",
-      "path": "DnsQuestionRegisteredDomain"
+      "path": "dns.question.registered_domain"
     }
   }
 }

cypress/fixtures/integration_tests/detector/create_usb_detector_data.json

@@ -27,19 +27,19 @@
   "triggers": [
     {
       "name": "USB plugged in alert",
-      "sev_levels": ["low"],
+      "sev_levels": ["high"],
       "tags": ["windows.usb"],
       "actions": [
         {
           "id": "",
           "name": "Triggered alert condition:  - Severity: 1 (Highest) - Threat detector: USB Detector",
           "destination_id": "",
           "subject_template": {
-            "source": "Triggered alert condition:  - Severity: 1 (Highest) - Threat detector: USB Detector",
+            "source": "Triggered alert condition:  - Severity: 1 (Highest)  - Threat detector: USB Detector",
             "lang": "mustache"
           },
           "message_template": {
-            "source": "Triggered alert condition: \nSeverity: 1 (Highest)\nThreat detector: USB Detector\nDescription: Detect USB plugged in.\nDetector data sources:\n\twindows",
+            "source": "Triggered alert condition: \nSeverity: 1 (Highest) \nThreat detector: USB Detector\nDescription: Detect USB plugged in.\nDetector data sources:\n\twindows",
             "lang": "mustache"
           },
           "throttle_enabled": false,

cypress/fixtures/integration_tests/detector/create_usb_detector_mappings_data.json

@@ -1,28 +1,12 @@
 {
   "properties": {
-    "event_uid": {
+    "winlog-event_id": {
       "type": "alias",
-      "path": "EventID"
+      "path": "winlog.event_id"
     },
-    "windows-event_data-CommandLine": {
+    "winlog-provider_name": {
       "type": "alias",
-      "path": "CommandLine"
-    },
-    "windows-hostname": {
-      "type": "alias",
-      "path": "HostName"
-    },
-    "windows-message": {
-      "type": "alias",
-      "path": "Message"
-    },
-    "windows-provider-name": {
-      "type": "alias",
-      "path": "Provider_Name"
-    },
-    "windows-servicename": {
-      "type": "alias",
-      "path": "ServiceName"
+      "path": "winlog.provider_name"
     }
   }
 }

cypress/fixtures/integration_tests/index/add_dns_index_data.json

@@ -1,5 +1,5 @@
 {
-  "DnsAnswerType": "QWE",
-  "DnsQuestionRegisteredDomain": "EC2AMAZ-EPWO7HKA",
-  "DnsQuestionName": "QWE"
+  "dns.answers.type": "AnswerType",
+  "dns.question.registered_domain": "EC2AMAZ-EPWO7HKA",
+  "dns.question.name": "QuestionName"
 }

cypress/fixtures/integration_tests/index/add_windows_index_data.json

@@ -1,39 +1,3 @@
 {
-  "EventTime": "2020-02-04T14:59:39.343541+00:00",
-  "HostName": "EC2AMAZ-EPO7HKA",
-  "Keywords": "9223372036854775808",
-  "SeverityValue": 2,
-  "Severity": "ERROR",
-  "EventID": 2003,
-  "SourceName": "Microsoft-Windows-Sysmon",
-  "ProviderGuid": "{5770385F-C22A-43E0-BF4C-06F5698FFBD9}",
-  "Version": 5,
-  "TaskValue": 22,
-  "OpcodeValue": 0,
-  "RecordNumber": 9532,
-  "ExecutionProcessID": 1996,
-  "ExecutionThreadID": 2616,
-  "Channel": "Microsoft-Windows-Sysmon/Operational",
-  "Domain": "NT AUTHORITY",
-  "AccountName": "SYSTEM",
-  "UserID": "S-1-5-18",
-  "AccountType": "User",
-  "Message": "Dns query:\r\nRuleName: \r\nUtcTime: 2020-02-04 14:59:38.349\r\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\r\nProcessId: 1904\r\nQueryName: EC2AMAZ-EPO7HKA\r\nQueryStatus: 0\r\nQueryResults: 172.31.46.38;\r\nImage: C:\\Program Files\\nxlog\\nxlog.exe",
-  "Category": "Dns query (rule: DnsQuery)",
-  "Opcode": "Info",
-  "UtcTime": "2020-02-04 14:59:38.349",
-  "ProcessGuid": "{b3c285a4-3cda-5dc0-0000-001077270b00}",
-  "ProcessId": "1904",
-  "QueryName": "EC2AMAZ-EPO7HKA",
-  "QueryStatus": "0",
-  "QueryResults": "172.31.46.38;",
-  "Image": "C:\\Program Files\\nxlog\\regsvr32.exe",
-  "EventReceivedTime": "2020-02-04T14:59:40.780905+00:00",
-  "SourceModuleName": "in",
-  "SourceModuleType": "im_msvistalog",
-  "CommandLine": "eachtest",
-  "Initiated": "true",
-  "Provider_Name": "Service_ws_Control_ws_Manager",
-  "TargetObject": "\\SOFTWARE\\Microsoft\\Office\\Outlook\\Security",
-  "EventType": "SetValue"
+  "winlog.event_id": "2003"
 }

cypress/fixtures/integration_tests/index/create_dns_settings.json

@@ -1,13 +1,13 @@
 {
   "mappings": {
     "properties": {
-      "DnsAnswerType": {
+      "dns.answers.type": {
         "type": "text"
       },
-      "DnsQuestionRegisteredDomain": {
+      "dns.question.name": {
         "type": "text"
       },
-      "DnsQuestionName": {
+      "dns.question.registered_domain": {
         "type": "text"
       }
     }

cypress/fixtures/integration_tests/index/create_windows_settings.json

@@ -1,22 +1,10 @@
 {
   "mappings": {
     "properties": {
-      "CommandLine": {
-        "type": "text"
-      },
-      "EventID": {
+      "winlog.event_id": {
         "type": "integer"
       },
-      "HostName": {
-        "type": "text"
-      },
-      "Message": {
-        "type": "text"
-      },
-      "Provider_Name": {
-        "type": "text"
-      },
-      "ServiceName": {
+      "winlog.provider_name": {
         "type": "text"
       }
     }

cypress/fixtures/integration_tests/rule/create_dns_rule.json → cypress/fixtures/integration_tests/rule/create_dns_rule_with_name_selection.json

@@ -12,12 +12,12 @@
   ],
   "tags": [
     {
-      "value": "dns.low"
+      "value": "dns.high"
     }
   ],
   "log_source": "",
-  "detection": "selection:\n  query:\n    - QWE\n    - ASD\n    - YXC\ncondition: selection",
-  "level": "low",
+  "detection": "selection:\n  dns-question-name:\n    - QuestionName\ncondition: selection",
+  "level": "high",
   "false_positives": [
     {
       "value": ""

cypress/fixtures/integration_tests/rule/create_dns_rule_with_type_selection.json

@@ -0,0 +1,26 @@
+{
+  "id": "25b9c01c-350d-4b95-bed1-836d04a4f325",
+  "category": "dns",
+  "title": "Cypress DNS Type Rule",
+  "description": "Detects DNS type as QWE",
+  "status": "experimental",
+  "author": "Cypress Tests",
+  "references": [
+    {
+      "value": ""
+    }
+  ],
+  "tags": [
+    {
+      "value": "dns.high"
+    }
+  ],
+  "log_source": "",
+  "detection": "selection:\n  dns-answers-type:\n    - AnswerType\ncondition: selection",
+  "level": "high",
+  "false_positives": [
+    {
+      "value": ""
+    }
+  ]
+}

cypress/fixtures/integration_tests/rule/create_network_rule.json

@@ -12,12 +12,12 @@
   ],
   "tags": [
     {
-      "value": "network.low"
+      "value": "network.high"
     }
   ],
   "log_source": "",
   "detection": "selection:\n  keywords:\n    - erase\n    - delete\n    - YXC\ncondition: selection",
-  "level": "low",
+  "level": "high",
   "false_positives": [
     {
       "value": ""

cypress/fixtures/integration_tests/rule/create_windows_usb_rule.json

@@ -16,8 +16,8 @@
     }
   ],
   "log_source": "",
-  "detection": "selection:\n  EventID:\n    - 2003\n    - 2100\n    - 2102\ncondition: selection",
-  "level": "low",
+  "detection": "selection:\n  winlog-event_id:\n    - 2003\n    - 2100\n    - 2102\ncondition: selection",
+  "level": "high",
   "false_positives": [
     {
       "value": ""

cypress/fixtures/integration_tests/rule/sample_dns_field_mappings.json

@@ -0,0 +1,5 @@
+{
+  "dns-question-registered_domain": "dns.question.registered_domain",
+  "dns-question-name": "dns.question.name",
+  "dns-answers-type": "dns.answers.type"
+}

cypress/fixtures/sample_alias_mappings.json

@@ -1,16 +1,8 @@
 {
   "properties": {
-    "source_ip": {
+    "winlog-event_id": {
       "type": "alias",
-      "path": "src_ip"
-    },
-    "windows-event_data-CommandLine": {
-      "path": "CommandLine",
-      "type": "alias"
-    },
-    "event_uid": {
-      "path": "EventID",
-      "type": "alias"
+      "path": "winlog.event_id"
     }
   }
 }

cypress/fixtures/sample_detector.json

@@ -20,26 +20,30 @@
             "id": "1a4bd6e3-4c6e-405d-a9a3-53a116e341d4"
           }
         ],
-        "custom_rules": []
+        "custom_rules": [
+          {
+            "id": ""
+          }
+        ]
       }
     }
   ],
   "triggers": [
     {
       "name": "sample_alert_condition",
-      "sev_levels": [],
+      "sev_levels": ["high"],
       "tags": [],
       "actions": [
         {
           "id": "",
           "name": "Triggered alert condition:  - Severity: 1 (Highest) - Threat detector: sample_detector",
           "destination_id": "",
           "subject_template": {
-            "source": "Triggered alert condition:  - Severity: 1 (Highest) - Threat detector: sample_detector",
+            "source": "Triggered alert condition:  - Severity: 1 (Highest)  - Threat detector: sample_detector",
             "lang": "mustache"
           },
           "message_template": {
-            "source": "Triggered alert condition: \nSeverity: 1 (Highest)\nThreat detector: sample_detector\nDescription: Description for sample_detector.\nDetector data sources:\n\twindows",
+            "source": "Triggered alert condition: \nSeverity: 1 (Highest) \nThreat detector: sample_detector\nDescription: Description for sample_detector.\nDetector data sources:\n\twindows",
             "lang": "mustache"
           },
           "throttle_enabled": false,
@@ -51,7 +55,7 @@
       ],
       "types": ["windows"],
       "severity": "4",
-      "ids": ["1a4bd6e3-4c6e-405d-a9a3-53a116e341d4"]
+      "ids": []
     }
   ]
 }

cypress/fixtures/sample_dns_index_settings.json

@@ -0,0 +1,21 @@
+{
+  "mappings": {
+    "properties": {
+      "dns.question.name": {
+        "type": "text"
+      },
+      "dns.answers.type": {
+        "type": "text"
+      },
+      "dns.question.registered_domain": {
+        "type": "text"
+      }
+    }
+  },
+  "settings": {
+    "index": {
+      "number_of_shards": "1",
+      "number_of_replicas": "1"
+    }
+  }
+}