Feature artifact extractor

conf/lambda.json

@@ -50,6 +50,29 @@
       "subnet_ids": []
     }
   },
+  "artifact_extractor_config": {
+    "concurrency_limit": 10,
+    "enabled": false,
+    "firehose_buffer_size": 128,
+    "firehose_buffer_interval": 900,
+    "log_level": "info",
+    "memory": 128,
+    "metric_alarms": {
+      "errors": {
+        "enabled": true,
+        "evaluation_periods": 1,
+        "period_secs": 300,
+        "threshold": 0
+      },
+      "throttles": {
+        "enabled": true,
+        "evaluation_periods": 1,
+        "period_secs": 300,
+        "threshold": 0
+      }
+    },
+    "timeout": 300
+  },
   "athena_partitioner_config": {
     "concurrency_limit": 10,
     "memory": 128,

conf/schemas/carbonblack.json

@@ -1043,7 +1043,29 @@
         "uid",
         "username",
         "sha256"
-      ]
+      ],
+      "normalization": {
+        "command": [
+          {
+            "path": ["command_line"],
+            "function": "Command line"
+          }
+        ],
+        "path": [
+          {
+            "path": ["path"],
+            "function": "Process path"
+          },
+          {
+            "path": ["parent_path"],
+            "function": "Process parent path"
+          },
+          {
+            "path": ["process_path"],
+            "function": "Process parent path"
+          }
+        ]
+      }
     }
   },
   "carbonblack:ingress.event.regmod": {

conf/schemas/cloudwatch.json

@@ -88,7 +88,55 @@
       "time": "string",
       "version": "string"
     },
-    "parser": "json"
+    "parser": "json",
+    "configuration": {
+      "normalization": {
+        "event_name": ["detail", "eventName"],
+        "account": [
+          {
+            "path": [
+              "account"
+            ],
+            "function": "Destination account ID"
+          },
+          {
+            "path": [
+              "detail",
+              "userIdentity",
+              "principalId"
+            ],
+            "function": "Source account ID"
+          }
+        ],
+        "ip_address": [
+          {
+            "path": [
+              "detail",
+              "sourceIPAddress"
+            ],
+            "function": "Source IP addresses"
+          }
+        ],
+        "user_agent": [
+          "detail",
+          "userAgent"
+        ],
+        "user_identity": [
+          {
+            "path": ["detail", "userIdentity", "type"],
+            "function": "User identity type"
+          },
+          {
+            "path": ["detail", "userIdentity", "arn"],
+            "function": "User identity arn"
+          },
+          {
+            "path": ["detail", "userIdentity", "userName"],
+            "function": "User identity username"
+          }
+        ]
+      }
+    }
   },
   "cloudwatch:flow_logs": {
     "schema": {

conf/schemas/osquery.json

@@ -48,7 +48,21 @@
         "log_type",
         "logNumericsAsNumbers",
         "numerics"
-      ]
+      ],
+      "normalization": {
+        "command": [
+          {
+            "path": ["columns", "command"],
+            "function": "Command line from shell history"
+          }
+        ],
+        "file_path": [
+          {
+            "path": ["columns", "history_file"],
+            "function": "Shell history file path"
+          }
+        ]
+      }
     }
   },
   "osquery:snapshot": {

docs/source/index.rst

@@ -82,6 +82,7 @@ Table of Contents
    rule-promotion
    historical-search
    scheduled-queries
+   normalization
    conf-schemas-examples
    troubleshooting
    faq