New package sample_parser.

Custom elasticsearch template ip address field.

conf/spamscope.example.yml

@@ -111,8 +111,9 @@ attachments:
 
         # File extensions to submit to thug
         extensions:
-            - .js
             - .html
+            - .js
+            - .jse
 
         # More details:
         # http://buffer.github.io/thug/doc/usage.html#basic-usage

conf/templates/spamscope.json

@@ -86,6 +86,15 @@
             "match": "path_mail"
           }
         },
+        {
+          "ipaddress": {
+            "mapping": {
+              "type": "ip"
+            },
+            "match_pattern": "regex",
+            "match": "(^|.*\\.)(sender_ip|srcip|http_iv_remote_address|clientip|syslog_host|ip)$"
+          }
+        },
         {
           "all_not_analyzed": {
             "mapping": {

src/bolts/attachments.py

@@ -54,8 +54,8 @@ def _load_lists(self):
                     raise ImproperlyConfigured(
                         "Keywords content types \
                         details list '{}' not valid".format(k))
-                keywords = [i.lower() for i in keywords]
-                self._tika_valid_content_types |= set(keywords)
+                keywords = {i.lower() for i in keywords}
+                self._tika_valid_content_types |= keywords
                 self.log("Content types Tika '{}' loaded".format(k))
 
         # Load content types for blacklist
@@ -67,8 +67,8 @@ def _load_lists(self):
                 raise ImproperlyConfigured(
                     "Keywords content types blacklist \
                     list '{}' not valid".format(k))
-            keywords = [i.lower() for i in keywords]
-            self._cont_type_bl |= set(keywords)
+            keywords = {i.lower() for i in keywords}
+            self._cont_type_bl |= keywords
             self.log("Content types blacklist '{}' loaded".format(k))
 
     def process_tick(self, freq):

src/modules/sample_parser/__init__.py

@@ -0,0 +1,17 @@
+"""
+Copyright 2016 Fedele Mantuano (https://twitter.com/fedelemantuano)
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from .sample_parser import SampleParser

src/modules/sample_parser/abstract_processing.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Copyright 2016 Fedele Mantuano (https://twitter.com/fedelemantuano)
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from abc import ABCMeta, abstractmethod
+from .exceptions import InvalidAttachment
+
+
+class AbstractProcessing(object):
+
+    __metaclass__ = ABCMeta
+
+    def __init__(self, **kwargs):
+        self._kwargs = kwargs
+        self._check_arguments()
+
+    def __getattr__(self, name):
+        try:
+            return self._kwargs[name]
+        except KeyError:
+            msg = "'{0}' object has no attribute '{1}'"
+            raise AttributeError(msg.format(type(self).__name__, name))
+
+    def __setattr__(self, name, value):
+        super(AbstractProcessing, self).__setattr__(name, value)
+
+    @abstractmethod
+    def process(self, attachments):
+        if not isinstance(attachments, dict):
+            raise InvalidAttachment("Attachment result is not a dict")
+
+    @abstractmethod
+    def _check_arguments(self):
+        pass

src/modules/sample_parser/exceptions.py

@@ -0,0 +1,42 @@
+"""
+Copyright 2016 Fedele Mantuano (https://twitter.com/fedelemantuano)
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+__all__ = ["Base64Error", "TempIOError", "InvalidAttachment",
+           "VirusTotalApiKeyInvalid", "InvalidContentTypes", "MissingArgument"]
+
+
+class Base64Error(ValueError):
+    pass
+
+
+class TempIOError(Exception):
+    pass
+
+
+class InvalidAttachment(ValueError):
+    pass
+
+
+class VirusTotalApiKeyInvalid(ValueError):
+    pass
+
+
+class InvalidContentTypes(ValueError):
+    pass
+
+
+class MissingArgument(ValueError):
+    pass