RFC: XML to toml transition for templates

bin/conpot

@@ -37,10 +37,13 @@ import conpot.core as conpot_core
 from conpot import protocols
 from conpot.core.log_worker import LogWorker
 from conpot.protocols.proxy.proxy import Proxy
+import conpot.protocols.schemas
 from conpot.utils import ext_ip
 from conpot.utils.greenlet import spawn_startable_greenlet
 from conpot.utils import mac_addr
 from conpot.utils.networking import fix_sslwrap
+from conpot.templates import validate as template_validate
+from conpot.templates import parse as template_parse
 
 logger = logging.getLogger()
 package_directory = os.path.dirname(os.path.abspath(conpot.__file__))
@@ -131,17 +134,6 @@ def drop_privileges(uid_name=None, gid_name=None):
         )
     )
 
-
-def validate_template(xml_file, xsd_file):
-    xml_schema = etree.parse(xsd_file)
-    xsd = etree.XMLSchema(xml_schema)
-    xml = etree.parse(xml_file)
-    xsd.validate(xml)
-    if xsd.error_log:
-        logger.error("Error parsing XML template: {}".format(xsd.error_log))
-        sys.exit(1)
-
-
 def main():
     logo()
 
@@ -188,18 +180,18 @@ def main():
 
     setup_logging(args.logfile, args.verbose)
 
-    core_interface.config = ConfigParser(os.environ)
-    config = core_interface.config
+    config = ConfigParser(os.environ)
+    core_interface.config = config
 
     if os.getuid() == 0:
         if not args.force:
             logger.critical(
-                "Can't start conpot with root. Please ref user docs for more info."
+                "Can't start conpot as root. Please ref user docs for more info."
             )
             sys.exit(3)
         else:
             logger.warning(
-                "Running conpot with root. Running conpot with root isn't recommended. "
+                "Running conpot as root. Running conpot as root isn't recommended."
             )
 
     if os.getuid() == 0:
@@ -225,10 +217,7 @@ def main():
     else:
         # sanity check: see that both config and template arguments are provided - else exit
         if not (args.config and args.template):
-            print(
-                "Invalid arguments supplied. Please check that you pass both template and config arguments before"
-                " running Conpot"
-            )
+            print("Invalid arguments supplied. Please check that you pass both template and config arguments before running Conpot")
             sys.exit(3)
         try:
             if not os.path.isfile(os.path.join(package_directory, args.config)):
@@ -294,11 +283,11 @@ def main():
         sys.exit(0)
 
     # Custom template supplied
-    if os.path.exists(os.path.join(args.template, "template.xml")):
+    if os.path.exists(os.path.join(args.template, "template.toml")):
         root_template_directory = args.template
-    # Check if the template name can be in the default templates directory
+    # Check if the template name can be found in the default templates directory
     elif os.path.isfile(
-        os.path.join(package_directory, "templates", args.template, "template.xml")
+        os.path.join(package_directory, "templates", args.template, "template.toml")
     ):
         root_template_directory = os.path.join(
             package_directory, "templates", args.template
@@ -317,18 +306,21 @@ def main():
 
     servers = list()
 
-    template_base = os.path.join(root_template_directory, "template.xml")
+    template_base = os.path.join(root_template_directory, "template.toml")
+    template = dict()
     if os.path.isfile(template_base):
-        validate_template(
-            template_base, os.path.join(package_directory, "template.xsd")
-        )
-        dom_base = etree.parse(template_base)
+        try:
+            template = template_parse.parse_toml_config(template_base)
+            template_validate.validate_toml_template(template, template_validate.base_schema)
+        except schema.SchemaError as se:
+            logger.error("Template validation error: {}".format(se))
+            sys.exit(1)
     else:
-        logger.error("Could not access template configuration")
+        logger.error("Template not found")
         sys.exit(1)
 
     session_manager = conpot_core.get_sessionManager()
-    conpot_core.get_databus().initialize(template_base)
+    conpot_core.get_databus().initialize(template)
 
     # initialize the virtual file system
     fs_url = config.get("virtual_file_system", "fs_url")
@@ -376,46 +368,32 @@ def main():
     if pid == 0:
         for protocol_name, server_class in protocols.name_mapping.items():
             protocol_template = os.path.join(
-                root_template_directory, protocol_name, "{0}.xml".format(protocol_name)
+                root_template_directory, "{0}.toml".format(protocol_name)
             )
             if os.path.isfile(protocol_template):
-                xsd_file = os.path.join(
-                    package_directory,
-                    "protocols",
-                    protocol_name,
-                    "{0}.xsd".format(protocol_name),
-                )
-                validate_template(protocol_template, xsd_file)
-                dom_protocol = etree.parse(protocol_template)
-                if dom_protocol.xpath("//{0}".format(protocol_name)):
-                    if ast.literal_eval(
-                        dom_protocol.xpath("//{0}/@enabled".format(protocol_name))[0]
-                    ):
-                        host = dom_protocol.xpath("//{0}/@host".format(protocol_name))[
-                            0
-                        ]
-                        # -- > Are we running on testing config?
-                        if "testing.cfg" in args.config:
-                            if "127." not in host:
-                                if not args.force:
-                                    logger.error(
-                                        "To run conpot on a non local interface, please specify -f option"
-                                    )
-                                    sys.exit(1)
-                        port = ast.literal_eval(
-                            dom_protocol.xpath("//{0}/@port".format(protocol_name))[0]
-                        )
-                        server = server_class(
-                            protocol_template, root_template_directory, args
-                        )
-                        greenlet = spawn_startable_greenlet(server, host, port)
-                        greenlet.link_exception(on_unhandled_greenlet_exception)
-                        servers.append((server, greenlet))
-                        logger.info(
-                            "Found and enabled {} protocol.".format(
-                                protocol_name, server
-                            )
+                schema = getattr(conpot.protocols.schemas, protocol_name)
+                protocol = template_parse.parse_toml_config(protocol_template)
+                template_validate.validate_toml_template(protocol, schema)
+                if protocol["tftp"]["enabled"]:
+                    # -- > Are we running on testing config?
+                    if "testing.cfg" in args.config:
+                        if "127." not in protocol["tftp"]["host"]:
+                            if not args.force:
+                                logger.error(
+                                    "To run conpot on a non local interface, please specify -f option"
+                                )
+                                sys.exit(1)
+                    server = server_class(
+                        protocol[protocol_name], root_template_directory, args
+                    )
+                    greenlet = spawn_startable_greenlet(server, protocol["tftp"]["host"], protocol["tftp"]["port"])
+                    greenlet.link_exception(on_unhandled_greenlet_exception)
+                    servers.append((server, greenlet))
+                    logger.info(
+                        "Found and enabled {} protocol.".format(
+                            protocol_name, server
                         )
+                    )
                 else:
                     logger.info(
                         "{} available but disabled by configuration.".format(
@@ -429,7 +407,7 @@ def main():
                     )
                 )
 
-        log_worker = LogWorker(config, dom_base, session_manager, public_ip)
+        log_worker = LogWorker(config, template, session_manager, public_ip)
         greenlet = spawn_startable_greenlet(log_worker)
         greenlet.link_exception(on_unhandled_greenlet_exception)
         servers.append((log_worker, greenlet))
@@ -440,7 +418,7 @@ def main():
             xsd_file = os.path.join(
                 os.path.dirname(inspect.getfile(Proxy)), "proxy.xsd"
             )
-            validate_template(template_proxy, xsd_file)
+            template_validate.validate_xml_template(template_proxy, xsd_file)
             dom_proxy = etree.parse(template_proxy)
             if dom_proxy.xpath("//proxies"):
                 if ast.literal_eval(dom_proxy.xpath("//proxies/@enabled")[0]):

conpot/core/databus.py

@@ -23,7 +23,6 @@
 
 import gevent
 import gevent.event
-from lxml import etree
 
 
 logger = logging.getLogger(__name__)
@@ -75,33 +74,22 @@ def observe_value(self, key, callback):
             self._observer_map[key] = []
         self._observer_map[key].append(callback)
 
-    def initialize(self, config_file):
+    def initialize(self, template):
         self.reset()
         assert self.initialized.isSet() is False
-        logger.debug("Initializing databus using %s.", config_file)
-        dom = etree.parse(config_file)
-        entries = dom.xpath("//core/databus/key_value_mappings/*")
-        for entry in entries:
-            key = entry.attrib["name"]
-            value = entry.xpath("./value/text()")[0].strip()
-            value_type = str(entry.xpath("./value/@type")[0])
+        logger.debug("Initializing databus")
+        for key, value in template["core"]["databus"]["key_value_mappings"].items():
             assert key not in self._data
-            logging.debug("Initializing %s with %s as a %s.", key, value, value_type)
-            if value_type == "value":
-                self.set_value(key, eval(value))
-            elif value_type == "function":
+            logging.debug("Initializing %s with %s", key, value)
+            if value.startswith("conpot"):
                 namespace, _classname = value.rsplit(".", 1)
-                params = entry.xpath("./value/@param")
                 module = __import__(namespace, fromlist=[_classname])
                 _class = getattr(module, _classname)
-                if len(params) > 0:
-                    # eval param to list
-                    params = eval(params[0])
-                    self.set_value(key, _class(*(tuple(params))))
-                else:
-                    self.set_value(key, _class())
+                # No params supported
+                self.set_value(key, _class())
             else:
-                raise Exception("Unknown value type: {0}".format(value_type))
+                # TODO (lr): eval value
+                self.set_value(key, value)
         self.initialized.set()
 
     def reset(self):

conpot/core/log_worker.py

@@ -35,7 +35,7 @@
 
 
 class LogWorker(object):
-    def __init__(self, config, dom, session_manager, public_ip):
+    def __init__(self, config, template, session_manager, public_ip):
         self.config = config
         self.log_queue = session_manager.log_queue
         self.session_manager = session_manager
@@ -78,7 +78,7 @@ def __init__(self, config, dom, session_manager, public_ip):
 
         if config.getboolean("taxii", "enabled"):
             # TODO: support for certificates
-            self.taxii_logger = TaxiiLogger(config, dom)
+            self.taxii_logger = TaxiiLogger(config, template)
 
         self.enabled = True
 

conpot/core/loggers/stix_transform.py

@@ -16,7 +16,6 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
 import json
-import ast
 import textwrap
 
 from mixbox import idgen
@@ -49,24 +48,13 @@
 
 
 class StixTransformer(object):
-    def __init__(self, config, dom):
+    def __init__(self, config, template):
         self.protocol_to_port_mapping = dict(
             modbus=502,
             snmp=161,
             http=80,
             s7comm=102,
         )
-        port_path_list = [
-            "//conpot_template/protocols/" + x + "/@port"
-            for x in list(self.protocol_to_port_mapping.keys())
-        ]
-        for port_path in port_path_list:
-            try:
-                protocol_port = ast.literal_eval(dom.xpath(port_path)[0])
-                protocol_name = port_path.rsplit("/", 2)[1]
-                self.protocol_to_port_mapping[protocol_name] = protocol_port
-            except IndexError:
-                continue
         conpot_namespace = Namespace(CONPOT_NAMESPACE_URL, CONPOT_NAMESPACE, "")
         idgen.set_id_namespace(conpot_namespace)
 

conpot/core/loggers/taxii_log.py

@@ -28,15 +28,15 @@
 
 
 class TaxiiLogger(object):
-    def __init__(self, config, dom):
+    def __init__(self, config, template):
         self.host = config.get("taxii", "host")
         self.port = config.getint("taxii", "port")
         self.inbox_path = config.get("taxii", "inbox_path")
         self.use_https = config.getboolean("taxii", "use_https")
 
         self.client = HttpClient()
         self.client.setProxy("noproxy")
-        self.stix_transformer = StixTransformer(config, dom)
+        self.stix_transformer = StixTransformer(config, template)
 
     def log(self, event):
         # converts from conpot log format to STIX compatible xml

conpot/protocols/schemas.py

@@ -0,0 +1,15 @@
+from schema import Schema, And
+
+bacnet = Schema({})
+tftp = Schema(
+    {
+        "tftp": {
+            "enabled": bool,
+            "host": str,
+            "port": int,
+            "tftp_root_path": str,
+            "add_src": str,
+            "data_fs_subdir": str,
+        }
+    }
+)

conpot/protocols/tftp/tftp_server.py

@@ -20,14 +20,16 @@
 
 import gevent
 import os
-from lxml import etree
-from conpot.protocols.tftp import tftp_handler
+import logging
+
+from tftpy import TftpException, TftpTimeout
 from gevent.server import DatagramServer
+
 import conpot.core as conpot_core
+from conpot.protocols.tftp import tftp_handler
 from conpot.core.protocol_wrapper import conpot_protocol
 from conpot.utils.networking import get_interface_ip
-from tftpy import TftpException, TftpTimeout
-import logging
+
 
 logger = logging.getLogger(__name__)
 
@@ -54,13 +56,12 @@ def __init__(self, template, template_directory, args, timeout=5):
         logger.debug("TFTP server initialized.")
 
     def _init_vfs(self, template):
-        dom = etree.parse(template)
-        self.root_path = dom.xpath("//tftp/tftp_root_path/text()")[0].lower()
-        if len(dom.xpath("//tftp/add_src/text()")) == 0:
+        self.root_path = template["tftp_root_path"].lower()
+        if len(template["add_src"]) == 0:
             self.add_src = None
         else:
-            self.add_src = dom.xpath("//tftp/add_src/text()")[0].lower()
-        self.data_fs_subdir = dom.xpath("//tftp/data_fs_subdir/text()")[0].lower()
+            self.add_src = template["add_src"].lower()
+        self.data_fs_subdir = template["data_fs_subdir"].lower()
         # Create a file system.
         self.vfs, self.data_fs = conpot_core.add_protocol(
             protocol_name="tftp",