Remove custom MIB compile logic (#495)

* SNMP: Warn if OID symbol is skipped

* Fix ifPhysAddress in IEC104 template that failed to parse

* SNMP: Use PySNMP on-demand compiler rather than custom MIB compile logic

* Remove --raw_mib argument, always read from protocol directory instead

* Rename --mibpaths to more adequate --mibcache

* Remove outdated SNMP-related dependencies

.travis.yml

@@ -23,8 +23,6 @@ addons:
     - unzip
     - python-dev
     - libevent-dev
-    - libsmi2ldbl
-    - snmp-mibs-downloader
     - libxml2
     - libxml2-dev
     - libxml2-utils

bin/conpot

@@ -163,17 +163,8 @@ def main():
                         help="The logfile to use",
                         default="conpot.log"
                         )
-    parser.add_argument("-a", "--raw_mib",
-                        help="Path to raw MIB files."
-                             "(will automatically get compiled by build-pysnmp-mib)",
-                        action='append',
-                        default=[]
-                        )
-    parser.add_argument("-m", "--mibpaths",
-                        action='append',
-                        help="Path to compiled PySNMP MIB files."
-                             "(must be compiled with build-pysnmp-mib)",
-                        default=[]
+    parser.add_argument("-m", "--mibcache",
+                        help="Cache directory for compiled PySNMP MIB files"
                         )
     parser.add_argument("--temp_dir",
                         help="Directory where all conpot vfs related files would be kept.",

conpot/protocols/snmp/command_responder.py

@@ -3,11 +3,12 @@
 
 import logging
 
+from pysmi.reader import FileReader, HttpReader
 from pysnmp.entity import config
 from pysnmp.entity.rfc3413 import context
 from pysnmp.carrier.asynsock.dgram import udp
 from pysnmp.entity import engine
-from pysnmp.smi import builder
+from pysnmp.smi.compiler import addMibCompiler
 import gevent
 
 from conpot.protocols.snmp import conpot_cmdrsp
@@ -45,7 +46,7 @@ def getTimerResolution(self):
 
 
 class CommandResponder(object):
-    def __init__(self, host, port, mibpaths):
+    def __init__(self, host, port, raw_mibs, compiled_mibs):
 
         self.oid_mapping = {}
         self.databus_mediator = DatabusMediator(self.oid_mapping)
@@ -54,13 +55,11 @@ def __init__(self, host, port, mibpaths):
         # Create SNMP engine
         self.snmpEngine = engine.SnmpEngine()
 
-        # path to custom mibs
-        mibBuilder = self.snmpEngine.msgAndPduDsp.mibInstrumController.mibBuilder
-        mibSources = mibBuilder.getMibSources()
-
-        for mibpath in mibpaths:
-            mibSources += (builder.DirMibSource(mibpath),)
-        mibBuilder.setMibSources(*mibSources)
+        # Configure SNMP compiler
+        mib_builder = self.snmpEngine.getMibBuilder()
+        addMibCompiler(mib_builder, destination=compiled_mibs)
+        mib_builder.getMibCompiler().addSources(FileReader(raw_mibs))
+        mib_builder.getMibCompiler().addSources(HttpReader('mibs.snmplabs.com', 80, '/asn1/@mib@'))
 
         # Transport setup
         udp_sock = gevent.socket.socket(gevent.socket.AF_INET, gevent.socket.SOCK_DGRAM)
@@ -139,18 +138,14 @@ def register(self, mibname, symbolname, instance, value, profile_map_name):
             logger.debug('Registered: OID %s Instance %s ASN.1 (%s @ %s) value %s dynrsp.', s.name, instance, s.label, mibname, value)
 
         else:
-            logger.debug('Skipped: OID for symbol %s not found in MIB %s', symbolname, mibname)
+            logger.warning('Skipped: OID for symbol %s not found in MIB %s', symbolname, mibname)
 
     def _get_mibSymbol(self, mibname, symbolname):
         modules = self.snmpEngine.msgAndPduDsp.mibInstrumController.mibBuilder.mibSymbols
         if mibname in modules:
             if symbolname in modules[mibname]:
                 return modules[mibname][symbolname]
 
-    def has_mib(self, mibname):
-        modules = self.snmpEngine.msgAndPduDsp.mibInstrumController.mibBuilder.mibSymbols
-        return mibname in modules
-
     def serve_forever(self):
         self.snmpEngine.transportDispatcher.serve_forever()
 

conpot/protocols/snmp/snmp_server.py

@@ -16,16 +16,13 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
 import logging
-import tempfile
-import shutil
 import os
 
 from lxml import etree
+
+import conpot.core as conpot_core
 from conpot.core.protocol_wrapper import conpot_protocol
 from conpot.protocols.snmp.command_responder import CommandResponder
-from conpot.protocols.snmp.build_pysnmp_mib_wrapper import find_mibs, compile_mib
-import conpot.core as conpot_core
-
 
 logger = logging.getLogger()
 
@@ -42,15 +39,8 @@ def __init__(self, template, template_directory, args):
         self.dom = etree.parse(template)
         self.cmd_responder = None
 
-        if args.mibpaths:
-            self.compiled_mibs = args.mibpaths
-        else:
-            self.compiled_mibs = [os.path.join(template_directory, 'snmp', 'mibs_compiled')]
-
-        if args.raw_mib:
-            self.raw_mibs = args.raw_mib
-        else:
-            self.raw_mibs = [os.path.join(template_directory, 'snmp', 'mibs_raw')]
+        self.compiled_mibs = args.mibcache
+        self.raw_mibs = os.path.join(template_directory, 'snmp', 'mibs')
 
     def xml_general_config(self, dom):
         snmp_config = dom.xpath('//snmp/config/*')
@@ -81,46 +71,35 @@ def xml_general_config(self, dom):
                     elif entity.attrib['command'].lower() == 'bulk':
                         self.cmd_responder.resp_app_bulk.threshold = self.config_sanitize_threshold(entity.text)
 
-    def xml_mib_config(self, dom, mibpaths, rawmibs_dirs):
-        try:
-            mibs = dom.xpath('//snmp/mibs/*')
-            tmp_mib_dir = tempfile.mkdtemp()
-            mibpaths.append(tmp_mib_dir)
-            available_mibs = find_mibs(rawmibs_dirs)
-
-            databus = conpot_core.get_databus()
-            # parse mibs and oid tables
-            for mib in mibs:
-                mib_name = mib.attrib['name']
-                # compile the mib file if it is found and not already loaded.
-                if mib_name in available_mibs and not self.cmd_responder.has_mib(mib_name):
-                    compile_mib(mib_name, tmp_mib_dir)
-                for symbol in mib:
-                    symbol_name = symbol.attrib['name']
-
-                    # retrieve instance from template
-                    if 'instance' in symbol.attrib:
-                        # convert instance to (int-)tuple
-                        symbol_instance = symbol.attrib['instance'].split('.')
-                        symbol_instance = tuple(map(int, symbol_instance))
-                    else:
-                        # use default instance (0)
-                        symbol_instance = (0,)
-
-
-                    # retrieve value from databus
-                    value = databus.get_value(symbol.xpath('./value/text()')[0])
-                    profile_map_name = symbol.xpath('./value/text()')[0]
-
-                    # register this MIB instance to the command responder
-                    self.cmd_responder.register(mib_name,
-                                                symbol_name,
-                                                symbol_instance,
-                                                value,
-                                                profile_map_name)
-        finally:
-            # cleanup compiled mib files
-            shutil.rmtree(tmp_mib_dir)
+    def xml_mib_config(self):
+        mibs = self.dom.xpath('//snmp/mibs/*')
+
+        # parse mibs and oid tables
+        for mib in mibs:
+            mib_name = mib.attrib['name']
+
+            for symbol in mib:
+                symbol_name = symbol.attrib['name']
+
+                # retrieve instance from template
+                if 'instance' in symbol.attrib:
+                    # convert instance to (int-)tuple
+                    symbol_instance = symbol.attrib['instance'].split('.')
+                    symbol_instance = tuple(map(int, symbol_instance))
+                else:
+                    # use default instance (0)
+                    symbol_instance = (0,)
+
+                # retrieve value from databus
+                value = conpot_core.get_databus().get_value(symbol.xpath('./value/text()')[0])
+                profile_map_name = symbol.xpath('./value/text()')[0]
+
+                # register this MIB instance to the command responder
+                self.cmd_responder.register(mib_name,
+                                            symbol_name,
+                                            symbol_instance,
+                                            value,
+                                            profile_map_name)
 
     def config_sanitize_tarpit(self, value):
 
@@ -178,9 +157,9 @@ def config_sanitize_threshold(self, value):
             return '0;0'
 
     def start(self, host, port):
-        self.cmd_responder = CommandResponder(host, port, self.compiled_mibs)
+        self.cmd_responder = CommandResponder(host, port, self.raw_mibs, self.compiled_mibs)
         self.xml_general_config(self.dom)
-        self.xml_mib_config(self.dom, self.compiled_mibs, self.raw_mibs)
+        self.xml_mib_config()
 
         logger.info('SNMP server started on: %s', (host, self.get_port()))
         self.cmd_responder.serve_forever()

conpot/templates/IEC104/template.xml

@@ -70,7 +70,7 @@
                 <value type="value">100000000</value>
             </key>
             <key name="ifPhysAddress">
-                <value type="value">"\x00\x0e\x8c\x29\xc5\x1a"</value>
+                <value type="value">"0x000e8c29c51a"</value>
             </key>
             <key name="ifAdminStatus">
                 <value type="value">1</value>

conpot/tests/test_iec104_server.py

@@ -35,8 +35,7 @@ def setUp(self):
 
         self.databus = conpot_core.get_databus()
         self.databus.initialize(template)
-        args = namedtuple('FakeArgs', 'mibpaths raw_mib')
-        self.iec104_inst = IEC104_server.IEC104Server(iec104_template, 'none', args)
+        self.iec104_inst = IEC104_server.IEC104Server(iec104_template, 'none', None)
         self.iec104_server = StreamServer(('127.0.0.1', 2404), self.iec104_inst.handle)
         self.iec104_server.start()
 

conpot/tests/test_modbus_server.py

@@ -48,8 +48,7 @@ def setUp(self):
 
         self.databus = conpot_core.get_databus()
         self.databus.initialize(template)
-        args = namedtuple('FakeArgs', 'mibpaths raw_mib')
-        self.modbus = modbus_server.ModbusServer(modbus_template, 'none', args, timeout=2)
+        self.modbus = modbus_server.ModbusServer(modbus_template, 'none', None, timeout=2)
         self.modbus_server = StreamServer(('127.0.0.1', 0), self.modbus.handle)
         self.modbus_server.start()