mibcopy.py tool implemented (#28)

Also:

* REVISIONS handled in MODULE-IDENTITY
* MIB `revision` field added to MIB info reporting
* Subtle bug fixed in `noDeps` flag support

CHANGES.rst

@@ -1,12 +1,16 @@
 
-Revision 0.3.0, XX-02-2018
+Revision 0.3.0, XX-04-2018
 --------------------------
 
+- The `mibcopy` tool implemented to copy MIB modules from files with
+  potentially messed up names into a directory under canonical MIB
+  names picking up the latest MIB revision along the way.
 - ZIP archive reader implemented to pull ASN.1 MIB files from .zip
   archives pretty much in the same way as from plain directories
 - Copyright notice bumped up to year 2018
 - Project site in the docs changes from SourceForge to snmplabs.com
 - PRODUCT-RELEASE generation added to the JSON code generator
+- Fixed missing REVISIONS generations in MODULE-IDENTITY
 
 Revision 0.2.2, 13-11-2017
 --------------------------

pysmi/codegen/jsondoc.py

@@ -72,6 +72,7 @@ def __init__(self):
         self._importMap = {}
         self._out = {}  # k, v = name, generated code
         self._moduleIdentityOid = None
+        self._moduleRevision = None
         self._enterpriseOid = None
         self._oids = set()
         self._complianceOids = []
@@ -273,6 +274,8 @@ def genModuleIdentity(self, data):
         if revisions:
             outDict['revisions'] = revisions
 
+            self._moduleRevision = revisions[0]['revision']
+
         if self.genRules['text']:
             if lastUpdated:
                 outDict['lastupdated'] = lastUpdated
@@ -960,6 +963,7 @@ def genCode(self, ast, symbolTable, **kwargs):
         return MibInfo(oid=moduleOid,
                        identity=self._moduleIdentityOid,
                        name=self.moduleName[0],
+                       revision=self._moduleRevision,
                        oids=self._oids,
                        enterprise=self._enterpriseOid,
                        compliance=self._complianceOids,

pysmi/codegen/pysnmp.py

@@ -116,6 +116,7 @@ def __init__(self):
         self._importMap = {}
         self._out = {}  # k, v = name, generated code
         self._moduleIdentityOid = None
+        self._moduleRevision = None
         self.moduleName = ['DUMMY']
         self.genRules = {'text': True}
         self.symbolTable = {}
@@ -340,7 +341,9 @@ def genModuleIdentity(self, data, classmode=False):
         outStr = name + ' = ModuleIdentity(' + oidStr + ')' + label + '\n'
 
         if revisionsAndDescrs:
-            revisions, descriptions = revisionsAndDescrs
+            last_revision, revisions, descriptions = revisionsAndDescrs
+
+            self._moduleRevision = last_revision
 
             if revisions:
                 outStr += name + revisions + '\n'
@@ -1030,7 +1033,9 @@ def genRevisions(self, data, classmode=False):
             [dorepr(self.textFilter('description', x[1][1])) for x in data[0]]
         )
 
-        return revisions, descriptions
+        lastRevision = data[0][0][0]
+
+        return lastRevision, revisions, descriptions
 
     def genRow(self, data, classmode=False):
         row = data[0]
@@ -1177,6 +1182,7 @@ def genCode(self, ast, symbolTable, **kwargs):
         return MibInfo(oid=moduleOid,
                        identity=self._moduleIdentityOid,
                        name=self.moduleName[0],
+                       revision=self._moduleRevision,
                        oids=[],
                        enterprise=None,
                        compliance=[],

pysmi/codegen/symtable.py

@@ -88,6 +88,7 @@ def __init__(self):
         self._symsOrder = []
         self._out = {}  # k, v = symbol, properties
         self.moduleName = ['DUMMY']
+        self._moduleRevision = None
         self.genRules = {'text': True}
 
     def symTrans(self, symbol):
@@ -220,6 +221,9 @@ def genModuleIdentity(self, data, classmode=False):
                     'oid': oid,
                     'origName': origName}
 
+        if revisions:
+            self._moduleRevision = revisions[0]
+
         self.regSym(pysmiName, symProps)
 
     # noinspection PyUnusedLocal
@@ -491,15 +495,16 @@ def genTime(self, data, classmode=False):
 
     # noinspection PyUnusedLocal,PyUnusedLocal,PyMethodMayBeStatic
     def genLastUpdated(self, data, classmode=False):
-        return ''
+        return data[0]
 
     # noinspection PyUnusedLocal,PyUnusedLocal,PyMethodMayBeStatic
     def genOrganization(self, data, classmode=False):
-        return ''
+        return data[0]
 
     # noinspection PyUnusedLocal,PyUnusedLocal,PyMethodMayBeStatic
     def genRevisions(self, data, classmode=False):
-        return ''
+        lastRevision, lastDescription = data[0][0][0], data[0][0][1][1]
+        return lastRevision, lastDescription
 
     def genRow(self, data, classmode=False):
         row = data[0]
@@ -622,4 +627,7 @@ def genCode(self, ast, symbolTable, **kwargs):
             'canonical MIB name %s (%s), imported MIB(s) %s, Symbol table size %s symbols' % (
                 self.moduleName[0], moduleOid, ','.join(importedModules) or '<none>', len(self._out)))
 
-        return MibInfo(oid=None, name=self.moduleName[0], imported=tuple([x for x in importedModules])), self._out
+        return MibInfo(oid=None,
+                       name=self.moduleName[0],
+                       revision=self._moduleRevision,
+                       imported=tuple([x for x in importedModules])), self._out

pysmi/compiler.py

@@ -191,6 +191,7 @@ class instances (values)
         builtMibs = {}
         symbolTableMap = {}
         mibsToParse = [x for x in mibnames]
+        canonicalMibNames = {}
 
         while mibsToParse:
             mibname = mibsToParse.pop(0)
@@ -223,6 +224,11 @@ class instances (values)
 
                         mibsToParse.extend(mibInfo.imported)
 
+                        if fileInfo.name in mibnames:
+                            if mibInfo.name not in canonicalMibNames:
+                                canonicalMibNames[mibInfo.name] = []
+                            canonicalMibNames[mibInfo.name].append(fileInfo.name)
+
                         debug.logger & debug.flagCompiler and debug.logger(
                             '%s (%s) read from %s, immediate dependencies: %s' % (
                                 mibInfo.name, mibname, fileInfo.path, ', '.join(mibInfo.imported) or '<none>'))
@@ -258,7 +264,7 @@ class instances (values)
                     processed[mibname] = statusMissing
 
         debug.logger & debug.flagCompiler and debug.logger(
-            'MIBs analized %s, MIBs failed %s' % (len(parsedMibs), len(failedMibs)))
+            'MIBs analyzed %s, MIBs failed %s' % (len(parsedMibs), len(failedMibs)))
 
         #
         # See what MIBs need generating
@@ -297,7 +303,7 @@ class instances (values)
                 debug.logger & debug.flagCompiler and debug.logger(
                     'no suitable compiled MIB %s found anywhere' % mibname)
 
-                if options.get('noDeps') and mibname not in mibnames:
+                if options.get('noDeps') and mibname not in canonicalMibNames:
                     debug.logger & debug.flagCompiler and debug.logger(
                         'excluding imported MIB %s from code generation' % mibname)
                     del parsedMibs[mibname]
@@ -362,7 +368,7 @@ class instances (values)
         #
 
         for mibname in failedMibs.copy():
-            if options.get('noDeps') and mibname not in mibnames:
+            if options.get('noDeps') and mibname not in canonicalMibNames:
                 debug.logger & debug.flagCompiler and debug.logger('excluding imported MIB %s from borrowing' % mibname)
                 continue
 
@@ -425,7 +431,7 @@ class instances (values)
                 debug.logger & debug.flagCompiler and debug.logger(
                     'no suitable compiled MIB %s found anywhere' % mibname)
 
-                if options.get('noDeps') and mibname not in mibnames:
+                if options.get('noDeps') and mibname not in canonicalMibNames:
                     debug.logger & debug.flagCompiler and debug.logger(
                         'excluding imported MIB %s from borrowing' % mibname)
                     processed[mibname] = statusUntouched
@@ -484,6 +490,7 @@ class instances (values)
                         oid=mibInfo.oid,
                         oids=mibInfo.oids,
                         identity=mibInfo.identity,
+                        revision=mibInfo.revision,
                         enterprise=mibInfo.enterprise,
                         compliance=mibInfo.compliance,
                     )

pysmi/mibinfo.py

@@ -25,6 +25,9 @@ class MibInfo(object):
     #: module OID
     oid = ''
 
+    #: MIB revision as `datetime`
+    revision = None
+
     #: all OIDs defined in this module
     oids = ()