"ztps --validate" validates all YAML config files

client/bootstrap

@@ -170,7 +170,7 @@ def _exit(code):
             shutil.move(backup_path, BOOT_EXTENSIONS_FOLDER)
     else:
         if code:
-            for path in [x for x in all_files(FLASH)
+            for path in [x for x in all_files_and_dirs(FLASH)
                          if x not in FLASH_FILES]:
                 log('Deleting %s...' % path)
                 if os.path.isdir(path):
@@ -248,7 +248,7 @@ def flash_usage():
 def flash_snapshot():
     #pylint: disable=W0603
     global FLASH_FILES
-    FLASH_FILES = all_files(FLASH)
+    FLASH_FILES = all_files_and_dirs(FLASH)
 
     for filename in [STARTUP_CONFIG, RC_EOS, BOOT_EXTENSIONS]:
         if os.path.isfile(filename):
@@ -276,7 +276,7 @@ def flash_snapshot():
 def get_first_token(sequence):
     return next((x for x in sequence if x), '')
 
-def all_files(path):
+def all_files_and_dirs(path):
     result = []
     for top, dirs, files in os.walk(path):
         result += [os.path.join(top, d) for d in dirs]

test/server/test_topology.py

@@ -40,14 +40,14 @@
 
 from ztpserver.topology import Neighbordb, Pattern
 from ztpserver.topology import create_node, load_file, load_neighbordb
-from ztpserver.topology import default_filename, replace_config_action
+from ztpserver.topology import neighbordb_path, replace_config_action
 from ztpserver.topology import load_pattern
 from server_test_lib import enable_logging, random_string
 
 class NeighbordbUnitTests(unittest.TestCase):
 
-    def test_default_filename(self):
-        result = default_filename()
+    def test_neighbordb_path(self):
+        result = neighbordb_path()
         self.assertEqual(result, '/usr/share/ztpserver/neighbordb')
 
     @patch('ztpserver.topology.load')

ztpserver/app.py

@@ -32,20 +32,22 @@
 # vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
 # pylint: disable=C0103
 #
-import os
-import sys
-import argparse
 
+import argparse
 import logging
+import os
+import sys
 
 from wsgiref.simple_server import make_server
 
+
 from ztpserver import config, controller
 
 from ztpserver.serializers import load
 from ztpserver.validators import NeighbordbValidator
 from ztpserver.constants import CONTENT_TYPE_YAML
-from ztpserver.topology import default_filename
+from ztpserver.topology import neighbordb_path
+from ztpserver.utils import all_files
 
 DEFAULT_CONF = config.GLOBAL_CONF_FILE_PATH
 
@@ -94,7 +96,7 @@ def load_config(conf=None):
     if os.path.exists(conf):
         config.runtime.read(conf)
 
-def start_wsgiapp(conf=None, debug=False):
+def start_wsgiapp():
     """ Provides the entry point into the application for wsgi compliant
     servers.   Accepts a single keyword argument ``conf``.   The ``conf``
     keyword argument specifies the path the server configuration file.  The
@@ -105,9 +107,6 @@ def start_wsgiapp(conf=None, debug=False):
 
     """
 
-    load_config(conf)
-    start_logging(debug)
-
     log.info('Logging started for ztpserver')
     log.info('Using repository %s', config.runtime.default.data_root)
 
@@ -116,7 +115,7 @@ def start_wsgiapp(conf=None, debug=False):
 
     return controller.Router()
 
-def run_server(conf, debug, version):
+def run_server(version):
     """ The :py:func:`run_server` is called by the main command line routine to
     run the server as standalone.   This function accepts a single argument
     that points towards the configuration file describing this server
@@ -126,7 +125,7 @@ def run_server(conf, debug, version):
     :param conf: string path pointing to configuration file
     """
 
-    app = start_wsgiapp(conf, debug)
+    app = start_wsgiapp()
 
     host = config.runtime.server.interface
     port = config.runtime.server.port
@@ -141,29 +140,71 @@ def run_server(conf, debug, version):
     except KeyboardInterrupt:
         log.info('Shutdown...')
 
-def run_validator(filename=None):
+def run_validator():
 
+    # Validating neighbordb
+    validator = NeighbordbValidator('N/A')
+    neighbordb = neighbordb_path()
+    print 'Validating neighbordb (\'%s\')...' % neighbordb
     try:
-        print 'Validating file \'%s\'\n' % filename
-        validator = NeighbordbValidator('VALIDATION_TEST')
-        filename = filename or default_filename()
-        validator.validate(load(filename, CONTENT_TYPE_YAML,
+        validator.validate(load(neighbordb, CONTENT_TYPE_YAML,
                                 'validator'))
-        print 'Valid Patterns (count: %d)' % len(validator.valid_patterns)
-        print '--------------------------'
-        for index, pattern in enumerate(sorted(validator.valid_patterns)):
-            print '[%d] %s' % (index, pattern[1])
-        print
-        print 'Failed Patterns (count: %d)' % len(validator.invalid_patterns)
-        print '---------------------------'
-        for index, pattern in enumerate(sorted(validator.invalid_patterns)):
-            print '[%d] %s' % (index, pattern[1])
-        print
-
+        total_patterns = len(validator.valid_patterns) + \
+            len(validator.invalid_patterns)
+
+        if validator.invalid_patterns:
+            print '\nERROR: Failed to validate neighbordb patterns'
+            print '   Invalid Patterns (count: %d/%d)' % \
+                (len(validator.invalid_patterns),
+                 total_patterns)
+            print '   ---------------------------'
+            for index, pattern in enumerate(
+                sorted(validator.invalid_patterns)):
+                print '   [%d] %s' % (index, pattern[1])
+        else:
+            print 'Ok!'            
     except Exception as exc:        #pylint: disable=W0703
-        print 'ERROR: Failed to validate neighbordb: %s' % exc
-
-
+        print 'ERROR: Failed to validate neighbordb\n%s' % exc
+
+    data_root = config.runtime.default.data_root
+
+    print '\nValidating definitions...'
+    for definition in all_files(os.path.join(data_root, 
+                                             'definitions')):
+        print 'Validating %s...' % definition,
+        try:
+            load(definition, CONTENT_TYPE_YAML,
+                 'validator')
+            print 'Ok!'
+        except Exception as exc:        #pylint: disable=W0703            
+            print '\nERROR: Failed to validate %s\n%s' % \
+                (definition, exc)
+
+    print '\nValidating resources...'
+    for resource in all_files(os.path.join(data_root, 
+                                             'resources')):
+        print 'Validating %s...' % resource,
+        try:
+            load(resource, CONTENT_TYPE_YAML,
+                 'validator')
+            print 'Ok!'
+        except Exception as exc:        #pylint: disable=W0703            
+            print '\nERROR: Failed to validate %s\n%s' % \
+                (resource, exc)
+
+    print '\nValidating nodes...'
+    for filename in [x for x in all_files(os.path.join(data_root, 
+                                                       'nodes'))
+                     if x.split('/')[-1] in ['definition',
+                                             'pattern']]:
+        print 'Validating %s...' % filename,
+        try:
+            load(filename, CONTENT_TYPE_YAML,
+                 'validator')
+            print 'Ok!'
+        except Exception as exc:        #pylint: disable=W0703            
+            print '\nERROR: Failed to validate %s\n%s' % \
+                (filename, exc)
 
 def main():
     """ The :py:func:`main` is the main entry point for the ztpserver if called
@@ -185,10 +226,9 @@ def main():
                         default=DEFAULT_CONF,
                         help='Specifies the configuration file to use')
 
-    parser.add_argument('--validate',
-                        type=str,
-                        metavar='FILENAME',
-                        help='Runs a validation check on neighbordb')
+    parser.add_argument('--validate-config', '-V',
+                        action='store_true',
+                        help='Validates config files')
 
     parser.add_argument('--debug',
                         action='store_true',
@@ -204,12 +244,13 @@ def main():
         pass
 
     if args.version:
-        print 'ztps version %s' % version
+        print 'ZTPServer version %s' % version
         sys.exit()
 
-    if args.validate is not None:
-        load_config(args.conf)
-        start_logging(args.debug)
-        sys.exit(run_validator(args.validate))
+    load_config(args.conf)
+    start_logging(args.debug)
+
+    if args.validate_config:
+        sys.exit(run_validator())
 
-    return run_server(args.conf, args.debug, version)
+    return run_server(version)

ztpserver/config.py

@@ -314,7 +314,7 @@ def read(self, filename):
 
 runtime.add_attribute(StrAttr(
     name='console_logging_format',
-    default='%(levelname)s: [%(module)s:%(lineno)d] %(message)s',
+    default='%(asctime)s:%(levelname)s:[%(module)s:%(lineno)d] %(message)s',
     environ='ZTPS_CONSOLE_LOGGING_FORMAT'
 ))
 

ztpserver/topology.py

@@ -58,7 +58,7 @@
 
 Neighbor = collections.namedtuple('Neighbor', ['device', 'interface'])
 
-def default_filename():
+def neighbordb_path():
     ''' Returns the path for neighbordb based on the conf file
     '''
 
@@ -85,8 +85,8 @@ def load_neighbordb(node_id, contents=None):
     try:
         if not contents:
             log.info('%s: loading neighbordb file: %s' % 
-                     (node_id, default_filename()))
-            contents = load_file(default_filename(), CONTENT_TYPE_YAML,
+                     (node_id, neighbordb_path()))
+            contents = load_file(neighbordb_path(), CONTENT_TYPE_YAML,
                                  node_id)
 
         # neighbordb is empty

ztpserver/utils.py

@@ -33,6 +33,8 @@
 
 import logging
 import re
+import os
+
 from urlparse import urlsplit, urlunsplit
 
 log = logging.getLogger(__name__)
@@ -258,3 +260,9 @@ def url_path_join(*parts):
 
 def get_first_token(sequence):
     return next((x for x in sequence if x), '')
+
+def all_files(path):
+    result = []
+    for top, _, files in os.walk(path):
+        result += [os.path.join(top, f) for f in files]
+    return result

ztpserver/validators.py

@@ -38,6 +38,7 @@
 import collections
 
 from ztpserver.utils import expand_range, parse_interface
+from ztpserver.config import runtime
 
 REQUIRED_PATTERN_ATTRIBUTES = ['name', 'definition']
 OPTIONAL_PATTERN_ATTRIBUTES = ['node', 'variables', 'interfaces']
@@ -47,7 +48,6 @@
 KW_NONE_RE = re.compile(r' *none *')
 WC_PORT_RE = re.compile(r'.*')
 
-
 INVALID_INTERFACE_PATTERNS = [(KW_ANY_RE, KW_ANY_RE, KW_NONE_RE),
                               (KW_ANY_RE, KW_NONE_RE, KW_NONE_RE),
                               (KW_ANY_RE, KW_NONE_RE, KW_ANY_RE),
@@ -248,10 +248,11 @@ def validate_node(self):
             return
 
         # if system MAC is used
-        node = str(self.data['node']).replace(':', '').replace('.', '')
-        if re.search(ANTINODE_PATTERN, node):
-            raise ValidationError('invalid value for \'node\' (%s)' %
-                                  self.data['node'])
+        if runtime.default.identifier == 'systemmac':
+            node = str(self.data['node']).replace(':', '').replace('.', '')
+            if re.search(ANTINODE_PATTERN, node):
+                raise ValidationError('invalid value for \'node\' (%s)' %
+                                      self.data['node'])
 
     def validate_variables(self):
         if not self.data: