Renamed the migration script to pulp-manage-db.

Also, this script is now responsible for loading content types to the
database instead of api.py. Added a unit test to ensure that the type
loading code gets called by this method.

I also converted the script to use exception handling instead of return
codes all over the place. There is much reformation work left to do for
this script, but this is a good start.

platform/bin/pulp-manage-db

@@ -1,7 +1,5 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-
-# Copyright © 2010 Red Hat, Inc.
+#!/usr/bin/env python
+# Copyright (c) 2010-2012 Red Hat, Inc.
 #
 # This software is licensed to you under the GNU General Public License,
 # version 2 (GPLv2). There is NO WARRANTY for this software, express or
@@ -16,8 +14,8 @@
 
 import sys
 
-from pulp.server.db.migrate import script
+from pulp.server.db import manage
 
 
 if __name__ == '__main__':
-    sys.exit(script.main())
+    sys.exit(manage.main())

platform/src/pulp/plugins/loader/api.py

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright © 2011-2012 Red Hat, Inc.
+# Copyright (c) 2011-2012 Red Hat, Inc.
 #
 # This software is licensed to you under the GNU General Public License as
 # published by the Free Software Foundation; either version 2 of the License
@@ -25,8 +23,6 @@
 from pulp.plugins.types import database, parser
 from pulp.plugins.types.model import TypeDescriptor
 
-# constants --------------------------------------------------------------------
-
 _LOG = logging.getLogger(__name__)
 
 # implicit singleton instance of PluginManager
@@ -54,9 +50,6 @@ def initialize(validate=True):
     # pre-initialization validation
     assert not _is_initialized()
 
-    # initialization
-    _load_content_types(_TYPES_DIR)
-
     _create_manager()
     # add plugins here in the form (path, base class, manager map)
     plugin_tuples =  ((_DISTRIBUTORS_DIR, Distributor, _MANAGER.distributors),
@@ -349,6 +342,7 @@ def get_profiler_by_id(id):
     cls, cfg = _MANAGER.profilers.get_plugin_by_id(id)
     return cls(), cfg
 
+
 def get_profiler_by_type(type_id):
     """
     Get a profiler instance that supports the specified content type.
@@ -364,6 +358,19 @@ def get_profiler_by_type(type_id):
     cls, cfg = _MANAGER.profilers.get_plugin_by_id(ids[0])
     return cls(), cfg
 
+
+def load_content_types(types_dir=_TYPES_DIR):
+    """
+    :type types_dir: str
+    """
+    if not os.access(types_dir, os.F_OK | os.R_OK):
+        msg = _('Cannot load types: path does not exist or cannot be read: %(p)s')
+        _LOG.critical(msg % {'p': types_dir})
+        raise IOError(msg)
+    descriptors = _load_type_descriptors(types_dir)
+    _load_type_definitions(descriptors)
+
+
 # initialization methods -------------------------------------------------------
 
 def _is_initialized():
@@ -376,17 +383,6 @@ def _create_manager():
     global _MANAGER
     _MANAGER = PluginManager()
 
-def _load_content_types(types_dir):
-    """
-    @type types_dir: str
-    """
-    if not os.access(types_dir, os.F_OK | os.R_OK):
-        msg = _('Cannot load types: path does not exist or cannot be read: %(p)s')
-        _LOG.critical(msg % {'p': types_dir})
-        return
-    descriptors = _load_type_descriptors(types_dir)
-    _load_type_definitions(descriptors)
-
 
 def _load_type_descriptors(path):
     """

platform/src/pulp/plugins/types/parser.py

@@ -215,7 +215,7 @@ def _validate_syntax(descriptors):
             # Make sure all required fields are present
             for key in REQUIRED_DEFINITION_FIELDS:
                 if key not in type_definition:
-                    LOG.error('Unexpected key [%s] from descriptor [%s] in type definition [%s]' % (key, d.filename, ', '.join(type_definition.keys())))
+                    LOG.error('Missing required key [%s] from descriptor [%s] in type definition [%s]' % (key, d.filename, ', '.join(type_definition.keys())))
                     missing_attribute_descriptors.append(d)
 
     if len(invalid_attribute_descriptors) > 0:
@@ -300,6 +300,7 @@ def _all_types(descriptors):
         return []
     return reduce(operator.add, [descriptor.parsed['types'] for descriptor in descriptors])
 
+
 def _all_type_ids(descriptors):
     """
     @return: list of all type IDs across all descriptors (potentially containing
@@ -311,6 +312,7 @@ def _all_type_ids(descriptors):
 
     return all_ids
 
+
 def _all_referenced_type_ids(descriptors):
     """
     @return: list of all IDs that are mentioned as referenced types across all types
@@ -327,8 +329,9 @@ def _all_referenced_type_ids(descriptors):
 
     return all_referenced_ids
 
+
 def _valid_id(id):
     """
     @return: True if the given ID is a valid type ID; false otherwise
     """
-    return TYPE_ID_REGEX.match(id) is not None
+    return TYPE_ID_REGEX.match(id) is not None

platform/src/pulp/server/db/migrate/script.py → platform/src/pulp/server/db/manage.py

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-
-# Copyright © 2010-2012 Red Hat, Inc.
+# Copyright (c) 2010-2012 Red Hat, Inc.
 #
 # This software is licensed to you under the GNU General Public
 # License as published by the Free Software Foundation; either version
@@ -17,6 +15,7 @@
 import sys
 from optparse import OptionParser, SUPPRESS_HELP
 
+from pulp.plugins.loader.api import load_content_types
 from pulp.server.db import connection
 
 # the db connection and auditing need to be initialied before any further
@@ -29,23 +28,31 @@
 from pulp.server.db.migrate.validate import validate
 from pulp.server.db.migrate.versions import get_migration_modules
 from pulp.server.db.version import (
-    VERSION, get_version_in_use, set_version, is_validated, set_validated, 
+    VERSION, get_version_in_use, set_version, is_validated, set_validated,
     revert_to_version, clean_db)
 
+
+class DataError(Exception):
+    """
+    This Exception is used when we want to return the os.EX_DATAERR code.
+    """
+    pass
+
+
 def parse_args():
     parser = OptionParser()
     parser.add_option('--force', action='store_true', dest='force',
                       default=False, help=SUPPRESS_HELP)
     parser.add_option('--from', dest='start', default=None,
-                      help='run the migration starting at the version passed in')
+                      help='Run the migration starting at the version passed in')
     parser.add_option('--test', action='store_true', dest='test',
-                      default=False, 
-                      help='run migration, but do not update version')
+                      default=False,
+                      help='Run migration, but do not update version')
     parser.add_option('--log-file', dest='log_file',
                       default='/var/log/pulp/db.log',
-                      help='file for log messages')
+                      help='File for log messages')
     parser.add_option('--log-level', dest='log_level', default='info',
-                      help='level of logging (debug, info, error, critical)')
+                      help='Level of logging (debug, info, error, critical)')
     options, args = parser.parse_args()
     if args:
         parser.error('unknown arguments: %s' % ', '.join(args))
@@ -60,71 +67,82 @@ def start_logging(options):
     logger.addHandler(handler)
 
 
-def datamodel_migration(options):
+def migrate_database(options):
     version = get_version_in_use()
     assert version <= VERSION, \
-            'version in use (%d) greater than expected version (%d)' % \
+            'Version in use (%d) greater than expected version (%d).' % \
             (version, VERSION)
     if version == VERSION:
-        print 'data model in use matches the current version'
-        return os.EX_OK
+        print 'Data model in use matches the current version.'
+        return
     for mod in get_migration_modules():
         # it is assumed here that each migration module will have two members:
         # 1. version - an integer value of the version the module migrates to
         # 2. migrate() - a function that performs the migration
         if mod.version <= version:
             continue
         if mod.version > VERSION:
-            print >> sys.stderr, \
-                    'migration provided for higher version than is expected'
-            return os.EX_OK
+            raise DataError('Migration provided for higher version than is expected.')
         try:
             mod.migrate()
         except Exception, e:
-            _log.critical(str(e))
-            _log.critical(''.join(traceback.format_exception(*sys.exc_info())))
-            _log.critical('migration to data model version %d failed' %
+            _log.critical('Migration to data model version %d failed.' %
                           mod.version)
             print >> sys.stderr, \
-                    'migration to version %d failed, see %s for details' % \
+                    'Migration to version %d failed, see %s for details.' % \
                     (mod.version, options.log_file)
-            return os.EX_SOFTWARE
+            raise e
         if not options.test:
             set_version(mod.version)
         version = mod.version
     if version < VERSION:
-        return os.EX_DATAERR
-    return os.EX_OK
+        raise DataError('The current version is still lower than the expected version, even ' +\
+                        'after migrations were applied.')
+    validate_database_migrations(options)
 
 
-def datamodel_validation(options):
+def validate_database_migrations(options):
     errors = 0
     if not is_validated():
         errors = validate()
     if errors:
-        print >> sys.stderr, '%d errors on validation, see %s for details' % \
-                (errors, options.log_file)
-        return os.EX_DATAERR
+        error_message = '%d errors on validation, see %s for details'%(errors, options.log_file)
+        raise DataError(error_message)
     if not options.test:
         set_validated()
-    return os.EX_OK
 
 
 def main():
-    options = parse_args()
-    start_logging(options)
+    try:
+        options = parse_args()
+        start_logging(options)
+        _auto_manage_db(options)
+    except DataError, e:
+        _log.critical(str(e))
+        _log.critical(''.join(traceback.format_exception(*sys.exc_info())))
+        print >> sys.stderr, str(e)
+        return os.EX_DATAERR
+    except Exception, e:
+        _log.critical(str(e))
+        _log.critical(''.join(traceback.format_exception(*sys.exc_info())))
+        print >> sys.stderr, str(e)
+        return os.EX_SOFTWARE
+
+def _auto_manage_db(options):
     if options.force:
-        print 'clearing previous versions'
+        print 'Clearing previous versions.'
         clean_db()
+
     if options.start is not None:
         last = int(options.start) - 1
-        print 'reverting db to version %d' % last
+        print 'Reverting db to version %d.' % last
         revert_to_version(last)
-    ret = datamodel_migration(options)
-    if ret != os.EX_OK:
-        return ret
-    ret = datamodel_validation(options)
-    if ret != os.EX_OK:
-        return ret
-    print 'database migration to version %d complete' % VERSION
+
+    print 'Beginning database migrations.'
+    migrate_database(options)
+    print 'Database migrations complete.'
+
+    print 'Loading content types.'
+    load_content_types()
+    print 'Content types loaded.'
     return os.EX_OK

platform/src/pulp/server/db/version.py

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-
-# Copyright © 2010 Red Hat, Inc.
+# Copyright (c) 2010 Red Hat, Inc.
 #
 # This software is licensed to you under the GNU General Public
 # License as published by the Free Software Foundation; either version
@@ -15,7 +13,6 @@
 
 import pymongo
 
-#from pulp.server.db.connection import get_object_db
 from pulp.server.db.model.base import Model
 
 

platform/test/unit/test_plugin_api.py

@@ -19,6 +19,10 @@ class TestAPI(base.PulpServerTests):
     @mock.patch('pulp.plugins.loader.loading.load_plugins_from_entry_point', autospec=True)
     def test_init_calls_entry_points(self, mock_load):
         api._MANAGER = None
+        # This test is problematic, because it relies on the pulp_rpm package, which depends on this
+        # package. We should really mock the type loading and test that the mocked types were loaded
+        # For now, we can get around the problem by just calling load_content_types.
+        api.load_content_types()
         api.initialize()
         # calls for 5 types of plugins
         self.assertEqual(mock_load.call_count, 5)