Merge pull request #70 from kbase/db-version-check

Add check on startup for exact db version number

lib/biokbase/catalog/Impl.py

@@ -21,7 +21,7 @@ class Catalog:
     #########################################
     VERSION = "0.0.1"
     GIT_URL = "git@github.com:kbase/catalog.git"
-    GIT_COMMIT_HASH = "52464b1e02762e4ffa5b6f4a0d8e50f71e5c57d0"
+    GIT_COMMIT_HASH = "fb01cbdd9c1f3e5861e843314640e493ae16bb76"
 
     #BEGIN_CLASS_HEADER
     #END_CLASS_HEADER

lib/biokbase/catalog/db.py

@@ -1254,6 +1254,11 @@ def check_db_schema(self):
             self.update_db_version(4)
             print('done.')
 
+        if db_version>4:
+            # bad version!
+            raise IOError('Incompatible DB versions.  Expecting DB V4, found DV V'+str(db_version) + 
+                '. You are probably running an old version of the service.  Start up failed.')
+
 
     def get_db_version(self):
         # version is a collection that should only have a single 

lib/biokbase/catalog/registrar.py

@@ -7,6 +7,7 @@
 import datetime
 import pprint
 import json
+import codecs
 import semantic_version
 
 #import git
@@ -242,7 +243,8 @@ def sanity_checks_and_parse(self, basedir, git_commit_hash):
             else:
                 yaml_filename = 'kbase.yml'
         # parse some stuff, and check for things
-        with open(os.path.join(basedir,yaml_filename)) as kb_yaml_file:
+
+        with codecs.open(os.path.join(basedir,yaml_filename), 'r', "utf-8", errors='ignore') as kb_yaml_file:
             kb_yaml_string = kb_yaml_file.read()
         self.kb_yaml = yaml.load(kb_yaml_string)
         self.log('=====kbase.yaml parse:')
@@ -429,17 +431,18 @@ def validate_method_specs(self, basedir):
                         raise ValueError('Invalid narrative method specification ('+m+'): No spec.json file defined.')
                     if not os.path.isfile(os.path.join(method_path,'display.yaml')):
                         raise ValueError('Invalid narrative method specification ('+m+'): No spec.json file defined.')
-                    with open(os.path.join(method_path,'spec.json')) as spec_json_file:
+
+                    with codecs.open(os.path.join(method_path,'spec.json'), 'r', "utf-8", errors='ignore') as spec_json_file:
                         spec_json = spec_json_file.read()
-                    with open(os.path.join(method_path,'display.yaml')) as display_yaml_file:
+                    with codecs.open(os.path.join(method_path,'display.yaml'), 'r', "utf-8", errors='ignore') as display_yaml_file:
                         display_yaml = display_yaml_file.read()
 
                     # gather any extra html files
                     extraFiles = {}
                     for extra_file_name in os.listdir(os.path.join(method_path)):
                         if not os.path.isfile(os.path.join(method_path,extra_file_name)): break
                         if not extra_file_name.endswith('.html'): break
-                        with open(os.path.join(method_path,extra_file_name)) as extra_file:
+                        with codecs.open(oos.path.join(method_path,extra_file_name), 'r', "utf-8", errors='ignore') as extra_file: 
                             extrafiles[extra_file_name] = extra_file.read()
 
                     # validate against the NMS target endpoint

test/catalog_test_util.py

@@ -18,11 +18,28 @@ class CatalogTestUtil:
     def __init__(self, test_dir):
         self.test_dir = os.path.abspath(test_dir)
 
+    def setUpEmpty(self, db_version=None):
+        self.log("setUp()")
+        self.log("test directory="+self.test_dir)
+        self._setup_config()
+        self._init_db_handles()
+        self._clear_db()
+
+        if db_version is not None:
+            self.db_version.insert({'version_doc':True, 'version':db_version})
+
 
     def setUp(self):
         self.log("setUp()")
         self.log("test directory="+self.test_dir)
 
+        self._setup_config()
+        self._init_db_handles()
+        self._clear_db()
+        self._initialize_mongo_data()
+
+    def _setup_config(self):
+
         # 1 read config file and pull out some stuff
         if not os.path.isfile(os.path.join(self.test_dir,'test.cfg')):
             raise ValueError('test.cfg does not exist in test dir')
@@ -42,6 +59,28 @@ def setUp(self):
         self.test_user_2 = self.test_cfg['test-user-2']
         #self.test_user_psswd_2 = self.test_cfg['test-user-psswd-2']
 
+        # 2 setup the scratch space
+        self.scratch_dir = os.path.join(self.test_dir,'temp_test_files',datetime.datetime.now().strftime("%Y-%m-%d-(%H-%M-%S-%f)"))
+        self.log("scratch directory="+self.scratch_dir)
+        os.makedirs(self.scratch_dir)
+
+        # 3 assemble the config file for the catalog service
+        self.catalog_cfg = {
+            'admin-users':self.test_user_2,
+            'mongodb-host':self.test_cfg['mongodb-host'],
+            'mongodb-database':self.test_cfg['mongodb-database'],
+            'temp-dir':self.scratch_dir,
+            'docker-base-url':self.test_cfg['docker-base-url'],
+            'docker-registry-host':self.test_cfg['docker-registry-host'],
+            'docker-push-allow-insecure':self.test_cfg['docker-push-allow-insecure'],
+            'nms-url':self.test_cfg['nms-url'],
+            'nms-admin-user':self.test_cfg['nms-admin-user'],
+            'nms-admin-psswd':self.test_cfg['nms-admin-psswd'],
+            'ref-data-base':self.test_cfg['ref-data-base'],
+            'kbase-endpoint':self.test_cfg['kbase-endpoint']
+        }
+
+    def _init_db_handles(self):
         # 2 check that db exists and collections are empty
         self.mongo = MongoClient('mongodb://'+self.test_cfg['mongodb-host'])
         db = self.mongo[self.test_cfg['mongodb-database']]
@@ -59,6 +98,7 @@ def setUp(self):
         self.exec_stats_apps = db[MongoCatalogDBI._EXEC_STATS_APPS]
         self.exec_stats_users = db[MongoCatalogDBI._EXEC_STATS_USERS]
 
+    def _clear_db(self):
         # just drop the test db
         self.db_version.drop()
         self.modules.drop()
@@ -76,35 +116,8 @@ def setUp(self):
         #if self.modules.count() > 0 :
         #    raise ValueError('mongo database collection "'+MongoCatalogDBI._MODULES+'"" not empty (contains '+str(self.modules.count())+' records).  aborting.')
 
-        self.initialize_mongo()
-
-        # 3 setup the scratch space
-        self.scratch_dir = os.path.join(self.test_dir,'temp_test_files',datetime.datetime.now().strftime("%Y-%m-%d-(%H-%M-%S-%f)"))
-        self.log("scratch directory="+self.scratch_dir)
-        os.makedirs(self.scratch_dir)
-
 
-        # 4 startup any dependencies (nms, docker registry?)
-
-
-        # 4 assemble the config file for the catalog service
-        self.catalog_cfg = {
-            'admin-users':self.test_user_2,
-            'mongodb-host':self.test_cfg['mongodb-host'],
-            'mongodb-database':self.test_cfg['mongodb-database'],
-            'temp-dir':self.scratch_dir,
-            'docker-base-url':self.test_cfg['docker-base-url'],
-            'docker-registry-host':self.test_cfg['docker-registry-host'],
-            'docker-push-allow-insecure':self.test_cfg['docker-push-allow-insecure'],
-            'nms-url':self.test_cfg['nms-url'],
-            'nms-admin-user':self.test_cfg['nms-admin-user'],
-            'nms-admin-psswd':self.test_cfg['nms-admin-psswd'],
-            'ref-data-base':self.test_cfg['ref-data-base'],
-            'kbase-endpoint':self.test_cfg['kbase-endpoint']
-        }
-
-
-    def initialize_mongo(self):
+    def _initialize_mongo_data(self):
         self.log("initializing mongo")
         # we only have one collection for now, but this should look over all collection folders
         load_count = 0
@@ -205,18 +218,7 @@ def getCatalogConfig(self):
 
     def tearDown(self):
         self.log("tearDown()")
-        self.modules.drop()
-        self.module_versions.drop()
-        self.local_functions.drop()
-        self.developers.drop()
-        self.build_logs.drop()
-        self.favorites.drop()
-        self.client_groups.drop()
-        self.volume_mounts.drop()
-
-        self.exec_stats_raw.drop()
-        self.exec_stats_apps.drop()
-        self.exec_stats_users.drop()
+        self._clear_db();
 
         # make sure NMS is clean after each test
         self.mongo.drop_database(self.nms_test_cfg['method-spec-mongo-dbname'])

test/startup_test.py

@@ -0,0 +1,58 @@
+
+
+import unittest
+import os
+
+from pprint import pprint
+import semantic_version
+
+from catalog_test_util import CatalogTestUtil
+from biokbase.catalog.Impl import Catalog
+
+
+# tests all the basic get methods
+class StartupTest(unittest.TestCase):
+
+
+    def test_startups(self):
+
+        #Test normal startup, should work
+        self.cUtil.setUp()
+        catalog = Catalog(self.cUtil.getCatalogConfig())
+        self.assertTrue(semantic_version.validate(catalog.version(self.cUtil.anonymous_ctx())[0]))
+
+        #Test empty startup without DB version should work
+        self.cUtil.setUpEmpty()
+        catalog = Catalog(self.cUtil.getCatalogConfig())
+        self.assertTrue(semantic_version.validate(catalog.version(self.cUtil.anonymous_ctx())[0]))
+
+        #Test empty startup with several different valid versions should work
+        self.cUtil.setUpEmpty(db_version=3)
+        catalog = Catalog(self.cUtil.getCatalogConfig())
+        self.assertTrue(semantic_version.validate(catalog.version(self.cUtil.anonymous_ctx())[0]))
+        self.cUtil.setUpEmpty(db_version=4)
+        catalog = Catalog(self.cUtil.getCatalogConfig())
+        self.assertTrue(semantic_version.validate(catalog.version(self.cUtil.anonymous_ctx())[0]))
+
+        #Startup with version that is too high should fail
+        self.cUtil.setUpEmpty(db_version=2525)
+
+        catalog = None
+        with self.assertRaises(IOError) as e:
+            catalog = Catalog(self.cUtil.getCatalogConfig())
+        self.assertEqual(str(e.exception),
+            'Incompatible DB versions.  Expecting DB V4, found DV V2525. You are probably running an old version of the service.  Start up failed.');
+
+
+    @classmethod
+    def setUpClass(cls):
+        print('++++++++++++ RUNNING startup_test.py +++++++++++')
+        cls.cUtil = CatalogTestUtil('.') # TODO: pass in test directory from outside
+
+    @classmethod
+    def tearDownClass(cls):
+        pass
+
+
+
+