VT Bundle: Make database subsystem more robust

scripts/create_db.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+###############################################################################
+##
+## Copyright (C) 2014-2015, New York University.
+## Copyright (C) 2011-2014, NYU-Poly.
+## Copyright (C) 2006-2011, University of Utah.
+## All rights reserved.
+## Contact: contact@vistrails.org
+##
+## This file is part of VisTrails.
+##
+## "Redistribution and use in source and binary forms, with or without
+## modification, are permitted provided that the following conditions are met:
+##
+##  - Redistributions of source code must retain the above copyright notice,
+##    this list of conditions and the following disclaimer.
+##  - Redistributions in binary form must reproduce the above copyright
+##    notice, this list of conditions and the following disclaimer in the
+##    documentation and/or other materials provided with the distribution.
+##  - Neither the name of the New York University nor the names of its
+##    contributors may be used to endorse or promote products derived from
+##    this software without specific prior written permission.
+##
+## THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+## AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+## THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+## PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+## CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+## EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+## PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+## OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+## WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+## OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+## ADVISED OF THE POSSIBILITY OF SUCH DAMAGE."
+##
+###############################################################################
+import os
+import sys
+import tempfile
+sys.path.append(os.path.dirname(os.path.dirname(
+                os.path.abspath(__file__))))
+from vistrails.db.services import io
+from db_utils import parse_db_cmd_line
+
+def create_db(config, version=None):
+    db_connection = io.default_open_db_connection(config)
+    clean = False
+    try:
+        io.get_db_version_from_db(db_connection, True)
+    except Exception:
+        # asssume we don't have anything in the database
+        clean = True
+    if not clean:
+        raise Exception("Database already exists. Delete or upgrade it")
+
+    if version is not None:
+        config['version'] = version
+    db_connection = io.open_db_connection(config)
+    io.create_db_tables(db_connection)
+    io.close_db_connection(db_connection)
+
+if __name__ == '__main__':
+    import vistrails.core.application
+
+    more_options = {'v:': ('schema version', False, 'version'),
+                    }
+    config, options = parse_db_cmd_line(sys.argv, more_options)
+    version = None
+    if options['v']:
+        version = options['v']
+
+    vistrails.core.application.init()
+    create_db(config, version)

scripts/delete_from_db.py

@@ -36,8 +36,8 @@
 ###############################################################################
 import os
 import sys
-sys.path.append(os.path.join(os.path.dirname(os.path.dirname(
-                os.path.abspath(__file__))), 'vistrails'))
+sys.path.append(os.path.dirname(os.path.dirname(
+                os.path.abspath(__file__))))
 from vistrails.db.services import io
 from db_utils import parse_db_cmd_line
 

scripts/merge_vistrails.py

@@ -53,9 +53,9 @@ def main(out_fname, in_fnames):
 
     """
     # FIXME this breaks when you use abstractions!
-    (save_bundle, vt_save_dir) = io.open_bundle_from_zip_xml(DBVistrail.vtType, in_fnames[0])
+    (save_bundle, vt_save_dir) = io.open_bundle_from_zip_xml(in_fnames[0])
     for in_fname in in_fnames[1:]:
-        (new_save_bundle, new_save_dir) = io.open_bundle_from_zip_xml(DBVistrail.vtType, in_fname)
+        (new_save_bundle, new_save_dir) = io.open_bundle_from_zip_xml(in_fname)
         vistrails.db.services.vistrail.merge(save_bundle, new_save_bundle, "", True, vt_save_dir, new_save_dir)
     io.save_bundle_to_zip_xml(save_bundle, out_fname)
 

scripts/update_db.py

@@ -38,8 +38,8 @@
 import os
 import sys
 import tempfile
-sys.path.append(os.path.join(os.path.dirname(os.path.dirname(
-                os.path.abspath(__file__))), 'vistrails'))
+sys.path.append(os.path.dirname(os.path.dirname(
+                os.path.abspath(__file__))))
 from vistrails.db.services import io
 from vistrails.db.versions import currentVersion
 from db_utils import parse_db_cmd_line
@@ -67,7 +67,8 @@ def update_db(config, new_version=None, tmp_dir=None, restore=False):
                     filenames.append(os.path.join(restore, fname))
     else:
         for obj_type in obj_types:
-            obj_id_lists[obj_type] = io.get_db_object_list(config, obj_type)
+            obj_id_lists[obj_type] = io.get_db_object_list(db_connection, 
+                                                           obj_type)
 
         # read data out of database
         thumbnail_dir = os.path.join(tmp_dir, 'thumbs')
@@ -92,12 +93,17 @@ def update_db(config, new_version=None, tmp_dir=None, restore=False):
                 io.save_vistrail_bundle_to_zip_xml(res, vt_name, local_tmp_dir)
 
         # drop the old database
+        io.drop_db_tables(db_connection)
+        io.close_db_connection(db_connection)
+
         # recreate with the new version of the specs
-        io.setup_db_tables(db_connection, None, old_version)
+        # we close and re-open the connection because the schema may change
+        db_connection = io.open_db_connection(config)
+        io.create_db_tables(db_connection)
 
     # add the new data back
     for filename in filenames:
-        (res, _) = io.open_vistrail_bundle_from_zip_xml(filename)
+        res = io.open_vistrail_bundle_from_zip_xml(filename)
         try:
             io.save_vistrail_bundle_to_db(res, db_connection, 'with_ids')
         except Exception, e:
@@ -116,7 +122,11 @@ def update_db(config, new_version=None, tmp_dir=None, restore=False):
     new_version = None
     if options['v']:
         new_version = options['v']
+    if options['d']:
+        tmp_dir = options['d']
+    else:
+        tmp_dir = None
 
     vistrails.core.application.init()
 
-    update_db(config, new_version, options['d'], options['e']) 
+    update_db(config, new_version, tmp_dir, options['e'])

vistrails/__init__.py

@@ -54,5 +54,5 @@
                      "upgrade Python or\ninstall unittest2, else VisTrails "
                      "might fail because of missing symbols.\n")
 
-
-from vistrails.core.api import *
+# FIXME circular import in tests since tests import vistrails.core.debug
+# from vistrails.core.api import *

vistrails/core/application.py

@@ -271,29 +271,18 @@ def process_interactive_input(self):
                         pe = version
                         version = None
                     else:
-                        if hasattr(locator, '_vnode') and \
-                                locator._vnode is not None:
-                            version = locator._vnode
-                        if hasattr(locator,'_vtag'):
-                            # if a tag is set, it should be used instead of the
-                            # version number
-                            if locator._vtag != '':
-                                version = locator._vtag
+                        version = locator.version
                     execute = self.temp_configuration.check('execute')
-                    mashuptrail = None
-                    mashupversion = None
-                    if hasattr(locator, '_mshptrail'):
-                        mashuptrail = locator._mshptrail
-                    if hasattr(locator, '_mshpversion'):
-                        mashupversion = locator._mshpversion
-                        if mashupversion:
-                            execute = True
+                    mashup_trail = locator.mashuptrail
+                    mashup_version = locator.mashupVersion
+                    if mashup_version:
+                        execute = True
                     if self.temp_configuration.showWindow:
                         self.showBuilderWindow()
                     self.builderWindow.open_vistrail_without_prompt(locator,
                                                                     version, execute,
-                                                                    mashuptrail=mashuptrail, 
-                                                                    mashupVersion=mashupversion)
+                                                                    mashuptrail=mashup_trail, 
+                                                                    mashupVersion=mashup_version)
 
                     if self.temp_configuration.check('parameterExploration'):
                         self.builderWindow.executeParameterExploration(pe)
@@ -410,9 +399,8 @@ def open_vistrail(self, locator=None, version=None, is_abstraction=False):
         if controller is None:
             # vistrail is not already open
             try:
-                loaded_objs = vistrails.core.db.io.load_vistrail(locator, is_abstraction)
-                controller = self.add_vistrail(loaded_objs[0], locator, 
-                                               *loaded_objs[1:])
+                bundle = vistrails.core.db.io.load_vistrail(locator, False)
+                controller = self.add_vistrail(bundle, locator)
                 if locator.is_untitled():
                     return controller
                 controller.is_abstraction = is_abstraction

vistrails/core/collection/__init__.py

@@ -291,6 +291,29 @@ def create_vistrail_entity(self, vistrail):
         self.add_entity(entity)
         return entity
 
+    def update_from_database(self, db_locator):
+        # db_conn = db_locator.get_connection()
+        config = {'host': db_locator._host,
+                  'port': int(db_locator._port),
+                  'db': db_locator._db,
+                  'user': db_locator._user,
+                  'passwd': db_locator._passwd}
+        db_connection = vistrails.db.services.io.open_db_connection(config)
+        rows = vistrails.db.services.io.get_db_object_list(db_connection, 
+                                                           'vistrail')
+        vistrails.db.services.io.close_db_connection(db_connection)
+        for row in rows:
+            if row[0] in [1,]:
+                continue
+            locator = DBLocator(config['host'], config['port'], config['db'],
+                                config['user'], config['passwd'],
+                                obj_type='vistrail', obj_id=row[0])
+            (vistrail, abstractions, thumbnails, mashups) = load_vistrail(locator)
+            vistrail.abstractions = abstractions
+            vistrail.thumbnails = thumbnails
+            vistrail.mashups = mashups
+            self.create_vistrail_entity(vistrail)
+
     def update_from_directory(self, directory):
         filenames = glob.glob(os.path.join(directory, '*.vt'))
         for filename in filenames:
@@ -331,10 +354,10 @@ def updateVistrail(self, url, vistrail=None):
         locator = BaseLocator.from_url(url)
         if locator.is_valid():
             if not vistrail:
-                (vistrail, abstractions, thumbnails, mashups) = load_vistrail(locator)
-                vistrail.abstractions = abstractions
-                vistrail.thumbnails = thumbnails
-                vistrail.mashups = mashups
+                bundle = load_vistrail(locator)
+                vistrail.abstractions = [a.obj for a in bundle.abstractions]
+                vistrail.thumbnails = [t.obj for t in bundle.thumbnails]
+                vistrail.mashups = [m.obj for m in bundle.mashups]
             entity = self.create_vistrail_entity(vistrail)
             for p in workspaces:
                 self.add_to_workspace(entity, p)

vistrails/core/collection/vistrail.py

@@ -105,7 +105,7 @@ def create_mashup_entity(self, trail_id, mashup, action):
         entity.create_time = action.db_date
         locator = BaseLocator.from_url(self.url)
         locator.kwargs['mashuptrail'] = trail_id
-        locator.kwargs['mashup'] = action.id
+        locator.kwargs['mashupVersion'] = action.id
         entity.url = locator.to_url()
         return entity
 

vistrails/core/configuration.py

@@ -62,6 +62,7 @@
 _usage_args = set()
 
 _simple_documentation = """
+addBundleData: Add data files to vistrail bundle
 autoConnect: Automatically connect dragged in modules
 autoSave: Automatically save backup vistrails every two minutes
 batch: Run in batch mode instead of interactive mode
@@ -72,6 +73,7 @@
 dbDefault: Save vistrails in a database by default
 debugLevel: How much information should VisTrails log
 defaultFileType: Default file type/extension for vistrails (.vt or .xml)
+deleteBundleData: Delete data files in vistrail bundle
 detachHistoryView: Show the version tree in a separate window
 dotVistrails: User configuration directory
 enablePackagesSilently: Automatically enable packages when needed
@@ -93,6 +95,7 @@
 jobList: List running workflows
 jobInfo: List jobs in running workflow
 loadPackages: Whether to load the packages enabled in the configuration file
+listBundleData: List data files in vistrail bundle
 logDir: Log files directory
 maxRecentVistrails: Number of recent vistrails
 maximizeWindows: VisTrails windows should be maximized
@@ -124,7 +127,6 @@
 showInlineParameterWidgets: Show editable parameters inside modules
 showScrollbars: Show scrollbars on the version tree and workflow canvases
 showSplash: Show VisTrails splash screen during startup
-showSpreadsheetOnly: Hides the VisTrails main window
 showVariantErrors: Show error when variant input value doesn't match type during execution
 showVistrailsNews: Show news from VisTrails (once per message)
 showWindow: Show the main window
@@ -152,6 +154,11 @@
 """
 
 _documentation = """
+addBundleData: Boolean
+
+    Add data files to vistrail bundle. Specify the file to add
+    or a directory to add files from.
+
 autoConnect: Boolean
 
     Try to automatically connect a newly dragged in module to the rest
@@ -178,6 +185,11 @@
 
     The location that VisTrails uses as a default directory for data.
 
+deleteBundleData: Boolean
+
+    Delete data files in vistrail bundle. Specify file or
+    a directory to delete all files in that directory.
+
 db: String
 
     The name for the database to load the vistrail from.
@@ -285,6 +297,11 @@
 
     Whether to load the packages enabled in the configuration file.
 
+listBundleData:
+
+   List data files in vistrail bundle. Specify path to
+   only list files matching that path.
+
 logDir: Path
 
     The path that indicates where log files should be stored.
@@ -465,10 +482,6 @@
 
     Whether the VisTrails splash screen should be shown on startup.
 
-showSpreadsheetOnly: Boolean
-
-    Whether the VisTrails main window should be hidden.
-
 showVariantErrors: Boolean
 
     Alert the user if the value along a connection coming from a
@@ -804,6 +817,10 @@ def __init__(self, name, sub_fields):
      ConfigField('jobAutorun', False, bool),
      ConfigField('jobList', False, bool, ConfigType.COMMAND_LINE_FLAG),
      ConfigField('jobInfo', False, bool, ConfigType.COMMAND_LINE_FLAG)],
+    "BundleData":
+    [ConfigField("addBundleData", False, bool, ConfigType.COMMAND_LINE_FLAG),
+     ConfigField("deleteBundleData", False, bool, ConfigType.COMMAND_LINE_FLAG),
+     ConfigField("listBundleData", False, bool, ConfigType.COMMAND_LINE_FLAG)],
 }
 
 # FIXME make sure that the platform-specific configs are added!