Improvements to database initialization.

- Use sqlalchemy-utils to create postgres / mysql database if they don't exist the way it happens automatically for sqlite.
- Add a new configuration parameter called database_template that allows creating postgres databases from an existing database using a TEMPLATE statement.
- Use the database template parameter to give each integration test case class a clean postgres database.

config/galaxy.ini.sample

@@ -118,6 +118,11 @@ paste.app_factory = galaxy.web.buildapp:app_factory
 # 'galaxy.model.orm.logging_connection_proxy'
 #database_query_profiling_proxy = False
 
+# If auto-creating a postgres database on startup - it can be based on an existing
+# template database. This will set that. This is probably only useful for testing but
+# documentation is included here for completeness.
+#database_template = None
+
 # Slow query logging.  Queries slower than the threshold indicated below will
 # be logged to debug.  A value of '0' is disabled.  For example, you would set
 # this to .005 to log all queries taking longer than 5 milliseconds.

lib/galaxy/config.py

@@ -132,6 +132,7 @@ def __init__(self, **kwargs):
         self.database_engine_options = get_database_engine_options(kwargs)
         self.database_create_tables = string_as_bool(kwargs.get("database_create_tables", "True"))
         self.database_query_profiling_proxy = string_as_bool(kwargs.get("database_query_profiling_proxy", "False"))
+        self.database_template = kwargs.get("database_template", None)
         self.slow_query_log_threshold = float(kwargs.get("slow_query_log_threshold", 0))
 
         # Don't set this to true for production databases, but probably should

lib/galaxy/dependencies/pinned-requirements.txt

@@ -3,6 +3,7 @@ bx-python==0.7.3
 MarkupSafe==0.23
 PyYAML==3.11
 SQLAlchemy==1.0.15
+sqlalchemy-utils==0.32.19
 mercurial==3.7.3
 numpy==1.9.2
 pycrypto==2.6.1

lib/galaxy/model/migrate/check.py

@@ -10,6 +10,7 @@
     Table
 )
 from sqlalchemy.exc import NoSuchTableError
+from sqlalchemy_utils import create_database, database_exists
 
 log = logging.getLogger(__name__)
 
@@ -29,6 +30,12 @@ def create_or_verify_database(url, galaxy_config_file, engine_options={}, app=No
     3) Database at state where migrate support introduced --> add version control information but make no changes (might still require manual update)
     4) Database versioned but out of date --> fail with informative message, user must run "sh manage_db.sh upgrade"
     """
+    # Create the base database if it doesn't yet exist.
+    new_database = not database_exists(url)
+    if new_database:
+        template = app and getattr(app.config, "database_template", None)
+        create_database(url, template=template)
+
     # Create engine and metadata
     engine = create_engine(url, **engine_options)
 
@@ -43,7 +50,7 @@ def migrate():
         migrate_to_current_version(engine, db_schema)
 
     meta = MetaData(bind=engine)
-    if app and getattr(app.config, 'database_auto_migrate', False):
+    if new_database or (app and getattr(app.config, 'database_auto_migrate', False)):
         migrate()
         return
 

test/base/driver_util.py

@@ -9,6 +9,7 @@
 import random
 import shutil
 import socket
+import string
 import struct
 import sys
 import tempfile
@@ -19,8 +20,11 @@
 import nose.core
 import nose.loader
 import nose.plugins.manager
+
 from paste import httpserver
 
+from six.moves.urllib.parse import urlparse
+
 from galaxy.app import UniverseApplication as GalaxyUniverseApplication
 from galaxy.util import asbool, download_to_file
 from galaxy.util.properties import load_app_properties
@@ -120,6 +124,7 @@ def setup_galaxy_config(
     shed_tool_conf=None,
     datatypes_conf=None,
     update_integrated_tool_panel=False,
+    prefer_template_database=False,
 ):
     """Setup environment and build config for test Galaxy instance."""
     if not os.path.exists(tmpdir):
@@ -209,7 +214,7 @@ def setup_galaxy_config(
         user_library_import_dir=user_library_import_dir,
         webhooks_dir=TEST_WEBHOOKS_DIR,
     )
-    config.update(database_conf(tmpdir))
+    config.update(database_conf(tmpdir, prefer_template_database=prefer_template_database))
     config.update(install_database_conf(tmpdir, default_merged=default_install_db_merged))
     if datatypes_conf is not None:
         config['datatypes_config_file'] = datatypes_conf
@@ -287,12 +292,21 @@ def copy_database_template(source, db_path):
         raise Exception("Failed to copy database template from source %s" % source)
 
 
-def database_conf(db_path, prefix="GALAXY"):
+def database_conf(db_path, prefix="GALAXY", prefer_template_database=False):
     """Find (and populate if needed) Galaxy database connection."""
     database_auto_migrate = False
     dburi_var = "%s_TEST_DBURI" % prefix
+    template_name = None
     if dburi_var in os.environ:
         database_connection = os.environ[dburi_var]
+        # only template if postgres - not mysql or sqlite
+        do_template = prefer_template_database and database_connection.startswith("p")
+        if do_template:
+            database_template_parsed = urlparse(database_connection)
+            template_name = database_template_parsed.path[1:]  # drop / from /galaxy
+            actual_db = "gxtest" + ''.join(random.choice(string.ascii_uppercase) for _ in range(10))
+            actual_database_parsed = database_template_parsed._replace(path="/%s" % actual_db)
+            database_connection = actual_database_parsed.geturl()
     else:
         default_db_filename = "%s.sqlite" % prefix.lower()
         template_var = "%s_TEST_DB_TEMPLATE" % prefix
@@ -313,6 +327,8 @@ def database_conf(db_path, prefix="GALAXY"):
     if not database_connection.startswith("sqlite://"):
         config["database_engine_option_max_overflow"] = "20"
         config["database_engine_option_pool_size"] = "10"
+    if template_name:
+        config["database_template"] = template_name
     return config
 
 
@@ -673,6 +689,7 @@ def setup(self, config_object=None):
                     default_install_db_merged=True,
                     default_tool_conf=default_tool_conf,
                     datatypes_conf=datatypes_conf_override,
+                    prefer_template_database=getattr(config_object, "prefer_template_database", False),
                 )
                 galaxy_config = setup_galaxy_config(
                     galaxy_db_path,

test/base/integration_util.py

@@ -15,6 +15,8 @@
 class IntegrationTestCase(TestCase, UsesApiTestCaseMixin):
     """Unit test case with utilities for spinning up Galaxy."""
 
+    prefer_template_database = True
+
     @classmethod
     def setUpClass(cls):
         """Configure and start Galaxy for a test."""