When upgrading, find upgrade paths to current versions of tables even…

… if they're unchanged, so that other upgraders can depend on them. Add support for loading upgraders from a migrations directory. Return upgrade SQL separate from debug prints, for easier export

schemup/schemup/commands.py

@@ -1,19 +1,15 @@
-from schemup import validator, upgraders, errors
+from schemup import loaders, validator, upgraders, errors
 
-def snapshot(dbSchema, ormSchema):
+
+def load(dirpath):
     """
-    Write current versions to DB schema table.
-    Used only to initialize schemup on an existing DB
+    Load upgraders from Python or YAML files in the given
+    directory.
     """
-
-    dbSchema.clearSchemaTable()
-
-    for tableName, version in ormSchema.getExpectedTableVersions():
-        dbSchema.setSchema(tableName, version)
-
-    dbSchema.commit()
-
+    for dirname, filename, loader in loaders.find(dirpath):
+        loader(dirname, filename)
 
+
 
 def validate(dbSchema, ormSchema):
     """
@@ -40,9 +36,7 @@ def upgrade(dbSchema, ormSchema):
     """
     Attempt to find upgrade paths for all out-of-sync tables,
      and run them.
-    """
-
-    import pprint
+    """    
 
     paths = [(tableName, upgraders.findUpgradePath(tableName, fromVersion, toVersion))
              for (tableName, fromVersion, toVersion)
@@ -51,14 +45,35 @@ def upgrade(dbSchema, ormSchema):
     if not paths:
         return
 
+    for tableName, currentVersion in dbSchema.getTableVersions():
+        paths.append((tableName, upgraders.pathToCurrent(tableName, currentVersion)))
+
     stepGraph = upgraders.UpgradeStepGraph()
 
     for tableName, path in paths:
         path.addToGraph(stepGraph)
 
     stepGraph.calculateEdges()
 
+    dbSchema.begin()
+
     for upgrader in stepGraph.topologicalSort():
         upgrader.run(dbSchema)
 
     dbSchema.commit()
+
+    return dbSchema.flushLog()
+
+
+def snapshot(dbSchema, ormSchema):
+    """
+    Write current versions to DB schema table.
+    Used only to initialize schemup on an existing DB
+    """
+
+    dbSchema.clearSchemaTable()
+
+    for tableName, version in ormSchema.getExpectedTableVersions():
+        dbSchema.setSchema(tableName, version)
+
+    dbSchema.commit()

schemup/schemup/dbs/postgres.py

@@ -24,10 +24,15 @@ def printLog(self):
         for line in self.flushLog():
             print line
 
+    def begin(self):
+        if self.dryRun:
+            self.runLog.append("START TRANSACTION")
+
     def commit(self):
         if self.dryRun:
-            return
-        self.conn.commit()
+            self.runLog.append("COMMIT")
+        else:
+            self.conn.commit()
 
     def ensureSchemaTable(self):
         cur = self.conn.cursor()

schemup/schemup/loaders.py

@@ -0,0 +1,22 @@
+import sys, os, os.path
+
+def find(dirpath):
+    for dirname, dirnames, filenames in os.walk(dirpath):
+        for filename in filenames:
+            loader = fileLoaders.get(os.path.splitext(filename)[1])
+            if loader is not None:
+                yield dirname, filename, loader
+
+
+def loadPython(dirname, filename):
+    print "Importing", dirname, filename
+    sys.path.insert(0, dirname)
+    __import__(filename.rstrip(".py"))
+    sys.path.pop(0)
+
+
+
+fileLoaders = {
+    '.py': loadPython,
+    '.yaml': lambda x, y: None
+}

schemup/schemup/upgraders.py

@@ -20,13 +20,9 @@ def run(self, dbSchema):
         if self.upgrader is None:
             return
 
-        print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
-        print "Upgrading %s" % self.tableName
-        print "%s => %s" % (self.fromVersion, self.toVersion)
+        print "Upgrading %s (%s => %s)" % (self.tableName, self.fromVersion, self.toVersion)
         self.upgrader(dbSchema)
         dbSchema.setSchema(self.tableName, self.toVersion, log=False)
-        dbSchema.printLog()
-
 
     def __repr__(self):
         return "<%s: %s => %s (%s)>" % (
@@ -65,18 +61,20 @@ def decorate(function):
 
 # ------------------------------------------------------------
 
+def pathToCurrent(tableName, currentVersion):
+    path = UpgradePath([])
+    for upgrader in findUpgradePath(tableName, None, currentVersion).steps:
+        stubUpgrader = upgrader.copy()
+        stubUpgrader.upgrader = None
+        path.push(stubUpgrader)
+    return path
+
+
 def findUpgradePath(tableName, fromVersion, toVersion):
     upgraders = registeredUpgraders.get(tableName, {})
 
-    initialPath = UpgradePath([])
-
-    if fromVersion is None:
-        initialPath.push(Upgrader(tableName, None, fromVersion, None))
-    else:
-        for upgrader in findUpgradePath(tableName, None, fromVersion).steps:
-            stubUpgrader = upgrader.copy()
-            stubUpgrader.upgrader = None
-            initialPath.push(stubUpgrader)
+    stubUpgrader = Upgrader(tableName, None, fromVersion, None)
+    initialPath = UpgradePath([ stubUpgrader ])
 
     paths = deque([ initialPath ])