Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

First phase of loading migrations from disk

  • Loading branch information...
commit 9ce83546720b9536c02817e802c9376eb74f811d 1 parent cb4b0de
Andrew Godwin authored May 10, 2013
1  django/db/migrations/__init__.py
... ...
@@ -0,0 +1 @@
  1
+from .migration import Migration
11  django/db/migrations/graph.py
... ...
@@ -1,7 +1,7 @@
1 1
 from django.utils.datastructures import SortedSet
2 2
 
3 3
 
4  
-class MigrationsGraph(object):
  4
+class MigrationGraph(object):
5 5
     """
6 6
     Represents the digraph of all migrations in a project.
7 7
 
@@ -19,7 +19,7 @@ class MigrationsGraph(object):
19 19
     replacing migration, and repoint any dependencies that pointed to the
20 20
     replaced migrations to point to the replacing one.
21 21
 
22  
-    A node should be a tuple: (applabel, migration_name) - but the code
  22
+    A node should be a tuple: (app_path, migration_name) - but the code
23 23
     here doesn't really care.
24 24
     """
25 25
 
@@ -70,7 +70,7 @@ def _dfs(start, get_children, path):
70 70
                 return cache[(start, get_children)]
71 71
             # If we've traversed here before, that's a circular dep
72 72
             if start in path:
73  
-                raise CircularDependencyException(path[path.index(start):] + [start])
  73
+                raise CircularDependencyError(path[path.index(start):] + [start])
74 74
             # Build our own results list, starting with us
75 75
             results = []
76 76
             results.append(start)
@@ -88,8 +88,11 @@ def _dfs(start, get_children, path):
88 88
             return results
89 89
         return _dfs(start, get_children, [])
90 90
 
  91
+    def __str__(self):
  92
+        return "Graph: %s nodes, %s edges" % (len(self.nodes), sum(len(x) for x in self.dependencies.values()))
91 93
 
92  
-class CircularDependencyException(Exception):
  94
+
  95
+class CircularDependencyError(Exception):
93 96
     """
94 97
     Raised when there's an impossible-to-resolve circular dependency.
95 98
     """
128  django/db/migrations/loader.py
... ...
@@ -0,0 +1,128 @@
  1
+import os
  2
+from django.utils.importlib import import_module
  3
+from django.db.models.loading import cache
  4
+from django.db.migrations.recorder import MigrationRecorder
  5
+from django.db.migrations.graph import MigrationGraph
  6
+
  7
+
  8
+class MigrationLoader(object):
  9
+    """
  10
+    Loads migration files from disk, and their status from the database.
  11
+
  12
+    Migration files are expected to live in the "migrations" directory of
  13
+    an app. Their names are entirely unimportant from a code perspective,
  14
+    but will probably follow the 1234_name.py convention.
  15
+
  16
+    On initialisation, this class will scan those directories, and open and
  17
+    read the python files, looking for a class called Migration, which should
  18
+    inherit from django.db.migrations.Migration. See
  19
+    django.db.migrations.migration for what that looks like.
  20
+
  21
+    Some migrations will be marked as "replacing" another set of migrations.
  22
+    These are loaded into a separate set of migrations away from the main ones.
  23
+    If all the migrations they replace are either unapplied or missing from
  24
+    disk, then they are injected into the main set, replacing the named migrations.
  25
+    Any dependency pointers to the replaced migrations are re-pointed to the
  26
+    new migration.
  27
+
  28
+    This does mean that this class MUST also talk to the database as well as
  29
+    to disk, but this is probably fine. We're already not just operating
  30
+    in memory.
  31
+    """
  32
+
  33
+    def __init__(self, connection):
  34
+        self.connection = connection
  35
+        self.disk_migrations = None
  36
+        self.applied_migrations = None
  37
+
  38
+    def load_disk(self):
  39
+        """
  40
+        Loads the migrations from all INSTALLED_APPS from disk.
  41
+        """
  42
+        self.disk_migrations = {}
  43
+        for app in cache.get_apps():
  44
+            # Get the migrations module directory
  45
+            module_name = ".".join(app.__name__.split(".")[:-1] + ["migrations"])
  46
+            app_label = module_name.split(".")[-2]
  47
+            try:
  48
+                module = import_module(module_name)
  49
+            except ImportError as e:
  50
+                # I hate doing this, but I don't want to squash other import errors.
  51
+                # Might be better to try a directory check directly.
  52
+                if "No module named migrations" in str(e):
  53
+                    continue
  54
+            directory = os.path.dirname(module.__file__)
  55
+            # Scan for .py[c|o] files
  56
+            migration_names = set()
  57
+            for name in os.listdir(directory):
  58
+                if name.endswith(".py") or name.endswith(".pyc") or name.endswith(".pyo"):
  59
+                    import_name = name.rsplit(".", 1)[0]
  60
+                    if import_name[0] not in "_.~":
  61
+                        migration_names.add(import_name)
  62
+            # Load them
  63
+            for migration_name in migration_names:
  64
+                migration_module = import_module("%s.%s" % (module_name, migration_name))
  65
+                if not hasattr(migration_module, "Migration"):
  66
+                    raise BadMigrationError("Migration %s in app %s has no Migration class" % (migration_name, app_label))
  67
+                self.disk_migrations[app_label, migration_name] = migration_module.Migration
  68
+
  69
+    def build_graph(self):
  70
+        """
  71
+        Builds a migration dependency graph using both the disk and database.
  72
+        """
  73
+        # Make sure we have the disk data
  74
+        if self.disk_migrations is None:
  75
+            self.load_disk()
  76
+        # And the database data
  77
+        if self.applied_migrations is None:
  78
+            recorder = MigrationRecorder(self.connection)
  79
+            self.applied_migrations = recorder.applied_migrations()
  80
+        # Do a first pass to separate out replacing and non-replacing migrations
  81
+        normal = {}
  82
+        replacing = {}
  83
+        for key, migration in self.disk_migrations.items():
  84
+            if migration.replaces:
  85
+                replacing[key] = migration
  86
+            else:
  87
+                normal[key] = migration
  88
+        # Calculate reverse dependencies - i.e., for each migration, what depends on it?
  89
+        # This is just for dependency re-pointing when applying replacements,
  90
+        # so we ignore run_before here.
  91
+        reverse_dependencies = {}
  92
+        for key, migration in normal.items():
  93
+            for parent in migration.dependencies:
  94
+                reverse_dependencies.setdefault(parent, set()).add(key)
  95
+        # Carry out replacements if we can - that is, if all replaced migrations
  96
+        # are either unapplied or missing.
  97
+        for key, migration in replacing.items():
  98
+            # Do the check
  99
+            can_replace = True
  100
+            for target in migration.replaces:
  101
+                if target in self.applied_migrations:
  102
+                    can_replace = False
  103
+                    break
  104
+            if not can_replace:
  105
+                continue
  106
+            # Alright, time to replace. Step through the replaced migrations
  107
+            # and remove, repointing dependencies if needs be.
  108
+            for replaced in migration.replaces:
  109
+                if replaced in normal:
  110
+                    del normal[replaced]
  111
+                for child_key in reverse_dependencies.get(replaced, set()):
  112
+                    normal[child_key].dependencies.remove(replaced)
  113
+                    normal[child_key].dependencies.append(key)
  114
+            normal[key] = migration
  115
+        # Finally, make a graph and load everything into it
  116
+        graph = MigrationGraph()
  117
+        for key, migration in normal.items():
  118
+            graph.add_node(key, migration)
  119
+            for parent in migration.dependencies:
  120
+                graph.add_dependency(key, parent)
  121
+        return graph
  122
+
  123
+
  124
+class BadMigrationError(Exception):
  125
+    """
  126
+    Raised when there's a bad migration (unreadable/bad format/etc.)
  127
+    """
  128
+    pass
30  django/db/migrations/migration.py
... ...
@@ -0,0 +1,30 @@
  1
+class Migration(object):
  2
+    """
  3
+    The base class for all migrations.
  4
+
  5
+    Migration files will import this from django.db.migrations.Migration
  6
+    and subclass it as a class called Migration. It will have one or more
  7
+    of the following attributes:
  8
+
  9
+     - operations: A list of Operation instances, probably from django.db.migrations.operations
  10
+     - dependencies: A list of tuples of (app_path, migration_name)
  11
+     - run_before: A list of tuples of (app_path, migration_name)
  12
+     - replaces: A list of migration_names
  13
+    """
  14
+
  15
+    # Operations to apply during this migration, in order.
  16
+    operations = []
  17
+
  18
+    # Other migrations that should be run before this migration.
  19
+    # Should be a list of (app, migration_name).
  20
+    dependencies = []
  21
+
  22
+    # Other migrations that should be run after this one (i.e. have
  23
+    # this migration added to their dependencies). Useful to make third-party
  24
+    # apps' migrations run after your AUTH_USER replacement, for example.
  25
+    run_before = []
  26
+
  27
+    # Migration names in this app that this migration replaces. If this is
  28
+    # non-empty, this migration will only be applied if all these migrations
  29
+    # are not applied.
  30
+    replaces = []
64  django/db/migrations/recorder.py
... ...
@@ -0,0 +1,64 @@
  1
+import datetime
  2
+from django.db import models
  3
+from django.db.models.loading import BaseAppCache
  4
+
  5
+
  6
+class MigrationRecorder(object):
  7
+    """
  8
+    Deals with storing migration records in the database.
  9
+
  10
+    Because this table is actually itself used for dealing with model
  11
+    creation, it's the one thing we can't do normally via syncdb or migrations.
  12
+    We manually handle table creation/schema updating (using schema backend)
  13
+    and then have a floating model to do queries with.
  14
+
  15
+    If a migration is unapplied its row is removed from the table. Having
  16
+    a row in the table always means a migration is applied.
  17
+    """
  18
+
  19
+    class Migration(models.Model):
  20
+        app = models.CharField(max_length=255)
  21
+        name = models.CharField(max_length=255)
  22
+        applied = models.DateTimeField(default=datetime.datetime.utcnow)
  23
+        class Meta:
  24
+            app_cache = BaseAppCache()
  25
+            app_label = "migrations"
  26
+            db_table = "django_migrations"
  27
+
  28
+    def __init__(self, connection):
  29
+        self.connection = connection
  30
+
  31
+    def ensure_schema(self):
  32
+        """
  33
+        Ensures the table exists and has the correct schema.
  34
+        """
  35
+        # If the table's there, that's fine - we've never changed its schema
  36
+        # in the codebase.
  37
+        if self.Migration._meta.db_table in self.connection.introspection.get_table_list(self.connection.cursor()):
  38
+            return
  39
+        # Make the table
  40
+        editor = self.connection.schema_editor()
  41
+        editor.start()
  42
+        editor.create_model(self.Migration)
  43
+        editor.commit()
  44
+
  45
+    def applied_migrations(self):
  46
+        """
  47
+        Returns a set of (app, name) of applied migrations.
  48
+        """
  49
+        self.ensure_schema()
  50
+        return set(tuple(x) for x in self.Migration.objects.values_list("app", "name"))
  51
+
  52
+    def record_applied(self, app, name):
  53
+        """
  54
+        Records that a migration was applied.
  55
+        """
  56
+        self.ensure_schema()
  57
+        self.Migration.objects.create(app=app, name=name)
  58
+
  59
+    def record_unapplied(self, app, name):
  60
+        """
  61
+        Records that a migration was unapplied.
  62
+        """
  63
+        self.ensure_schema()
  64
+        self.Migration.objects.filter(app=app, name=name).delete()
5  tests/migrations/migrations/0001_initial.py
... ...
@@ -0,0 +1,5 @@
  1
+from django.db import migrations
  2
+
  3
+
  4
+class Migration(migrations.Migration):
  5
+    pass
6  tests/migrations/migrations/0002_second.py
... ...
@@ -0,0 +1,6 @@
  1
+from django.db import migrations
  2
+
  3
+
  4
+class Migration(migrations.Migration):
  5
+
  6
+    dependencies = [("migrations", "0001_initial")]
0  tests/migrations/migrations/__init__.py
No changes.
29  tests/migrations/tests.py
... ...
@@ -1,5 +1,7 @@
1 1
 from django.test import TransactionTestCase
2  
-from django.db.migrations.graph import MigrationsGraph, CircularDependencyException
  2
+from django.db import connection
  3
+from django.db.migrations.graph import MigrationGraph, CircularDependencyError
  4
+from django.db.migrations.loader import MigrationLoader
3 5
 
4 6
 
5 7
 class GraphTests(TransactionTestCase):
@@ -16,7 +18,7 @@ def test_simple_graph(self):
16 18
         app_b:  0001 <-- 0002 <-/
17 19
         """
18 20
         # Build graph
19  
-        graph = MigrationsGraph()
  21
+        graph = MigrationGraph()
20 22
         graph.add_dependency(("app_a", "0004"), ("app_a", "0003"))
21 23
         graph.add_dependency(("app_a", "0003"), ("app_a", "0002"))
22 24
         graph.add_dependency(("app_a", "0002"), ("app_a", "0001"))
@@ -54,7 +56,7 @@ def test_complex_graph(self):
54 56
         app_c:         \ 0001 <-- 0002 <-
55 57
         """
56 58
         # Build graph
57  
-        graph = MigrationsGraph()
  59
+        graph = MigrationGraph()
58 60
         graph.add_dependency(("app_a", "0004"), ("app_a", "0003"))
59 61
         graph.add_dependency(("app_a", "0003"), ("app_a", "0002"))
60 62
         graph.add_dependency(("app_a", "0002"), ("app_a", "0001"))
@@ -85,7 +87,7 @@ def test_circular_graph(self):
85 87
         Tests a circular dependency graph.
86 88
         """
87 89
         # Build graph
88  
-        graph = MigrationsGraph()
  90
+        graph = MigrationGraph()
89 91
         graph.add_dependency(("app_a", "0003"), ("app_a", "0002"))
90 92
         graph.add_dependency(("app_a", "0002"), ("app_a", "0001"))
91 93
         graph.add_dependency(("app_a", "0001"), ("app_b", "0002"))
@@ -93,6 +95,23 @@ def test_circular_graph(self):
93 95
         graph.add_dependency(("app_b", "0001"), ("app_a", "0003"))
94 96
         # Test whole graph
95 97
         self.assertRaises(
96  
-            CircularDependencyException,
  98
+            CircularDependencyError,
97 99
             graph.forwards_plan, ("app_a", "0003"),
98 100
         )
  101
+
  102
+
  103
+class LoaderTests(TransactionTestCase):
  104
+    """
  105
+    Tests the disk and database loader.
  106
+    """
  107
+
  108
+    def test_load(self):
  109
+        """
  110
+        Makes sure the loader can load the migrations for the test apps.
  111
+        """
  112
+        migration_loader = MigrationLoader(connection)
  113
+        graph = migration_loader.build_graph()
  114
+        self.assertEqual(
  115
+            graph.forwards_plan(("migrations", "0002_second")),
  116
+            [("migrations", "0001_initial"), ("migrations", "0002_second")],
  117
+        )

0 notes on commit 9ce8354

Please sign in to comment.
Something went wrong with that request. Please try again.