Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Rewrote portions of the app- and model-cache initialisation to handle…

… some corner cases. It is now possible to use m2m relations before everything is imported and still get the right results later when importing is complete. Also, get_apps() should always return the same results, so apps won't randomly disappear in the admin interface.

Also reorganised the structure of loading.py, since the number of global variables was exploding. The public API is still backwards compatible.

Fixed #1796 and #2438 (he claims, optimistically).


git-svn-id: http://code.djangoproject.com/svn/django/trunk@5919 bcc190cf-cafb-0310-a4f2-bffc1f526a37
  • Loading branch information...
commit 3219a60167b6d5670a6df0d2f5b9944951245322 1 parent 103fe15
Malcolm Tredinnick authored August 17, 2007
287  django/db/models/loading.py
@@ -4,113 +4,188 @@
4 4
 from django.core.exceptions import ImproperlyConfigured
5 5
 import sys
6 6
 import os
  7
+import threading
7 8
 
8  
-__all__ = ('get_apps', 'get_app', 'get_models', 'get_model', 'register_models')
9  
-
10  
-_app_list = []   # Cache of installed apps.
11  
-                 # Entry is not placed in app_list cache until entire app is loaded.
12  
-_app_models = {} # Dictionary of models against app label
13  
-                 # Each value is a dictionary of model name: model class
14  
-                 # Applabel and Model entry exists in cache when individual model is loaded.
15  
-_app_errors = {} # Dictionary of errors that were experienced when loading the INSTALLED_APPS
16  
-                 # Key is the app_name of the model, value is the exception that was raised
17  
-                 # during model loading.
18  
-_loaded = False  # Has the contents of settings.INSTALLED_APPS been loaded?
19  
-                 # i.e., has get_apps() been called?
20  
-
21  
-def get_apps():
22  
-    "Returns a list of all installed modules that contain models."
23  
-    global _app_list
24  
-    global _loaded
25  
-    if not _loaded:
26  
-        _loaded = True
27  
-        for app_name in settings.INSTALLED_APPS:
28  
-            try:
29  
-                load_app(app_name)
30  
-            except Exception, e:
31  
-                # Problem importing the app
32  
-                _app_errors[app_name] = e
33  
-    return _app_list
34  
-
35  
-def get_app(app_label, emptyOK=False):
36  
-    "Returns the module containing the models for the given app_label. If the app has no models in it and 'emptyOK' is True, returns None."
37  
-    get_apps() # Run get_apps() to populate the _app_list cache. Slightly hackish.
38  
-    for app_name in settings.INSTALLED_APPS:
39  
-        if app_label == app_name.split('.')[-1]:
40  
-            mod = load_app(app_name)
41  
-            if mod is None:
42  
-                if emptyOK:
43  
-                    return None
44  
-            else:
45  
-                return mod
46  
-    raise ImproperlyConfigured, "App with label %s could not be found" % app_label
47  
-
48  
-def load_app(app_name):
49  
-    "Loads the app with the provided fully qualified name, and returns the model module."
50  
-    global _app_list
51  
-    mod = __import__(app_name, {}, {}, ['models'])
52  
-    if not hasattr(mod, 'models'):
53  
-        return None
54  
-    if mod.models not in _app_list:
55  
-        _app_list.append(mod.models)
56  
-    return mod.models
57  
-
58  
-def get_app_errors():
59  
-    "Returns the map of known problems with the INSTALLED_APPS"
60  
-    global _app_errors
61  
-    get_apps() # Run get_apps() to populate the _app_list cache. Slightly hackish.
62  
-    return _app_errors
63  
-
64  
-def get_models(app_mod=None):
65  
-    """
66  
-    Given a module containing models, returns a list of the models. Otherwise
67  
-    returns a list of all installed models.
68  
-    """
69  
-    app_list = get_apps() # Run get_apps() to populate the _app_list cache. Slightly hackish.
70  
-    if app_mod:
71  
-        return _app_models.get(app_mod.__name__.split('.')[-2], {}).values()
72  
-    else:
73  
-        model_list = []
74  
-        for app_mod in app_list:
75  
-            model_list.extend(get_models(app_mod))
76  
-        return model_list
77  
-
78  
-def get_model(app_label, model_name, seed_cache=True):
79  
-    """
80  
-    Returns the model matching the given app_label and case-insensitive
81  
-    model_name.
  9
+__all__ = ('get_apps', 'get_app', 'get_models', 'get_model', 'register_models',
  10
+        'load_app', 'cache_ready')
82 11
 
83  
-    Returns None if no model is found.
  12
+class Cache(object):
84 13
     """
85  
-    if seed_cache:
86  
-        get_apps()
87  
-    try:
88  
-        model_dict = _app_models[app_label]
89  
-    except KeyError:
90  
-        return None
91  
-
92  
-    try:
93  
-        return model_dict[model_name.lower()]
94  
-    except KeyError:
95  
-        return None
96  
-
97  
-def register_models(app_label, *models):
  14
+    A cache that stores installed applications and their models. Used to
  15
+    provide reverse-relations and for app introspection (e.g. admin).
98 16
     """
99  
-    Register a set of models as belonging to an app.
100  
-    """
101  
-    for model in models:
102  
-        # Store as 'name: model' pair in a dictionary
103  
-        # in the _app_models dictionary
104  
-        model_name = model._meta.object_name.lower()
105  
-        model_dict = _app_models.setdefault(app_label, {})
106  
-        if model_name in model_dict:
107  
-            # The same model may be imported via different paths (e.g.
108  
-            # appname.models and project.appname.models). We use the source
109  
-            # filename as a means to detect identity.
110  
-            fname1 = os.path.abspath(sys.modules[model.__module__].__file__)
111  
-            fname2 = os.path.abspath(sys.modules[model_dict[model_name].__module__].__file__)
112  
-            # Since the filename extension could be .py the first time and .pyc
113  
-            # or .pyo the second time, ignore the extension when comparing.
114  
-            if os.path.splitext(fname1)[0] == os.path.splitext(fname2)[0]:
115  
-                continue
116  
-        model_dict[model_name] = model
  17
+    # Use the Borg pattern to share state between all instances. Details at
  18
+    # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/66531.
  19
+    __shared_state = dict(
  20
+        # Keys of app_store are the model modules for each application.
  21
+        app_store = {},
  22
+
  23
+        # Mapping of app_labels to a dictionary of model names to model code.
  24
+        app_models = {},
  25
+
  26
+        # Mapping of app_labels to errors raised when trying to import the app.
  27
+        app_errors = {},
  28
+
  29
+        # -- Everything below here is only used when populating the cache --
  30
+        loaded = False,
  31
+        handled = {},
  32
+        postponed = [],
  33
+        nesting_level = 0,
  34
+        write_lock = threading.RLock(),
  35
+    )
  36
+
  37
+    def __init__(self):
  38
+        self.__dict__ = self.__shared_state
  39
+
  40
+    def _populate(self):
  41
+        """
  42
+        Fill in all the cache information. This method is threadsafe, in the
  43
+        sense that every caller will see the same state upon return, and if the
  44
+        cache is already initialised, it does no work.
  45
+        """
  46
+        if self.loaded:
  47
+            return
  48
+        self.write_lock.acquire()
  49
+        try:
  50
+            if self.loaded:
  51
+                return
  52
+            for app_name in settings.INSTALLED_APPS:
  53
+                if app_name in self.handled:
  54
+                    continue
  55
+                try:
  56
+                    self.load_app(app_name, True)
  57
+                except Exception, e:
  58
+                    # Problem importing the app
  59
+                    self.app_errors[app_name] = e
  60
+            if not self.nesting_level:
  61
+                for app_name in self.postponed:
  62
+                    self.load_app(app_name)
  63
+                self.loaded = True
  64
+        finally:
  65
+            self.write_lock.release()
  66
+
  67
+    def load_app(self, app_name, can_postpone=False):
  68
+        """
  69
+        Loads the app with the provided fully qualified name, and returns the
  70
+        model module.
  71
+        """
  72
+        self.handled[app_name] = None
  73
+        self.nesting_level += 1
  74
+        mod = __import__(app_name, {}, {}, ['models'])
  75
+        self.nesting_level -= 1
  76
+        if not hasattr(mod, 'models'):
  77
+            if can_postpone:
  78
+                # Either the app has no models, or the package is still being
  79
+                # imported by Python and the model module isn't available yet.
  80
+                # We will check again once all the recursion has finished (in
  81
+                # populate).
  82
+                self.postponed.append(app_name)
  83
+            return None
  84
+        if mod.models not in self.app_store:
  85
+            self.app_store[mod.models] = len(self.app_store)
  86
+        return mod.models
  87
+
  88
+    def cache_ready(self):
  89
+        """
  90
+        Returns true if the model cache is fully populated.
  91
+
  92
+        Useful for code that wants to cache the results of get_models() for
  93
+        themselves once it is safe to do so.
  94
+        """
  95
+        return self.loaded
  96
+
  97
+    def get_apps(self):
  98
+        "Returns a list of all installed modules that contain models."
  99
+        self._populate()
  100
+
  101
+        # Ensure the returned list is always in the same order (with new apps
  102
+        # added at the end). This avoids unstable ordering on the admin app
  103
+        # list page, for example.
  104
+        apps = [(v, k) for k, v in self.app_store.items()]
  105
+        apps.sort()
  106
+        return [elt[1] for elt in apps]
  107
+
  108
+    def get_app(self, app_label, emptyOK=False):
  109
+        """
  110
+        Returns the module containing the models for the given app_label. If
  111
+        the app has no models in it and 'emptyOK' is True, returns None.
  112
+        """
  113
+        self._populate()
  114
+        self.write_lock.acquire()
  115
+        try:
  116
+            for app_name in settings.INSTALLED_APPS:
  117
+                if app_label == app_name.split('.')[-1]:
  118
+                    mod = self.load_app(app_name, False)
  119
+                    if mod is None:
  120
+                        if emptyOK:
  121
+                            return None
  122
+                    else:
  123
+                        return mod
  124
+            raise ImproperlyConfigured, "App with label %s could not be found" % app_label
  125
+        finally:
  126
+            self.write_lock.release()
  127
+
  128
+    def get_app_errors(self):
  129
+        "Returns the map of known problems with the INSTALLED_APPS."
  130
+        self._populate()
  131
+        return self.app_errors
  132
+
  133
+    def get_models(self, app_mod=None):
  134
+        """
  135
+        Given a module containing models, returns a list of the models.
  136
+        Otherwise returns a list of all installed models.
  137
+        """
  138
+        self._populate()
  139
+        if app_mod:
  140
+            return self.app_models.get(app_mod.__name__.split('.')[-2], {}).values()
  141
+        else:
  142
+            model_list = []
  143
+            for app_entry in self.app_models.itervalues():
  144
+                model_list.extend(app_entry.values())
  145
+            return model_list
  146
+
  147
+    def get_model(self, app_label, model_name, seed_cache=True):
  148
+        """
  149
+        Returns the model matching the given app_label and case-insensitive
  150
+        model_name.
  151
+
  152
+        Returns None if no model is found.
  153
+        """
  154
+        if seed_cache:
  155
+            self._populate()
  156
+        return self.app_models.get(app_label, {}).get(model_name.lower())
  157
+
  158
+    def register_models(self, app_label, *models):
  159
+        """
  160
+        Register a set of models as belonging to an app.
  161
+        """
  162
+        for model in models:
  163
+            # Store as 'name: model' pair in a dictionary
  164
+            # in the _app_models dictionary
  165
+            model_name = model._meta.object_name.lower()
  166
+            model_dict = self.app_models.setdefault(app_label, {})
  167
+            if model_name in model_dict:
  168
+                # The same model may be imported via different paths (e.g.
  169
+                # appname.models and project.appname.models). We use the source
  170
+                # filename as a means to detect identity.
  171
+                fname1 = os.path.abspath(sys.modules[model.__module__].__file__)
  172
+                fname2 = os.path.abspath(sys.modules[model_dict[model_name].__module__].__file__)
  173
+                # Since the filename extension could be .py the first time and
  174
+                # .pyc or .pyo the second time, ignore the extension when
  175
+                # comparing.
  176
+                if os.path.splitext(fname1)[0] == os.path.splitext(fname2)[0]:
  177
+                    continue
  178
+            model_dict[model_name] = model
  179
+
  180
+cache = Cache()
  181
+
  182
+# These methods were always module level, so are kept that way for backwards
  183
+# compatibility.
  184
+get_apps = cache.get_apps
  185
+get_app = cache.get_app
  186
+get_app_errors = cache.get_app_errors
  187
+get_models = cache.get_models
  188
+get_model = cache.get_model
  189
+register_models = cache.register_models
  190
+load_app = cache.load_app
  191
+cache_ready = cache.cache_ready
5  django/db/models/options.py
@@ -2,7 +2,7 @@
2 2
 from django.db.models.related import RelatedObject
3 3
 from django.db.models.fields.related import ManyToManyRel
4 4
 from django.db.models.fields import AutoField, FieldDoesNotExist
5  
-from django.db.models.loading import get_models
  5
+from django.db.models.loading import get_models, cache_ready
6 6
 from django.db.models.query import orderlist2sql
7 7
 from django.db.models import Manager
8 8
 from django.utils.translation import activate, deactivate_all, get_language, string_concat
@@ -179,7 +179,8 @@ def get_all_related_many_to_many_objects(self):
179 179
                 for f in klass._meta.many_to_many:
180 180
                     if f.rel and self == f.rel.to._meta:
181 181
                         rel_objs.append(RelatedObject(f.rel.to, klass, f))
182  
-            self._all_related_many_to_many_objects = rel_objs
  182
+            if cache_ready():
  183
+                self._all_related_many_to_many_objects = rel_objs
183 184
             return rel_objs
184 185
 
185 186
     def get_ordered_objects(self):

0 notes on commit 3219a60

Please sign in to comment.
Something went wrong with that request. Please try again.