shell_plus --notebook not working in 1.7

[jonathanstrong]

from docs looks like there's a new setup() method.

---------------------------------------------------------------------------
AppRegistryNotReady                       Traceback (most recent call last)
<ipython-input-1-7474fc4fe95e> in <module>()
      [CODE ACCESSING MODELS VIA DJANGO ORM]

C:\Python\lib\site-packages\django-1.7c1-py2.7.egg\django\db\models\manager.pyc in manager_method(self, *args, **kwargs)
     90         def create_method(name, method):
     91             def manager_method(self, *args, **kwargs):
---> 92                 return getattr(self.get_queryset(), name)(*args, **kwargs)
     93             manager_method.__name__ = method.__name__
     94             manager_method.__doc__ = method.__doc__

C:\Python\lib\site-packages\django-1.7c1-py2.7.egg\django\db\models\query.pyc in filter(self, *args, **kwargs)
    687         set.
    688         """
--> 689         return self._filter_or_exclude(False, *args, **kwargs)
    690 
    691     def exclude(self, *args, **kwargs):

C:\Python\lib\site-packages\django-1.7c1-py2.7.egg\django\db\models\query.pyc in _filter_or_exclude(self, negate, *args, **kwargs)
    705             clone.query.add_q(~Q(*args, **kwargs))
    706         else:
--> 707             clone.query.add_q(Q(*args, **kwargs))
    708         return clone
    709 

C:\Python\lib\site-packages\django-1.7c1-py2.7.egg\django\db\models\sql\query.pyc in add_q(self, q_object)
   1285         existing_inner = set(
   1286             (a for a in self.alias_map if self.alias_map[a].join_type == self.INNER))
-> 1287         clause, require_inner = self._add_q(where_part, self.used_aliases)
   1288         self.where.add(clause, AND)
   1289         for hp in having_parts:

C:\Python\lib\site-packages\django-1.7c1-py2.7.egg\django\db\models\sql\query.pyc in _add_q(self, q_object, used_aliases, branch_negated, current_negated)
   1312                 child_clause, needed_inner = self.build_filter(
   1313                     child, can_reuse=used_aliases, branch_negated=branch_negated,
-> 1314                     current_negated=current_negated, connector=connector)
   1315                 joinpromoter.add_votes(needed_inner)
   1316             target_clause.add(child_clause, connector)

C:\Python\lib\site-packages\django-1.7c1-py2.7.egg\django\db\models\sql\query.pyc in build_filter(self, filter_expr, branch_negated, current_negated, can_reuse, connector)
   1136         if not arg:
   1137             raise FieldError("Cannot parse keyword query %r" % arg)
-> 1138         lookups, parts, reffed_aggregate = self.solve_lookup_type(arg)
   1139 
   1140         # Work out the lookup type and remove it from the end of 'parts',

C:\Python\lib\site-packages\django-1.7c1-py2.7.egg\django\db\models\sql\query.pyc in solve_lookup_type(self, lookup)
   1074             if aggregate:
   1075                 return aggregate_lookups, (), aggregate
-> 1076         _, field, _, lookup_parts = self.names_to_path(lookup_splitted, self.get_meta())
   1077         field_parts = lookup_splitted[0:len(lookup_splitted) - len(lookup_parts)]
   1078         if len(lookup_parts) == 0:

C:\Python\lib\site-packages\django-1.7c1-py2.7.egg\django\db\models\sql\query.pyc in names_to_path(self, names, opts, allow_many, fail_on_missing)
   1337                 name = opts.pk.name
   1338             try:
-> 1339                 field, model, direct, m2m = opts.get_field_by_name(name)
   1340             except FieldDoesNotExist:
   1341                 # We didn't found the current field, so move position back

C:\Python\lib\site-packages\django-1.7c1-py2.7.egg\django\db\models\options.pyc in get_field_by_name(self, name)
    414                 return self._name_map[name]
    415             except AttributeError:
--> 416                 cache = self.init_name_map()
    417                 return cache[name]
    418         except KeyError:

C:\Python\lib\site-packages\django-1.7c1-py2.7.egg\django\db\models\options.pyc in init_name_map(self)
    443         # We intentionally handle related m2m objects first so that symmetrical
    444         # m2m accessor names can be overridden, if necessary.
--> 445         for f, model in self.get_all_related_m2m_objects_with_model():
    446             cache[f.field.related_query_name()] = (f, model, False, True)
    447         for f, model in self.get_all_related_objects_with_model():

C:\Python\lib\site-packages\django-1.7c1-py2.7.egg\django\db\models\options.pyc in get_all_related_m2m_objects_with_model(self)
    559             cache = self._related_many_to_many_cache
    560         except AttributeError:
--> 561             cache = self._fill_related_many_to_many_cache()
    562         return list(six.iteritems(cache))
    563 

C:\Python\lib\site-packages\django-1.7c1-py2.7.egg\django\db\models\options.pyc in _fill_related_many_to_many_cache(self)
    573                 else:
    574                     cache[obj] = model
--> 575         for klass in self.apps.get_models():
    576             if not klass._meta.swapped:
    577                 for f in klass._meta.local_many_to_many:

C:\Python\lib\site-packages\django-1.7c1-py2.7.egg\django\utils\lru_cache.pyc in wrapper(*args, **kwds)
     99                         stats[HITS] += 1
    100                         return result
--> 101                     result = user_function(*args, **kwds)
    102                     cache[key] = result
    103                     stats[MISSES] += 1

C:\Python\lib\site-packages\django-1.7c1-py2.7.egg\django\apps\registry.pyc in get_models(self, app_mod, include_auto_created, include_deferred, include_swapped)
    166         Set the corresponding keyword argument to True to include such models.
    167         """
--> 168         self.check_models_ready()
    169         if app_mod:
    170             warnings.warn(

C:\Python\lib\site-packages\django-1.7c1-py2.7.egg\django\apps\registry.pyc in check_models_ready(self)
    129         """
    130         if not self.models_ready:
--> 131             raise AppRegistryNotReady("Models aren't loaded yet.")
    132 
    133     def get_app_configs(self):

AppRegistryNotReady: Models aren't loaded yet.

[trbs]

Thanks for reporting the problem.

But it seems to work for me, when I create a new project with Django 1.7 and work with models in the IPython notebook.

Which version of django-extensions are you using ? (use the latest git version to be sure this is not fixed already)

If problem still exists; please create a reproducible case (step-by-step guide + tarball with a full example which fails and requirements.txt file) for developers to start identifying the root cause and fixing it.

(Please reopen this ticket at that point)

[paulochf]

FYI, I got this problem today. I installed django-extensions on a Ubuntu 14.04 using pip install. The version was 1.4.4.

It worked after following @trbs's comment above (using latest git version).

[trbs]

After 1.4.4 in the git version we only have changes to AutoSlugField and admin_generator.

@paulochf if you install 1.4.4 again do you get the problem back ?

[paulochf]

Please follow me, I don't know if I did this right.

To remove the installed version from repo: sudo pip uninstall django-extensions
Then, installing pip package again: sudo pip install django-extensions

If I did this right, so no, I didn't get the problem back. I reran my working note from yesterday and it didn't show that error again.

As I don't know if the pip uninstall was enough, I'll wait for your confirmation.

[trbs]

Looks sound to me.

The only way to check if pip uninstall did its work correctly is to manually check if all files are gone. (including the files that setuptools/distutils create at install)

But I won't expect any real differences from those... the error indicates that Django is not properly initialized, this should be fixed for django-extensions in changeset: 57d33a4

From what I can see from the traceback above the error is raised only when executing code in the notebook ? If that's the case then maybe something is not initializing Django correctly in the running notebook kernel.

It's good to check the logs and make sure you see this line:

[IPKernelApp] Loading IPython extension: django_extensions.management.notebook_extension

Because that extension will make sure that django.setup() has run on Django 1.7.

[paulochf]

For the sake of debugging, I did a clean install again.

I looked for any files after running a pip uninstall, and found nothing. Good!

pip install, then ran the notebook with python manage.py shell_plus --notebook. Everything fine with my code.

I checked the versions here: Python 2.7.6, Django 1.6.1 (for my surprise, I thought that was 1.7!).
I think because of this, the line you said is not there.

Well, everything ok, then. Thanks!

[paulochf]

@trbs, the history ain't over!

After I realize I was using Django 1.6.1, I updated it to 1.7 cleanly (uninstall 1.6, install 1.7). Then, when I went back to notebook, I got this error again.

[trbs]

Any change you can give me a install guide I can follow to reproduce this on my machines ?

[paulochf]

Sure, @trbs. Please let me know if you want some specifics.

sudo apt-get install ipython-notebook
sudo pip install Django django-extensions

# create django project with MySQL
# create some model
# add one row in DB table

python manage.py shell_plus --notebook

Then in notebook:

from project.models import Model

Model.objects.get(id=1)

# the error

[trbs]

(Working) Test Case:

$ virtualenv test526
New python executable in test526/bin/python2.7
Also creating executable in test526/bin/python
Installing setuptools, pip...done.

$ cd test526
$ source bin/activate
$ pip install ipython pyzmq jinja2 tornado
$ pip install Django django-extensions

# follow the configuration guide for shell_plus --notebook at http://django-extensions.readthedocs.org/en/latest/shell_plus.html#configuration 

$ django-admin startproject ham
$ cd ham
$ ./manage.py syncdb
$ ./manage.py startapp eggs
$ cat > eggs/models.py << EOF
from django.db import models

class Egg(models.Model):
    size = models.IntegerField(default=1)
EOF

# ham/settings.py
# - add django-extensions to INSTALLED_APPS
# - add eggs to INSTALLED_APPS
# - add IPYTHON_ARGUMENTS = ['--ext', 'django_extensions.management.notebook_extension', '--debug']

$ ./manage.py makemigrations
$ ./manage.py migrate

$ pip freeze -l
Django==1.7
Jinja2==2.7.3
MarkupSafe==0.23
backports.ssl-match-hostname==3.4.0.2
certifi==14.05.14
django-extensions==1.4.5
gnureadline==6.3.3
ipython==2.3.0
pyzmq==14.4.0
six==1.8.0
tornado==4.0.2

./manage.py shell_plus --notebook

In the notebook:

# no need to import the model Egg here since shell_plus did that for us.
Egg.objects.get(pk=0)

Notes:

1. Make sure that if you specify IPYTHON_ARGUMENTS in settings.py that you also load the notebook_extension. If you do not specify IPYTHON_ARGUMENTS at all in settings.py the extension will automatically be added to IPYTHON_ARGUMENTS by default. Without this notebook extension Django will not setup it's applications properly and you will have to call django.setup() manually.

2. Add "--debug" to IPYTHON_ARGUMENTS so you can see whether or not the extension gets loaded, some error occurs or it does not even try, look for: [IPKernelApp] Loading IPython extension: django_extensions.management.notebook_extension

3. If you have a ~/.ipython/profile_default/ipython_config.py file that overwrites the extensions to be loaded by IPython then the IPYTHON_ARGUMENTS won't work. Either specify "--profile=nbserver" or use .append() or .extend() in the ipython_config.py file instead of overwriting the extensions list.

BAD:

c = get_config()
c.InteractiveShellApp.extensions = [
   'some.extension',
]

GOOD:

c = get_config()
c.InteractiveShellApp.extensions.append('some.extension')

Follow Up in Django-Extensions

Looked a bit into IPython and could not find an easier way that would guarantee the extensions (or something similar) gets loaded since the notebook is a separate process it does not help to run django.setup() from the command itself.

In b165647 we now give a fairly big warning when we startup shell_plus --notebook when IPYTHON_ARGUMENTS does not contain the 'django_extensions.management.notebook_extension' extension. However this does not catch the case when a users ipython_config.py is overriding the extensions list.

If somebody is willing to make a PR for the documentation to try to explain this more clearly and explicitly it would be much appreciated.

[paulochf]

The test case worked. Then, in another terminal, I did sudo pip install Django (1.7.1-final) and tested my notebook, and it worked as well. I hope these are good signs.

I don't know what part was missing here.
I can I do have a ~/.ipython/profile_default/ipython_config.py but I followed the GOOD way to config it. Also, after this test, my python manage.py shell gives an exception.

[TerminalIPythonApp] ERROR | Exception while loading config file /home/paulo/.ipython/profile_default/ipython_config.py
Traceback (most recent call last):
  File "/usr/lib/python2.7/dist-packages/IPython/config/application.py", line 508, in load_config_file
    config = loader.load_config()
  File "/usr/lib/python2.7/dist-packages/IPython/config/loader.py", line 289, in load_config
    self._read_file_as_dict()
  File "/usr/lib/python2.7/dist-packages/IPython/config/loader.py", line 346, in _read_file_as_dict
    py3compat.execfile(conf_filename, namespace)
  File "/usr/lib/python2.7/dist-packages/IPython/utils/py3compat.py", line 204, in execfile
    __builtin__.execfile(filename, *where)
  File "/home/paulo/.ipython/profile_default/ipython_config.py", line 48, in <module>
    c.InteractiveShellApp.extensions.append("autoreload")
  File "/usr/lib/python2.7/dist-packages/IPython/config/loader.py", line 187, in __getattr__
    raise AttributeError(e)
AttributeError: 'extensions'

edit: I commented ~/.ipython/profile_default/ipython_config.py lines about autoreload cited in the exception and it fixed.

As I don't know well IPython's configurations neither virtualenv, I don't know how to help more.

[paulochf]

So long I have used this, today I was trying to use it again and it wasn't loading my modules. After some tests, I realized that notebooks located at mydjangoproject/ loads modules correctly, but inside other folders (i.e. mydjangoproject/notebooks , for the sake of code organization) did not.

Is there any solution for this issue?

Thanks!

[trbs]

Sorry for the moment the only solution is to make sure the notebook server is started from the project directory.

[paulochf]

Sorry for the late reply.

@trbs, you mean running python manage.py shell_plus --notebook from project root should be able to run ipynb files inside mydjangoproject/notebooks ?

Because it's not the case.

from mydjangoproject.somefolder.somefile import *


---------------------------------------------------------------------------
ImportError                               Traceback (most recent call last)
<ipython-input-2-4db0e2101940> in <module>()
     14 
     15 from mydjangoproject.somefolder.somefile import *
     16 
     17 

ImportError: No module named 'mydjangoproject'

Now I'm on Python 3.4.3, Django 1.8.3 and django-extensions 1.5.5 .

[paulochf]

One more thing. I'm not getting select_related() to work. Individual models does work, but using select_related() one with ForeignKey isn't. Using Django shell works normally.

Using an extended test based on @trbs above:

Note: using Python 3.4.3, Django 1.8.3 and django-extensions 1.5.5 .

$ cat > eggs/models.py << EOF
from django.db import models

class Bacon(models.Model):
    strips = models.IntegerField(default=1)

class Egg(models.Model):
    size = models.IntegerField(default=1)
    bacon_strips = models.ForeignKey(Bacon, null=False, default=1)
EOF

then create some data

from eggs.models import Egg, Bacon
Egg.objects.get_or_create(size=1)
Egg.objects.get_or_create(size=4)

Bacon.objects.get_or_create(strips=1)
Bacon.objects.get_or_create(strips=3)

egg, _ = Egg.objects.get(pk=2)
bacon, _ = Bacon.objects.get(pk=2)

egg.bacon_strips = bacon
egg.save()
print(egg.size, egg.bacon_strips.id, egg.bacon_strips.strips) # 4, 2, 3

Then try to use select_related()

eggs_with_bacon = Egg.objects.select_related("bacon_strips")


---------------------------------------------------------------------------
AppRegistryNotReady                       Traceback (most recent call last)
/home/paulo/.virtualenvs/teste/lib/python3.4/site-packages/IPython/core/formatters.py in __call__(self, obj)
    688                 type_pprinters=self.type_printers,
    689                 deferred_pprinters=self.deferred_printers)
--> 690             printer.pretty(obj)
    691             printer.flush()
    692             return stream.getvalue()

/home/paulo/.virtualenvs/teste/lib/python3.4/site-packages/IPython/lib/pretty.py in pretty(self, obj)
    407                             if callable(meth):
    408                                 return meth(obj, self, cycle)
--> 409             return _default_pprint(obj, self, cycle)
    410         finally:
    411             self.end_group()

/home/paulo/.virtualenvs/teste/lib/python3.4/site-packages/IPython/lib/pretty.py in _default_pprint(obj, p, cycle)
    527     if _safe_getattr(klass, '__repr__', None) not in _baseclass_reprs:
    528         # A user-provided repr. Find newlines and replace them with p.break_()
--> 529         _repr_pprint(obj, p, cycle)
    530         return
    531     p.begin_group(1, '<')

/home/paulo/.virtualenvs/teste/lib/python3.4/site-packages/IPython/lib/pretty.py in _repr_pprint(obj, p, cycle)
    709     """A pprint that just redirects to the normal repr function."""
    710     # Find newlines and replace them with p.break_()
--> 711     output = repr(obj)
    712     for idx,output_line in enumerate(output.splitlines()):
    713         if idx:

/home/paulo/.virtualenvs/teste/lib/python3.4/site-packages/django/db/models/query.py in __repr__(self)
    136 
    137     def __repr__(self):
--> 138         data = list(self[:REPR_OUTPUT_SIZE + 1])
    139         if len(data) > REPR_OUTPUT_SIZE:
    140             data[-1] = "...(remaining elements truncated)..."

/home/paulo/.virtualenvs/teste/lib/python3.4/site-packages/django/db/models/query.py in __iter__(self)
    160                - Responsible for turning the rows into model objects.
    161         """
--> 162         self._fetch_all()
    163         return iter(self._result_cache)
    164 

/home/paulo/.virtualenvs/teste/lib/python3.4/site-packages/django/db/models/query.py in _fetch_all(self)
    963     def _fetch_all(self):
    964         if self._result_cache is None:
--> 965             self._result_cache = list(self.iterator())
    966         if self._prefetch_related_lookups and not self._prefetch_done:
    967             self._prefetch_related_objects()

/home/paulo/.virtualenvs/teste/lib/python3.4/site-packages/django/db/models/query.py in iterator(self)
    236         # Execute the query. This will also fill compiler.select, klass_info,
    237         # and annotations.
--> 238         results = compiler.execute_sql()
    239         select, klass_info, annotation_col_map = (compiler.select, compiler.klass_info,
    240                                                   compiler.annotation_col_map)

/home/paulo/.virtualenvs/teste/lib/python3.4/site-packages/django/db/models/sql/compiler.py in execute_sql(self, result_type)
    827             result_type = NO_RESULTS
    828         try:
--> 829             sql, params = self.as_sql()
    830             if not sql:
    831                 raise EmptyResultSet

/home/paulo/.virtualenvs/teste/lib/python3.4/site-packages/django/db/models/sql/compiler.py in as_sql(self, with_limits, with_col_aliases, subquery)
    376         refcounts_before = self.query.alias_refcount.copy()
    377         try:
--> 378             extra_select, order_by, group_by = self.pre_sql_setup()
    379             if with_limits and self.query.low_mark == self.query.high_mark:
    380                 return '', ()

/home/paulo/.virtualenvs/teste/lib/python3.4/site-packages/django/db/models/sql/compiler.py in pre_sql_setup(self)
     46         might not have all the pieces in place at that time.
     47         """
---> 48         self.setup_query()
     49         order_by = self.get_order_by()
     50         extra_select = self.get_extra_select(order_by, self.select)

/home/paulo/.virtualenvs/teste/lib/python3.4/site-packages/django/db/models/sql/compiler.py in setup_query(self)
     37         if all(self.query.alias_refcount[a] == 0 for a in self.query.tables):
     38             self.query.get_initial_alias()
---> 39         self.select, self.klass_info, self.annotation_col_map = self.get_select()
     40         self.col_count = len(self.select)
     41 

/home/paulo/.virtualenvs/teste/lib/python3.4/site-packages/django/db/models/sql/compiler.py in get_select(self)
    204 
    205         if self.query.select_related:
--> 206             related_klass_infos = self.get_related_selections(select)
    207             klass_info['related_klass_infos'] = related_klass_infos
    208 

/home/paulo/.virtualenvs/teste/lib/python3.4/site-packages/django/db/models/sql/compiler.py in get_related_selections(self, select, opts, root_alias, cur_depth, requested, restricted)
    698             select_fields = []
    699             _, _, _, joins, _ = self.query.setup_joins(
--> 700                 [f.name], opts, root_alias)
    701             alias = joins[-1]
    702             columns = self.get_default_columns(start_alias=alias, opts=f.rel.to._meta)

/home/paulo/.virtualenvs/teste/lib/python3.4/site-packages/django/db/models/sql/query.py in setup_joins(self, names, opts, alias, can_reuse, allow_many)
   1469         # First, generate the path for the names
   1470         path, final_field, targets, rest = self.names_to_path(
-> 1471             names, opts, allow_many, fail_on_missing=True)
   1472 
   1473         # Then, add the path to the query's joins. Note that we can't trim

/home/paulo/.virtualenvs/teste/lib/python3.4/site-packages/django/db/models/sql/query.py in names_to_path(self, names, opts, allow_many, fail_on_missing)
   1370                 # model (like a GenericForeignKey) cannot generate reverse
   1371                 # relations and therefore cannot be used for reverse querying.
-> 1372                 if field.is_relation and not field.related_model:
   1373                     raise FieldError(
   1374                         "Field %r does not generate an automatic reverse "

/home/paulo/.virtualenvs/teste/lib/python3.4/site-packages/django/utils/functional.py in __get__(self, instance, type)
     58         if instance is None:
     59             return self
---> 60         res = instance.__dict__[self.name] = self.func(instance)
     61         return res
     62 

/home/paulo/.virtualenvs/teste/lib/python3.4/site-packages/django/db/models/fields/related.py in related_model(self)
    108     def related_model(self):
    109         # Can't cache this property until all the models are loaded.
--> 110         apps.check_models_ready()
    111         return self.rel.to
    112 

/home/paulo/.virtualenvs/teste/lib/python3.4/site-packages/django/apps/registry.py in check_models_ready(self)
    129         """
    130         if not self.models_ready:
--> 131             raise AppRegistryNotReady("Models aren't loaded yet.")
    132 
    133     def get_app_configs(self):

AppRegistryNotReady: Models aren't loaded yet.

Thanks!

[paulochf]

For completeness, the error also occurs when trying to run SomeModel.some_many_to_many_field.all(), when shell runs it normally.

[trbs]

@paulochf Your example works for me... it's also strange that something like .select_related("bacon_strips") would not work with the AppRegistryNotReady: Models aren't loaded yet. exception. Could it be that something else has disrupted Django from initializing those models ?

Tested with Python 2.7:

$ pip freeze -l
backports.ssl-match-hostname==3.4.0.2
certifi==2015.4.28
Django==1.8.3
-e git+git@github.com:django-extensions/django-extensions.git@91705b7ead08e40b68f579a9c6b07a3f5cab6da0#egg=django_extensions-master
functools32==3.2.3.post2
gnureadline==6.3.3
ipython==3.2.1
Jinja2==2.8
jsonschema==2.5.1
MarkupSafe==0.23
pyzmq==14.7.0
six==1.9.0
tornado==4.2.1
wheel==0.24.0

[paulochf]

Hey @trbs,

I'm on Python 3.4.3, Django 1.8.3 and django-extensions 1.5.5 . I can't say nothing about Python 2.7 .
But I used django-extensions from pip. I'll give the edge version a try.

Edit: didn't work either.Should I open another issue for Python 3/Django 1.8 or we can continue to discuss this here?

[jrjames83]

Was dealing with this yesterday. Putting the following in my jupyter notebook before all the model imports helped circumvent and get back to fiddling:

import os, django
django.setup()

Relevant Modules

Django==1.10.1
django-debug-toolbar==1.8
django-extensions==1.8.1

ipykernel==4.6.1
ipython==6.1.0
ipython-genutils==0.2.0
ipywidgets==7.0.0

jupyter==1.0.0
jupyter-client==5.1.0
jupyter-console==5.2.0
jupyter-core==4.3.0

[ehacinom]

If the settings file is at PATH_TO_APP/APP/settings.py:

import sys, os, django
sys.path.append("PATH_TO_APP")
os.environ['DJANGO_SETTINGS_MODULE'] = 'APP.settings'
django.setup()