Add support for bpython console.

Adds support for configuration of shells from scrapy.cfg
and SCRAPY_PYTHON_SHELL.

config snippet:

cat <<EOF >> ~/.scrapy.cfg
[settings]
# shell can be one of ipython, bpython or python;
# to be used as the interactive python console, if available.
# (default is ipython, fallbacks in the order listed above)
shell = python
EOF

(closes #270, #1100)

docs/topics/commands.rst

@@ -13,6 +13,8 @@ just call "commands" or "Scrapy commands".
 The Scrapy tool provides several commands, for multiple purposes, and each one
 accepts a different set of arguments and options.
 
+.. _topics-config-settings:
+
 Configuration settings
 ======================
 
@@ -31,8 +33,9 @@ and project-wide settings will override all others, when defined.
 Scrapy also understands, and can be configured through, a number of environment
 variables. Currently these are:
 
-* ``SCRAPY_SETTINGS_MODULE`` (See :ref:`topics-settings-module-envvar`)
+* ``SCRAPY_SETTINGS_MODULE`` (see :ref:`topics-settings-module-envvar`)
 * ``SCRAPY_PROJECT``
+* ``SCRAPY_PYTHON_SHELL`` (see :ref:`topics-shell`)
 
 .. _topics-project-structure:
 

docs/topics/shell.rst

@@ -17,6 +17,9 @@ spider, without having to run the spider to test every change.
 Once you get familiarized with the Scrapy shell, you'll see that it's an
 invaluable tool for developing and debugging your spiders.
 
+Configuring the shell
+=====================
+
 If you have `IPython`_ installed, the Scrapy shell will use it (instead of the
 standard Python console). The `IPython`_ console is much more powerful and
 provides smart auto-completion and colorized output, among other things.
@@ -25,8 +28,20 @@ We highly recommend you install `IPython`_, specially if you're working on
 Unix systems (where `IPython`_ excels). See the `IPython installation guide`_
 for more info.
 
+Scrapy also has support for `bpython`_, and will try to use it where `IPython`_
+is unavailable.
+
+Through scrapy's settings you can configure it to use any one of
+``ipython``, ``bpython`` or the standard ``python`` shell, regardless of which
+are installed. This is done by setting the ``SCRAPY_PYTHON_SHELL`` environment
+variable; or by defining it in your :ref:`scrapy.cfg <topics-config-settings>`::
+
+    [settings]
+    shell = bpython
+
 .. _IPython: http://ipython.org/
 .. _IPython installation guide: http://ipython.org/install.html
+.. _bpython: http://www.bpython-interpreter.org/
 
 Launch the shell
 ================

scrapy/shell.py

@@ -5,6 +5,7 @@
 """
 from __future__ import print_function
 
+import os
 import signal
 import warnings
 
@@ -21,6 +22,7 @@
 from scrapy.utils.console import start_python_console
 from scrapy.utils.misc import load_object
 from scrapy.utils.response import open_in_browser
+from scrapy.utils.conf import get_config
 
 
 class Shell(object):
@@ -52,7 +54,27 @@ def start(self, url=None, request=None, response=None, spider=None):
         if self.code:
             print(eval(self.code, globals(), self.vars))
         else:
-            start_python_console(self.vars)
+            # detect interactive shell setting in scrapy.cfg
+            ''' e.g.: ~/.scrapy.cfg
+            [settings]
+            # shell can be one of ipython, bpython or python;
+            # to be used as the interactive python console, if available.
+            # (default is ipython, fallbacks in the order listed above)
+            shell = python
+            '''
+            cfg = get_config()
+            section, option = 'settings', 'shell'
+            env = os.environ.get('SCRAPY_PYTHON_SHELL')
+            shells = []
+            if env:
+                shells += env.strip().lower().split(',')
+            elif cfg.has_option(section, option):
+                shells += [cfg.get(section, option).strip().lower()]
+            else: # try all by default
+                shells += ['ipython', 'bpython']
+            # always have standard shell as fallback
+            shells += ['python']
+            start_python_console(self.vars, shells=shells)
 
     def _schedule(self, request, spider):
         spider = self._open_spider(request, spider)

scrapy/utils/console.py

@@ -1,37 +1,77 @@
+from functools import wraps
+
+from scrapy.exceptions import NotConfigured
 
-def start_python_console(namespace=None, noipython=False, banner=''):
-    """Start Python console binded to the given namespace. If IPython is
-    available, an IPython console will be started instead, unless `noipython`
-    is True. Also, tab completion will be used on Unix systems.
-    """
-    if namespace is None:
-        namespace = {}
 
+def _embed_ipython_shell(namespace={}, banner=''):
+    """Start an IPython Shell"""
     try:
-        try: # use IPython if available
-            if noipython:
-                raise ImportError()
+        from IPython.terminal.embed import InteractiveShellEmbed
+        from IPython.terminal.ipapp import load_default_config
+    except ImportError:
+        from IPython.frontend.terminal.embed import InteractiveShellEmbed
+        from IPython.frontend.terminal.ipapp import load_default_config
+
+    @wraps(_embed_ipython_shell)
+    def wrapper(namespace=namespace, banner=''):
+        config = load_default_config()
+        shell = InteractiveShellEmbed(
+            banner1=banner, user_ns=namespace, config=config)
+        shell()
+    return wrapper
+
+def _embed_bpython_shell(namespace={}, banner=''):
+    """Start a bpython shell"""
+    import bpython
+    @wraps(_embed_bpython_shell)
+    def wrapper(namespace=namespace, banner=''):
+        bpython.embed(locals_=namespace, banner=banner)
+    return wrapper
+
+def _embed_standard_shell(namespace={}, banner=''):
+    """Start a standard python shell"""
+    import code
+    try: # readline module is only available on unix systems
+        import readline
+    except ImportError:
+        pass
+    else:
+        import rlcompleter
+        readline.parse_and_bind("tab:complete")
+    @wraps(_embed_standard_shell)
+    def wrapper(namespace=namespace, banner=''):
+        code.interact(banner=banner, local=namespace)
+    return wrapper
+_embed_python_shell = _embed_standard_shell
 
+def get_shell_embed_func(shells):
+    """Return the first acceptable shell-embed function
+    from a given list of shell names.
+    """
+    for shell in shells:
+        try:
+            _f = globals()['_embed_%s_shell' % (shell,)]
+        except KeyError: continue
+        if callable(_f):
             try:
-                from IPython.terminal.embed import InteractiveShellEmbed
-                from IPython.terminal.ipapp import load_default_config
+                # function test: run all setup code (imports),
+                # but dont fall into the shell
+                return _f()
             except ImportError:
-                from IPython.frontend.terminal.embed import InteractiveShellEmbed
-                from IPython.frontend.terminal.ipapp import load_default_config
+                continue
 
-            config = load_default_config()
-            shell = InteractiveShellEmbed(
-                banner1=banner, user_ns=namespace, config=config)
+def start_python_console(namespace=None, banner='', shells=None):
+    """Start Python console bound to the given namespace.
+    Readline support and tab completion will be used on Unix, if available.
+    """
+    if namespace is None:
+        namespace = {}
+    if shells is None:
+        shells = ['ipython', 'bpython', 'python']
+
+    try:
+        shell = get_shell_embed_func(shells)
+        if shell is not None:
             shell()
-        except ImportError:
-            import code
-            try: # readline module is only available on unix systems
-                import readline
-            except ImportError:
-                pass
-            else:
-                import rlcompleter
-                readline.parse_and_bind("tab:complete")
-            code.interact(banner=banner, local=namespace)
     except SystemExit: # raised when using exit() in python code.interact
         pass

tests/test_utils_console.py

@@ -0,0 +1,17 @@
+import unittest
+
+from scrapy.utils.console import get_shell_embed_func
+
+class UtilsConsoleTestCase(unittest.TestCase):
+
+    def test_get_shell_embed_func(self):
+        shell = get_shell_embed_func(['invalid'])
+        self.assertEqual(shell, None)
+
+        shell = get_shell_embed_func(['invalid','python'])
+        assert callable(shell)
+        self.assertEqual(shell.__name__, '_embed_standard_shell')
+
+
+if __name__ == "__main__":
+    unittest.main()