diff --git a/.travis.yml b/.travis.yml index 7463d827e2..73f38f8f81 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ before_install: - 'if [[ $GROUP == js* ]]; then npm install -g casperjs; fi' - git clone --quiet --depth 1 https://github.com/minrk/travis-wheels travis-wheels install: - - pip install -f travis-wheels/wheelhouse --pre file://$PWD#egg=notebook[test] coveralls + - pip install -f travis-wheels/wheelhouse file://$PWD#egg=notebook[test] coveralls script: - 'if [[ $GROUP == js* ]]; then python -m notebook.jstest ${GROUP:3}; fi' - 'if [[ $GROUP == python ]]; then nosetests --with-coverage --cover-package=notebook notebook; fi' diff --git a/bower.json b/bower.json index 114b4ab1a2..0508be5f65 100644 --- a/bower.json +++ b/bower.json @@ -15,7 +15,7 @@ "MathJax": "components/MathJax#~2.6", "moment": "~2.8.4", "requirejs": "~2.1", - "term.js": "chjj/term.js#~0.0.7", + "xterm.js": "sourcelair/xterm.js#0.33", "text-encoding": "~0.1", "underscore": "components/underscore#~1.5", "jquery-typeahead": "~2.0.0" diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 9926cf06de..a377967ac6 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -6,6 +6,53 @@ Jupyter notebook changelog A summary of changes in the Jupyter notebook. For more detailed information, see `GitHub `__. +.. tip:: + + Use ``pip install notebook --upgrade`` or ``conda upgrade notebook`` to + upgrade to the latest release. + + +.. _release-4.2.1: + +4.2.1 +----- + +4.2.1 is a small bugfix release on 4.2. Highlights: + +- Compatibility fixes for some versions of ipywidgets +- Fix for ignored CSS on Windows +- Fix specifying destination when installing nbextensions + +.. seealso:: + + 4.2.1 `on GitHub `__. + +.. _release-4.2.0: + +4.2.0 +----- + +Release 4.2 adds a new API for enabling and installing extensions. +Extensions can now be enabled at the system-level, rather than just per-user. +An API is defined for installing directly from a Python package, as well. + +.. seealso:: + + :doc:`./examples/Notebook/rstversions/Distributing Jupyter Extensions as Python Packages` + + +Highlighted changes: + +- Upgrade MathJax to 2.6 to fix vertical-bar appearing on some equations. +- Restore ability for notebook directory to be root (4.1 regression) +- Large outputs are now throttled, reducing the ability of output floods to + kill the browser. +- Fix the notebook ignoring cell executions while a kernel is starting by queueing the messages. +- Fix handling of url prefixes (e.g. JupyterHub) in terminal and edit pages. +- Support nested SVGs in output. + +And various other fixes and improvements. + .. _release-4.1.0: 4.1.0 diff --git a/docs/source/examples/Notebook/Distributing Jupyter Extensions as Python Packages.ipynb b/docs/source/examples/Notebook/Distributing Jupyter Extensions as Python Packages.ipynb new file mode 100644 index 0000000000..d14cb4f509 --- /dev/null +++ b/docs/source/examples/Notebook/Distributing Jupyter Extensions as Python Packages.ipynb @@ -0,0 +1,281 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Distributing Jupyter Extensions as Python Packages" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Overview\n", + "### How can the notebook be extended?\n", + "The Jupyter Notebook client and server application are both deeply customizable. Their behavior can be extended by creating, respectively:\n", + "\n", + "- nbextension: a notebook extension\n", + " - a single JS file, or directory of JavaScript, Cascading StyleSheets, etc. that contain at\n", + " minimum a JavaScript module packaged as an\n", + " [AMD modules](https://en.wikipedia.org/wiki/Asynchronous_module_definition)\n", + " that exports a function `load_ipython_extension`\n", + "- server extension: an importable Python module\n", + " - that implements `load_jupyter_server_extension`" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Why create a Python package for Jupyter extensions?\n", + "Since it is rare to have a server extension that does not have any frontend components (an nbextension), for convenience and consistency, all these client and server extensions with their assets can be packaged and versioned together as a Python package with a few simple commands. This makes installing the package of extensions easier and less error-prone for the user. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Installation of Jupyter Extensions\n", + "### Install a Python package containing Jupyter Extensions\n", + "There are several ways that you may get a Python package containing Jupyter Extensions. Commonly, you will use a package manager for your system:\n", + "```shell\n", + "pip install helpful_package\n", + "# or\n", + "conda install helpful_package\n", + "# or\n", + "apt-get install helpful_package\n", + "\n", + "# where 'helpful_package' is a Python package containing one or more Jupyter Extensions\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Enable a Server Extension\n", + "\n", + "The simplest case would be to enable a server extension which has no frontend components. \n", + "\n", + "A `pip` user that wants their configuration stored in their home directory would type the following command:\n", + "```shell\n", + "jupyter serverextension enable --py helpful_package\n", + "```\n", + "\n", + "Alternatively, a `virtualenv` or `conda` user can pass `--sys-prefix` which keeps their environment isolated and reproducible. For example:\n", + "```shell\n", + "# Make sure that your virtualenv or conda environment is activated\n", + "[source] activate my-environment\n", + "\n", + "jupyter serverextension enable --py helpful_package --sys-prefix\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Install the nbextension assets" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If a package also has an nbextension with frontend assets that must be available (but not neccessarily enabled by default), install these assets with the following command:\n", + "```shell\n", + "jupyter nbextension install --py helpful_package # or --sys-prefix if using virtualenv or conda\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Enable nbextension assets\n", + "If a package has assets that should be loaded every time a Jupyter app (e.g. lab, notebook, dashboard, terminal) is loaded in the browser, the following command can be used to enable the nbextension:\n", + "```shell\n", + "jupyter nbextension enable --py helpful_package # or --sys-prefix if using virtualenv or conda\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Did it work? Check by listing Jupyter Extensions.\n", + "After running one or more extension installation steps, you can list what is presently known about nbextensions or server extension. The following commands will list which extensions are available, whether they are enabled, and other extension details:\n", + "\n", + "```shell\n", + "jupyter nbextension list\n", + "jupyter serverextension list\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Additional resources on creating and distributing packages \n", + "\n", + "> Of course, in addition to the files listed, there are number of other files one needs to build a proper package. Here are some good resources:\n", + "- [The Hitchhiker's Guide to Packaging](http://the-hitchhikers-guide-to-packaging.readthedocs.org/en/latest/quickstart.html)\n", + "- [Repository Structure and Python](http://www.kennethreitz.org/essays/repository-structure-and-python) by Kenneth Reitz\n", + "\n", + "> How you distribute them, too, is important:\n", + "- [Packaging and Distributing Projects](http://python-packaging-user-guide.readthedocs.org/en/latest/distributing/)\n", + "- [conda: Building packages](http://conda.pydata.org/docs/building/build.html)\n", + "\n", + "> Here are some tools to get you started:\n", + "- [generator-nbextension](https://github.com/Anaconda-Server/generator-nbextension)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Example - Server extension" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Creating a Python package with a server extension\n", + "\n", + "Here is an example of a python module which contains a server extension directly on itself. It has this directory structure:\n", + "```\n", + "- setup.py\n", + "- MANIFEST.in\n", + "- my_module/\n", + " - __init__.py\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Defining the server extension\n", + "This example shows that the server extension and its `load_jupyter_server_extension` function are defined in the `__init__.py` file.\n", + "\n", + "#### `my_module/__init__.py`\n", + "\n", + "```python\n", + "def _jupyter_server_extension_paths():\n", + " return [{\n", + " \"module\": \"my_module\"\n", + " }]\n", + "\n", + "\n", + "def load_jupyter_server_extension(nbapp):\n", + " nbapp.log.info(\"my module enabled!\")\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Install and enable the server extension\n", + "Which a user can install with:\n", + "```\n", + "jupyter serverextension enable --py my_module [--sys-prefix]\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Example - Server extension and nbextension" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Creating a Python package with a server extension and nbextension\n", + "Here is another server extension, with a front-end module. It assumes this directory structure:\n", + "\n", + "```\n", + "- setup.py\n", + "- MANIFEST.in\n", + "- my_fancy_module/\n", + " - __init__.py\n", + " - static/\n", + " index.js\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "### Defining the server extension and nbextension\n", + "This example again shows that the server extension and its `load_jupyter_server_extension` function are defined in the `__init__.py` file. This time, there is also a function `_jupyter_nbextension_path` for the nbextension.\n", + "\n", + "#### `my_fancy_module/__init__.py`\n", + "\n", + "```python\n", + "def _jupyter_server_extension_paths():\n", + " return [{\n", + " \"module\": \"my_fancy_module\"\n", + " }]\n", + "\n", + "# Jupyter Extension points\n", + "def _jupyter_nbextension_paths():\n", + " return [dict(\n", + " section=\"notebook\",\n", + " # the path is relative to the `my_fancy_module` directory\n", + " src=\"static\",\n", + " # directory in the `nbextension/` namespace\n", + " dest=\"my_fancy_module\",\n", + " # _also_ in the `nbextension/` namespace\n", + " require=\"my_fancy_module/index\")]\n", + "\n", + "def load_jupyter_server_extension(nbapp):\n", + " nbapp.log.info(\"my module enabled!\")\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Install and enable the server extension and nbextension\n", + "\n", + "The user can install and enable the extensions with the following set of commands:\n", + "```\n", + "jupyter nbextension install --py my_fancy_module [--sys-prefix|--user]\n", + "jupyter nbextension enable --py my_fancy_module [--sys-prefix|--system]\n", + "jupyter serverextension enable --py my_fancy_module [--sys-prefix|--system]\n", + "```" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.1" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/docs/source/examples/Notebook/rstversions/Distributing Jupyter Extensions as Python Packages.rst b/docs/source/examples/Notebook/rstversions/Distributing Jupyter Extensions as Python Packages.rst new file mode 100644 index 0000000000..98c977d8ca --- /dev/null +++ b/docs/source/examples/Notebook/rstversions/Distributing Jupyter Extensions as Python Packages.rst @@ -0,0 +1,245 @@ + +`View the original notebook on nbviewer `__ + +Distributing Jupyter Extensions as Python Packages +================================================== + +Overview +-------- + +How can the notebook be extended? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The Jupyter Notebook client and server application are both deeply +customizable. Their behavior can be extended by creating, respectively: + +- nbextension: a notebook extension + + - a single JS file, or directory of JavaScript, Cascading + StyleSheets, etc. that contain at minimum a JavaScript module + packaged as an `AMD + modules `__ + that exports a function ``load_ipython_extension`` + +- server extension: an importable Python module + + - that implements ``load_jupyter_server_extension`` + +Why create a Python package for Jupyter extensions? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Since it is rare to have a server extension that does not have any +frontend components (an nbextension), for convenience and consistency, +all these client and server extensions with their assets can be packaged +and versioned together as a Python package with a few simple commands. +This makes installing the package of extensions easier and less +error-prone for the user. + +Installation of Jupyter Extensions +---------------------------------- + +Install a Python package containing Jupyter Extensions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +There are several ways that you may get a Python package containing +Jupyter Extensions. Commonly, you will use a package manager for your +system: + +.. code:: shell + + pip install helpful_package + # or + conda install helpful_package + # or + apt-get install helpful_package + + # where 'helpful_package' is a Python package containing one or more Jupyter Extensions + +Enable a Server Extension +~~~~~~~~~~~~~~~~~~~~~~~~~ + +The simplest case would be to enable a server extension which has no +frontend components. + +A ``pip`` user that wants their configuration stored in their home +directory would type the following command: + +.. code:: shell + + jupyter serverextension enable --py helpful_package + +Alternatively, a ``virtualenv`` or ``conda`` user can pass +``--sys-prefix`` which keeps their environment isolated and +reproducible. For example: + +.. code:: shell + + # Make sure that your virtualenv or conda environment is activated + [source] activate my-environment + + jupyter serverextension enable --py helpful_package --sys-prefix + +Install the nbextension assets +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If a package also has an nbextension with frontend assets that must be +available (but not neccessarily enabled by default), install these +assets with the following command: + +.. code:: shell + + jupyter nbextension install --py helpful_package # or --sys-prefix if using virtualenv or conda + +Enable nbextension assets +~~~~~~~~~~~~~~~~~~~~~~~~~ + +If a package has assets that should be loaded every time a Jupyter app +(e.g. lab, notebook, dashboard, terminal) is loaded in the browser, the +following command can be used to enable the nbextension: + +.. code:: shell + + jupyter nbextension enable --py helpful_package # or --sys-prefix if using virtualenv or conda + +Did it work? Check by listing Jupyter Extensions. +------------------------------------------------- + +After running one or more extension installation steps, you can list +what is presently known about nbextensions or server extension. The +following commands will list which extensions are available, whether +they are enabled, and other extension details: + +.. code:: shell + + jupyter nbextension list + jupyter serverextension list + +Additional resources on creating and distributing packages +---------------------------------------------------------- + + Of course, in addition to the files listed, there are number of + other files one needs to build a proper package. Here are some good + resources: - `The Hitchhiker's Guide to + Packaging `__ + - `Repository Structure and + Python `__ + by Kenneth Reitz + + How you distribute them, too, is important: - `Packaging and + Distributing + Projects `__ + - `conda: Building + packages `__ + + Here are some tools to get you started: - + `generator-nbextension `__ + +Example - Server extension +-------------------------- + +Creating a Python package with a server extension +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Here is an example of a python module which contains a server extension +directly on itself. It has this directory structure: + +:: + + - setup.py + - MANIFEST.in + - my_module/ + - __init__.py + +Defining the server extension +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This example shows that the server extension and its +``load_jupyter_server_extension`` function are defined in the +``__init__.py`` file. + +``my_module/__init__.py`` +^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code:: python + + def _jupyter_server_extension_paths(): + return [{ + "module": "my_module" + }] + + + def load_jupyter_server_extension(nbapp): + nbapp.log.info("my module enabled!") + +Install and enable the server extension +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Which a user can install with: + +:: + + jupyter serverextension enable --py my_module [--sys-prefix] + +Example - Server extension and nbextension +------------------------------------------ + +Creating a Python package with a server extension and nbextension +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Here is another server extension, with a front-end module. It assumes +this directory structure: + +:: + + - setup.py + - MANIFEST.in + - my_fancy_module/ + - __init__.py + - static/ + index.js + +Defining the server extension and nbextension +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This example again shows that the server extension and its +``load_jupyter_server_extension`` function are defined in the +``__init__.py`` file. This time, there is also a function +``_jupyter_nbextension_path`` for the nbextension. + +``my_fancy_module/__init__.py`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code:: python + + def _jupyter_server_extension_paths(): + return [{ + "module": "my_fancy_module" + }] + + # Jupyter Extension points + def _jupyter_nbextension_paths(): + return [dict( + section="notebook", + # the path is relative to the `my_fancy_module` directory + src="static", + # directory in the `nbextension/` namespace + dest="my_fancy_module", + # _also_ in the `nbextension/` namespace + require="my_fancy_module/index")] + + def load_jupyter_server_extension(nbapp): + nbapp.log.info("my module enabled!") + +Install and enable the server extension and nbextension +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The user can install and enable the extensions with the following set of +commands: + +:: + + jupyter nbextension install --py my_fancy_module [--sys-prefix|--user] + jupyter nbextension enable --py my_fancy_module [--sys-prefix|--system] + jupyter serverextension enable --py my_fancy_module [--sys-prefix|--system] + +`View the original notebook on nbviewer `__ diff --git a/docs/source/examples/Notebook/rstversions/JavaScript Notebook Extensions.rst b/docs/source/examples/Notebook/rstversions/JavaScript Notebook Extensions.rst index 5a465cbc28..826eaf5aeb 100644 --- a/docs/source/examples/Notebook/rstversions/JavaScript Notebook Extensions.rst +++ b/docs/source/examples/Notebook/rstversions/JavaScript Notebook Extensions.rst @@ -318,7 +318,7 @@ Exercise: ^^^^^^^^^ Try to wrap the all code in a file, put this file in -``{profile}/static/custom/.js``, and add +``{jupyter_dir}/custom/.js``, and add :: diff --git a/docs/source/examples/Notebook/rstversions/Notebook Basics.rst b/docs/source/examples/Notebook/rstversions/Notebook Basics.rst index 67b4324aa4..4f245db334 100644 --- a/docs/source/examples/Notebook/rstversions/Notebook Basics.rst +++ b/docs/source/examples/Notebook/rstversions/Notebook Basics.rst @@ -4,60 +4,6 @@ Notebook Basics =============== -Running the Notebook Server ---------------------------- - -The Jupyter notebook server is a custom web server that runs the -notebook web application. Most of the time, users run the notebook -server on their local computer using the command line interface. - -Starting the notebook server using the command line -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -You can start the notebook server from the command line (Terminal on -Mac/Linux, CMD prompt on Windows) by running the following command: - -:: - - jupyter notebook - -This will print some information about the notebook server in your -terminal, including the URL of the web application (by default, -``http://127.0.0.1:8888``). It will then open your default web browser -to this URL. - -When the notebook opens, you will see the **notebook dashboard**, which -will show a list of the notebooks, files, and subdirectories in the -directory where the notebook server was started (as seen in the next -section, below). Most of the time, you will want to start a notebook -server in the highest directory in your filesystem where notebooks can -be found. Often this will be your home directory. - -Additional options -~~~~~~~~~~~~~~~~~~ - -By default, the notebook server starts on port 8888. If port 8888 is -unavailable, the notebook server searchs the next available port. - -You can also specify the port manually: - -:: - - jupyter notebook --port 9999 - -Or start notebook server without opening a web browser. - -:: - - jupyter notebook --no-browser - -The notebook server has a number of other command line arguments that -can be displayed with the ``--help`` flag: - -:: - - jupyter notebook --help - The Notebook dashboard ---------------------- diff --git a/docs/source/extending/frontend_extensions.rst b/docs/source/extending/frontend_extensions.rst index e23dfdd533..405b362d9e 100644 --- a/docs/source/extending/frontend_extensions.rst +++ b/docs/source/extending/frontend_extensions.rst @@ -38,7 +38,7 @@ extension: }); .. note:: - + Although for historical reasons the function is called ``load_ipython_extension``, it does apply to the Jupyter notebook in general, and will work regardless of the kernel in use. @@ -112,7 +112,7 @@ place: }); .. note:: - + The standard keybindings might not work correctly on non-US keyboards. Unfortunately, this is a limitation of browser implementations and the status of keyboard event handling on the web in general. We appreciate your @@ -184,6 +184,7 @@ actions defined in an extension, it makes sense to use the extension name as the prefix. For the action name, the following guidelines should be considered: .. adapted from notebook/static/notebook/js/actions.js + * First pick a noun and a verb for the action. For example, if the action is "restart kernel," the verb is "restart" and the noun is "kernel". * Omit terms like "selected" and "active" by default, so "delete-cell", rather @@ -202,10 +203,12 @@ the prefix. For the action name, the following guidelines should be considered: Installing and enabling extensions ---------------------------------- -You can install your nbextension with the command: +You can install your nbextension with the command:: - jupyter nbextension install path/to/my_extension/ + jupyter nbextension install path/to/my_extension/ [--user|--sys-prefix] +The default installation is system-wide. You can use ``--user`` to do a per-user installation, +or ``--sys-prefix`` to install to Python's prefix (e.g. in a virtual or conda environment). Where my_extension is the directory containing the Javascript files. This will copy it to a Jupyter data directory (the exact location is platform dependent - see :ref:`jupyter_path`). @@ -214,11 +217,15 @@ For development, you can use the ``--symlink`` flag to symlink your extension rather than copying it, so there's no need to reinstall after changes. To use your extension, you'll also need to **enable** it, which tells the -notebook interface to load it. You can do that with another command: +notebook interface to load it. You can do that with another command:: - jupyter nbextension enable my_extension/main + jupyter nbextension enable my_extension/main [--sys-prefix] The argument refers to the Javascript module containing your ``load_ipython_extension`` function, which is ``my_extension/main.js`` in this example. There is a corresponding ``disable`` command to stop using an extension without uninstalling it. + +.. versionchanged:: 4.2 + + Added ``--sys-prefix`` argument diff --git a/docs/source/extending/index.rst b/docs/source/extending/index.rst index da6e4c36e2..d9805c4547 100644 --- a/docs/source/extending/index.rst +++ b/docs/source/extending/index.rst @@ -12,3 +12,4 @@ override the notebook's defaults with your own custom behavior. contents savehooks handlers + frontend_extensions diff --git a/docs/source/index.rst b/docs/source/index.rst index 5fdca5c346..0de6411c14 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -2,22 +2,6 @@ The Jupyter notebook ==================== -.. sidebar:: What's New in Jupyter Notebook - :subtitle: Release :ref:`release-4.1.0` - - `Release Announcement `_ - - - Cell toolbar selector moved to View menu - - Restart & Run All Cells added to Kernel menu - - Multiple-cell selection and actions including cut, copy, paste and execute - - Command palette added for executing Jupyter actions - - Find and replace added to Edit menu - - To upgrade to the release: - ``pip install notebook --upgrade`` - or - ``conda upgrade notebook`` - .. toctree:: :maxdepth: 1 :caption: User Documentation @@ -49,6 +33,7 @@ The Jupyter notebook :maxdepth: 1 :caption: Community documentation + examples/Notebook/rstversions/Distributing Jupyter Extensions as Python Packages examples/Notebook/rstversions/Examples and Tutorials Index .. toctree:: diff --git a/notebook/_version.py b/notebook/_version.py index 6a072c6837..cbb936081f 100644 --- a/notebook/_version.py +++ b/notebook/_version.py @@ -9,5 +9,5 @@ # Next beta/alpha/rc release: The version number for beta is X.Y.ZbN **without dots**. -version_info = (4, 2, 0, '.dev') +version_info = (4, 2, 1) __version__ = '.'.join(map(str, version_info[:3])) + ''.join(version_info[3:]) diff --git a/notebook/auth/login.py b/notebook/auth/login.py index 759d86bc7d..24ac954ca2 100644 --- a/notebook/auth/login.py +++ b/notebook/auth/login.py @@ -39,15 +39,15 @@ def hashed_password(self): def post(self): typed_password = self.get_argument('password', default=u'') + cookie_options = self.settings.get('cookie_options', {}) + cookie_options.setdefault('httponly', True) if self.login_available(self.settings): if passwd_check(self.hashed_password, typed_password): - # tornado <4.2 have a bug that consider secure==True as soon as + # tornado <4.2 has a bug that considers secure==True as soon as # 'secure' kwarg is passed to set_secure_cookie if self.settings.get('secure_cookie', self.request.protocol == 'https'): - kwargs = {'secure': True} - else: - kwargs = {} - self.set_secure_cookie(self.cookie_name, str(uuid.uuid4()), **kwargs) + cookie_options.setdefault('secure', True) + self.set_secure_cookie(self.cookie_name, str(uuid.uuid4()), **cookie_options) else: self.set_status(401) self._render(message={'error': 'Invalid password'}) diff --git a/notebook/base/handlers.py b/notebook/base/handlers.py index 127a6ba20c..9d3d2a691e 100644 --- a/notebook/base/handlers.py +++ b/notebook/base/handlers.py @@ -389,7 +389,6 @@ def finish(self, *args, **kwargs): self.set_header('Content-Type', 'application/json') return super(APIHandler, self).finish(*args, **kwargs) - @web.authenticated def options(self, *args, **kwargs): self.set_header('Access-Control-Allow-Headers', 'accept, content-type') self.set_header('Access-Control-Allow-Methods', diff --git a/notebook/nbconvert/handlers.py b/notebook/nbconvert/handlers.py index 8d8792d731..e5614956cf 100644 --- a/notebook/nbconvert/handlers.py +++ b/notebook/nbconvert/handlers.py @@ -7,7 +7,7 @@ import os import zipfile -from tornado import web +from tornado import web, escape from ..base.handlers import ( IPythonHandler, FilesRedirectHandler, @@ -38,7 +38,7 @@ def respond_zip(handler, name, output, resources): # Headers zip_filename = os.path.splitext(name)[0] + '.zip' handler.set_header('Content-Disposition', - 'attachment; filename="%s"' % zip_filename) + 'attachment; filename="%s"' % escape.url_escape(zip_filename)) handler.set_header('Content-Type', 'application/zip') # Prepare the zip file @@ -112,7 +112,7 @@ def get(self, format, path): if self.get_argument('download', 'false').lower() == 'true': filename = os.path.splitext(name)[0] + resources['output_extension'] self.set_header('Content-Disposition', - 'attachment; filename="%s"' % filename) + 'attachment; filename="%s"' % escape.url_escape(filename)) # MIME type if exporter.output_mimetype: diff --git a/notebook/nbextensions.py b/notebook/nbextensions.py index de40733469..c52dd242b7 100644 --- a/notebook/nbextensions.py +++ b/notebook/nbextensions.py @@ -11,7 +11,7 @@ import sys import tarfile import zipfile -from os.path import basename, join as pjoin +from os.path import basename, join as pjoin, normpath try: from urllib.parse import urlparse # Py3 @@ -20,63 +20,42 @@ from urlparse import urlparse from urllib import urlretrieve -from jupyter_core.paths import jupyter_data_dir, jupyter_path, SYSTEM_JUPYTER_PATH +from jupyter_core.paths import ( + jupyter_data_dir, jupyter_config_dir, jupyter_config_path, + SYSTEM_JUPYTER_PATH, ENV_JUPYTER_PATH, ENV_CONFIG_PATH, SYSTEM_CONFIG_PATH +) from ipython_genutils.path import ensure_dir_exists from ipython_genutils.py3compat import string_types, cast_unicode_py2 from ipython_genutils.tempdir import TemporaryDirectory from ._version import __version__ -class ArgumentConflict(ValueError): - pass +from traitlets.config.manager import BaseJSONConfigManager +from traitlets.utils.importstring import import_item +from tornado.log import LogFormatter -def _should_copy(src, dest, verbose=1): - """should a file be copied?""" - if not os.path.exists(dest): - return True - if os.stat(src).st_mtime - os.stat(dest).st_mtime > 1e-6: - # we add a fudge factor to work around a bug in python 2.x - # that was fixed in python 3.x: http://bugs.python.org/issue12904 - if verbose >= 2: - print("%s is out of date" % dest) - return True - if verbose >= 2: - print("%s is up to date" % dest) - return False +# Constants for pretty print extension listing function. +# Window doesn't support coloring in the commandline +GREEN_ENABLED = '\033[32m enabled \033[0m' if os.name != 'nt' else 'enabled ' +RED_DISABLED = '\033[31mdisabled\033[0m' if os.name != 'nt' else 'disabled' +DEPRECATED_ARGUMENT = object() -def _maybe_copy(src, dest, verbose=1): - """copy a file if it needs updating""" - if _should_copy(src, dest, verbose): - if verbose >= 1: - print("copying %s -> %s" % (src, dest)) - shutil.copy2(src, dest) +NBCONFIG_SECTIONS = ['common', 'notebook', 'tree', 'edit', 'terminal'] -def _safe_is_tarfile(path): - """safe version of is_tarfile, return False on IOError""" - try: - return tarfile.is_tarfile(path) - except IOError: - return False +GREEN_OK = '\033[32mOK\033[0m' if os.name != 'nt' else 'ok' +RED_X = '\033[31m X\033[0m' if os.name != 'nt' else ' X' +#------------------------------------------------------------------------------ +# Public API +#------------------------------------------------------------------------------ -def _get_nbext_dir(nbextensions_dir=None, user=False, prefix=None): - """Return the nbextension directory specified""" - if sum(map(bool, [user, prefix, nbextensions_dir])) > 1: - raise ArgumentConflict("Cannot specify more than one of user, prefix, or nbextensions_dir.") - if user: - nbext = pjoin(jupyter_data_dir(), u'nbextensions') - else: - if prefix: - nbext = pjoin(prefix, 'share', 'jupyter', 'nbextensions') - elif nbextensions_dir: - nbext = nbextensions_dir - else: - nbext = pjoin(SYSTEM_JUPYTER_PATH[0], 'nbextensions') - return nbext + +class ArgumentConflict(ValueError): + pass -def check_nbextension(files, user=False, prefix=None, nbextensions_dir=None): +def check_nbextension(files, user=False, prefix=None, nbextensions_dir=None, sys_prefix=False): """Check whether nbextension files have been installed Returns True if all files are found, False if any are missing. @@ -87,15 +66,17 @@ def check_nbextension(files, user=False, prefix=None, nbextensions_dir=None): files : list(paths) a list of relative paths within nbextensions. user : bool [default: False] - Whether to check the user's .ipython/nbextensions directory. + Whether to check the user's .jupyter/nbextensions directory. Otherwise check a system-wide install (e.g. /usr/local/share/jupyter/nbextensions). prefix : str [optional] Specify install prefix, if it should differ from default (e.g. /usr/local). Will check prefix/share/jupyter/nbextensions nbextensions_dir : str [optional] Specify absolute path of nbextensions directory explicitly. + sys_prefix : bool [default: False] + Install into the sys.prefix, i.e. environment """ - nbext = _get_nbext_dir(nbextensions_dir, user, prefix) + nbext = _get_nbextension_dir(user=user, sys_prefix=sys_prefix, prefix=prefix, nbextensions_dir=nbextensions_dir) # make sure nbextensions dir exists if not os.path.exists(nbext): return False @@ -107,7 +88,11 @@ def check_nbextension(files, user=False, prefix=None, nbextensions_dir=None): return all(os.path.exists(pjoin(nbext, f)) for f in files) -def install_nbextension(path, overwrite=False, symlink=False, user=False, prefix=None, nbextensions_dir=None, destination=None, verbose=1): +def install_nbextension(path, overwrite=False, symlink=False, + user=False, prefix=None, nbextensions_dir=None, + destination=None, verbose=DEPRECATED_ARGUMENT, + logger=None, sys_prefix=False + ): """Install a Javascript extension for the notebook Stages files and/or directories into the nbextensions directory. @@ -140,11 +125,17 @@ def install_nbextension(path, overwrite=False, symlink=False, user=False, prefix name the nbextension is installed to. For example, if destination is 'foo', then the source file will be installed to 'nbextensions/foo', regardless of the source name. This cannot be specified if an archive is given as the source. - verbose : int [default: 1] - Set verbosity level. The default is 1, where file actions are printed. - set verbose=2 for more output, or verbose=0 for silence. + logger : Jupyter logger [optional] + Logger instance to use """ - nbext = _get_nbext_dir(nbextensions_dir, user, prefix) + if verbose != DEPRECATED_ARGUMENT: + import warnings + warnings.warn("`install_nbextension`'s `verbose` parameter is deprecated, it will have no effects and will be removed in Notebook 5.0", DeprecationWarning) + + # the actual path to which we eventually installed + full_dest = None + + nbext = _get_nbextension_dir(user=user, sys_prefix=sys_prefix, prefix=prefix, nbextensions_dir=nbextensions_dir) # make sure nbextensions dir exists ensure_dir_exists(nbext) @@ -164,18 +155,19 @@ def install_nbextension(path, overwrite=False, symlink=False, user=False, prefix with TemporaryDirectory() as td: filename = urlparse(path).path.split('/')[-1] local_path = os.path.join(td, filename) - if verbose >= 1: - print("downloading %s to %s" % (path, local_path)) + if logger: + logger.info("Downloading: %s -> %s" % (path, local_path)) urlretrieve(path, local_path) # now install from the local copy - install_nbextension(local_path, overwrite=overwrite, symlink=symlink, nbextensions_dir=nbext, destination=destination, verbose=verbose) + full_dest = install_nbextension(local_path, overwrite=overwrite, symlink=symlink, + nbextensions_dir=nbext, destination=destination, logger=logger) elif path.endswith('.zip') or _safe_is_tarfile(path): if symlink: raise ValueError("Cannot symlink from archives") if destination: raise ValueError("Cannot give destination for archives") - if verbose >= 1: - print("extracting %s to %s" % (path, nbext)) + if logger: + logger.info("Extracting: %s -> %s" % (path, nbext)) if path.endswith('.zip'): archive = zipfile.ZipFile(path) @@ -183,14 +175,16 @@ def install_nbextension(path, overwrite=False, symlink=False, user=False, prefix archive = tarfile.open(path) archive.extractall(nbext) archive.close() + # TODO: what to do here + full_dest = None else: if not destination: destination = basename(path) destination = cast_unicode_py2(destination) - full_dest = pjoin(nbext, destination) + full_dest = normpath(pjoin(nbext, destination)) if overwrite and os.path.lexists(full_dest): - if verbose >= 1: - print("removing %s" % full_dest) + if logger: + logger.info("Removing: %s" % full_dest) if os.path.isdir(full_dest) and not os.path.islink(full_dest): shutil.rmtree(full_dest) else: @@ -199,60 +193,466 @@ def install_nbextension(path, overwrite=False, symlink=False, user=False, prefix if symlink: path = os.path.abspath(path) if not os.path.exists(full_dest): - if verbose >= 1: - print("symlink %s -> %s" % (full_dest, path)) + if logger: + logger.info("Symlinking: %s -> %s" % (full_dest, path)) os.symlink(path, full_dest) elif os.path.isdir(path): path = pjoin(os.path.abspath(path), '') # end in path separator for parent, dirs, files in os.walk(path): dest_dir = pjoin(full_dest, parent[len(path):]) if not os.path.exists(dest_dir): - if verbose >= 2: - print("making directory %s" % dest_dir) + if logger: + logger.info("Making directory: %s" % dest_dir) os.makedirs(dest_dir) for file in files: src = pjoin(parent, file) - # print("%r, %r" % (dest_dir, file)) dest_file = pjoin(dest_dir, file) - _maybe_copy(src, dest_file, verbose) + _maybe_copy(src, dest_file, logger=logger) else: src = path - _maybe_copy(src, full_dest, verbose) + _maybe_copy(src, full_dest, logger=logger) + + return full_dest + + +def install_nbextension_python(module, overwrite=False, symlink=False, + user=False, sys_prefix=False, prefix=None, nbextensions_dir=None, logger=None): + """Install an nbextension bundled in a Python package. + + Returns a list of installed/updated directories. + + See install_nbextension for parameter information.""" + m, nbexts = _get_nbextension_metadata(module) + base_path = os.path.split(m.__file__)[0] + + full_dests = [] + + for nbext in nbexts: + src = os.path.join(base_path, nbext['src']) + dest = nbext['dest'] + + if logger: + logger.info("Installing %s -> %s" % (src, dest)) + full_dest = install_nbextension( + src, overwrite=overwrite, symlink=symlink, + user=user, sys_prefix=sys_prefix, prefix=prefix, nbextensions_dir=nbextensions_dir, + destination=dest, logger=logger + ) + validate_nbextension_python(nbext, full_dest, logger) + full_dests.append(full_dest) + + return full_dests + + +def uninstall_nbextension(dest, require=None, user=False, sys_prefix=False, prefix=None, + nbextensions_dir=None, logger=None): + """Uninstall a Javascript extension of the notebook + + Removes staged files and/or directories in the nbextensions directory and + removes the extension from the frontend config. + + Parameters + ---------- + + dest : str + path to file, directory, zip or tarball archive, or URL to install + name the nbextension is installed to. For example, if destination is 'foo', then + the source file will be installed to 'nbextensions/foo', regardless of the source name. + This cannot be specified if an archive is given as the source. + require : str [optional] + require.js path used to load the extension. + If specified, frontend config loading extension will be removed. + user : bool [default: False] + Whether to install to the user's nbextensions directory. + Otherwise do a system-wide install (e.g. /usr/local/share/jupyter/nbextensions). + prefix : str [optional] + Specify install prefix, if it should differ from default (e.g. /usr/local). + Will install to ``/share/jupyter/nbextensions`` + nbextensions_dir : str [optional] + Specify absolute path of nbextensions directory explicitly. + logger : Jupyter logger [optional] + Logger instance to use + """ + nbext = _get_nbextension_dir(user=user, sys_prefix=sys_prefix, prefix=prefix, nbextensions_dir=nbextensions_dir) + dest = cast_unicode_py2(dest) + full_dest = pjoin(nbext, dest) + if os.path.lexists(full_dest): + if logger: + logger.info("Removing: %s" % full_dest) + if os.path.isdir(full_dest) and not os.path.islink(full_dest): + shutil.rmtree(full_dest) + else: + os.remove(full_dest) + + # Look through all of the config sections making sure that the nbextension + # doesn't exist. + config_dir = os.path.join(_get_config_dir(user=user, sys_prefix=sys_prefix), 'nbconfig') + cm = BaseJSONConfigManager(config_dir=config_dir) + if require: + for section in NBCONFIG_SECTIONS: + cm.update(section, {"load_extensions": {require: None}}) + + +def uninstall_nbextension_python(module, + user=False, sys_prefix=False, prefix=None, nbextensions_dir=None, + logger=None): + """Uninstall an nbextension bundled in a Python package. + + See parameters of `install_nbextension_python` + """ + m, nbexts = _get_nbextension_metadata(module) + for nbext in nbexts: + dest = nbext['dest'] + require = nbext['require'] + if logger: + logger.info("Uninstalling {} {}".format(dest, require)) + uninstall_nbextension(dest, require, user=user, sys_prefix=sys_prefix, + prefix=prefix, nbextensions_dir=nbextensions_dir, logger=logger) + + +def _set_nbextension_state(section, require, state, + user=True, sys_prefix=False, logger=None): + """Set whether the section's frontend should require the named nbextension + + Returns True if the final state is the one requested. + + Parameters + ---------- + section : string + The section of the server to change, one of NBCONFIG_SECTIONS + require : string + An importable AMD module inside the nbextensions static path + state : bool + The state in which to leave the extension + user : bool [default: True] + Whether to update the user's .jupyter/nbextensions directory + sys_prefix : bool [default: False] + Whether to update the sys.prefix, i.e. environment. Will override + `user`. + logger : Jupyter logger [optional] + Logger instance to use + """ + user = False if sys_prefix else user + config_dir = os.path.join( + _get_config_dir(user=user, sys_prefix=sys_prefix), 'nbconfig') + cm = BaseJSONConfigManager(config_dir=config_dir) + if logger: + logger.info("{} {} extension {}...".format( + "Enabling" if state else "Disabling", + section, + require + )) + cm.update(section, {"load_extensions": {require: state}}) + + validate_nbextension(require, logger=logger) + + return cm.get(section).get(require) == state + + +def _set_nbextension_state_python(state, module, user, sys_prefix, + logger=None): + """Enable or disable some nbextensions stored in a Python package + + Returns a list of whether the state was achieved (i.e. changed, or was + already right) + + Parameters + ---------- + + state : Bool + Whether the extensions should be enabled + module : str + Importable Python module exposing the + magic-named `_jupyter_nbextension_paths` function + user : bool + Whether to enable in the user's nbextensions directory. + sys_prefix : bool + Enable/disable in the sys.prefix, i.e. environment + logger : Jupyter logger [optional] + Logger instance to use + """ + m, nbexts = _get_nbextension_metadata(module) + return [_set_nbextension_state(section=nbext["section"], + require=nbext["require"], + state=state, + user=user, sys_prefix=sys_prefix, + logger=logger) + for nbext in nbexts] + + +def enable_nbextension(section, require, user=True, sys_prefix=False, + logger=None): + """Enable a named nbextension + + Returns True if the final state is the one requested. + + Parameters + ---------- + + section : string + The section of the server to change, one of NBCONFIG_SECTIONS + require : string + An importable AMD module inside the nbextensions static path + user : bool [default: True] + Whether to enable in the user's nbextensions directory. + sys_prefix : bool [default: False] + Whether to enable in the sys.prefix, i.e. environment. Will override + `user` + logger : Jupyter logger [optional] + Logger instance to use + """ + return _set_nbextension_state(section=section, require=require, + state=True, + user=user, sys_prefix=sys_prefix, + logger=logger) + + +def disable_nbextension(section, require, user=True, sys_prefix=False, + logger=None): + """Disable a named nbextension + + Returns True if the final state is the one requested. + + Parameters + ---------- + + section : string + The section of the server to change, one of NBCONFIG_SECTIONS + require : string + An importable AMD module inside the nbextensions static path + user : bool [default: True] + Whether to enable in the user's nbextensions directory. + sys_prefix : bool [default: False] + Whether to enable in the sys.prefix, i.e. environment. Will override + `user`. + logger : Jupyter logger [optional] + Logger instance to use + """ + return _set_nbextension_state(section=section, require=require, + state=False, + user=user, sys_prefix=sys_prefix, + logger=logger) + + +def enable_nbextension_python(module, user=True, sys_prefix=False, + logger=None): + """Enable some nbextensions associated with a Python module. + + Returns a list of whether the state was achieved (i.e. changed, or was + already right) + + Parameters + ---------- + + module : str + Importable Python module exposing the + magic-named `_jupyter_nbextension_paths` function + user : bool [default: True] + Whether to enable in the user's nbextensions directory. + sys_prefix : bool [default: False] + Whether to enable in the sys.prefix, i.e. environment. Will override + `user` + logger : Jupyter logger [optional] + Logger instance to use + """ + return _set_nbextension_state_python(True, module, user, sys_prefix, + logger=logger) + + +def disable_nbextension_python(module, user=True, sys_prefix=False, + logger=None): + """Disable some nbextensions associated with a Python module. + + Returns True if the final state is the one requested. + + Parameters + ---------- + + module : str + Importable Python module exposing the + magic-named `_jupyter_nbextension_paths` function + user : bool [default: True] + Whether to enable in the user's nbextensions directory. + sys_prefix : bool [default: False] + Whether to enable in the sys.prefix, i.e. environment + logger : Jupyter logger [optional] + Logger instance to use + """ + return _set_nbextension_state_python(False, module, user, sys_prefix, + logger=logger) + + +def validate_nbextension(require, logger=None): + """Validate a named nbextension. + + Looks across all of the nbextension directories. + + Returns a list of warnings. + + require : str + require.js path used to load the extension + logger : Jupyter logger [optional] + Logger instance to use + """ + warnings = [] + infos = [] + + js_exists = False + for exts in _nbextension_dirs(): + # Does the Javascript entrypoint actually exist on disk? + js = u"{}.js".format(os.path.join(exts, *require.split("/"))) + js_exists = os.path.exists(js) + if js_exists: + break + + require_tmpl = u" - require? {} {}" + if js_exists: + infos.append(require_tmpl.format(GREEN_OK, require)) + else: + warnings.append(require_tmpl.format(RED_X, require)) + + if logger: + if warnings: + logger.warning(u" - Validating: problems found:") + for msg in warnings: + logger.warning(msg) + for msg in infos: + logger.info(msg) + else: + logger.info(u" - Validating: {}".format(GREEN_OK)) + + return warnings + + +def validate_nbextension_python(spec, full_dest, logger=None): + """Assess the health of an installed nbextension + + Returns a list of warnings. + + Parameters + ---------- + + spec : dict + A single entry of _jupyter_nbextension_paths(): + [{ + 'section': 'notebook', + 'src': 'mockextension', + 'dest': '_mockdestination', + 'require': '_mockdestination/index' + }] + full_dest : str + The on-disk location of the installed nbextension: this should end + with `nbextensions/` + logger : Jupyter logger [optional] + Logger instance to use + """ + infos = [] + warnings = [] + + section = spec.get("section", None) + if section in NBCONFIG_SECTIONS: + infos.append(u" {} section: {}".format(GREEN_OK, section)) + else: + warnings.append(u" {} section: {}".format(RED_X, section)) + + require = spec.get("require", None) + if require is not None: + require_path = os.path.join( + full_dest[0:-len(spec["dest"])], + u"{}.js".format(require)) + if os.path.exists(require_path): + infos.append(u" {} require: {}".format(GREEN_OK, require_path)) + else: + warnings.append(u" {} require: {}".format(RED_X, require_path)) + + if logger: + if warnings: + logger.warning("- Validating: problems found:") + for msg in warnings: + logger.warning(msg) + for msg in infos: + logger.info(msg) + logger.warning(u"Full spec: {}".format(spec)) + else: + logger.info(u"- Validating: {}".format(GREEN_OK)) + + return warnings + #---------------------------------------------------------------------- -# install nbextension app +# Applications #---------------------------------------------------------------------- -from traitlets import Bool, Enum, Unicode +from traitlets import Bool, Unicode, Any from jupyter_core.application import JupyterApp -flags = { +_base_flags = {} +_base_flags.update(JupyterApp.flags) +_base_flags.pop("y", None) +_base_flags.pop("generate-config", None) +_base_flags.update({ + "user" : ({ + "BaseNBExtensionApp" : { + "user" : True, + }}, "Apply the operation only for the given user" + ), + "system" : ({ + "BaseNBExtensionApp" : { + "user" : False, + "sys_prefix": False, + }}, "Apply the operation system-wide" + ), + "sys-prefix" : ({ + "BaseNBExtensionApp" : { + "sys_prefix" : True, + }}, "Use sys.prefix as the prefix for installing nbextensions (for environments, packaging)" + ), + "py" : ({ + "BaseNBExtensionApp" : { + "python" : True, + }}, "Install from a Python package" + ) +}) +_base_flags['python'] = _base_flags['py'] + +class BaseNBExtensionApp(JupyterApp): + """Base nbextension installer app""" + _log_formatter_cls = LogFormatter + flags = _base_flags + version = __version__ + + user = Bool(False, config=True, help="Whether to do a user install") + sys_prefix = Bool(False, config=True, help="Use the sys.prefix as the prefix") + python = Bool(False, config=True, help="Install from a Python package") + + # Remove for 5.0... + verbose = Any(None, config=True, help="DEPRECATED: Verbosity level") + + def _verbose_changed(self): + """Warn about verbosity changes""" + import warnings + warnings.warn("`verbose` traits of `{}` has been deprecated, has no effects and will be removed in notebook 5.0.".format(type(self).__name__), DeprecationWarning) + + def _log_format_default(self): + """A default format for messages""" + return "%(message)s" + + +flags = {} +flags.update(_base_flags) +flags.update({ "overwrite" : ({ "InstallNBExtensionApp" : { "overwrite" : True, }}, "Force overwrite of existing files" ), - "debug" : ({ - "InstallNBExtensionApp" : { - "verbose" : 2, - }}, "Extra output" - ), - "quiet" : ({ - "InstallNBExtensionApp" : { - "verbose" : 0, - }}, "Minimal output" - ), "symlink" : ({ "InstallNBExtensionApp" : { "symlink" : True, }}, "Create symlink instead of copying files" ), - "user" : ({ - "InstallNBExtensionApp" : { - "user" : True, - }}, "Install to the user's IPython directory" - ), -} +}) + flags['s'] = flags['symlink'] aliases = { @@ -261,14 +661,13 @@ def install_nbextension(path, overwrite=False, symlink=False, user=False, prefix "destination" : "InstallNBExtensionApp.destination", } -class InstallNBExtensionApp(JupyterApp): +class InstallNBExtensionApp(BaseNBExtensionApp): """Entry point for installing notebook extensions""" - version = __version__ description = """Install Jupyter notebook extensions Usage - jupyter nbextension install path/url + jupyter nbextension install path|url [--user|--sys-prefix] This copies a file or a folder into the Jupyter nbextensions directory. If a URL is given, it will be downloaded. @@ -285,132 +684,483 @@ class InstallNBExtensionApp(JupyterApp): overwrite = Bool(False, config=True, help="Force overwrite of existing files") symlink = Bool(False, config=True, help="Create symlinks instead of copying files") - user = Bool(False, config=True, help="Whether to do a user install") + prefix = Unicode('', config=True, help="Installation prefix") - nbextensions_dir = Unicode('', config=True, help="Full path to nbextensions dir (probably use prefix or user)") + nbextensions_dir = Unicode('', config=True, + help="Full path to nbextensions dir (probably use prefix or user)") destination = Unicode('', config=True, help="Destination for the copy or symlink") - verbose = Enum((0,1,2), default_value=1, config=True, - help="Verbosity level" - ) def _config_file_name_default(self): + """The default config file name.""" return 'jupyter_notebook_config' def install_extensions(self): + """Perform the installation of nbextension(s)""" if len(self.extra_args)>1: - raise ValueError("only one nbextension allowed at a time. Call multiple times to install multiple extensions.") - install_nbextension(self.extra_args[0], - overwrite=self.overwrite, - symlink=self.symlink, - verbose=self.verbose, - user=self.user, - prefix=self.prefix, - destination=self.destination, - nbextensions_dir=self.nbextensions_dir, - ) - + raise ValueError("Only one nbextension allowed at a time. " + "Call multiple times to install multiple extensions.") + + if self.python: + install = install_nbextension_python + kwargs = {} + else: + install = install_nbextension + kwargs = {'destination': self.destination} + + full_dests = install(self.extra_args[0], + overwrite=self.overwrite, + symlink=self.symlink, + user=self.user, + sys_prefix=self.sys_prefix, + prefix=self.prefix, + nbextensions_dir=self.nbextensions_dir, + logger=self.log, + **kwargs + ) + + if full_dests: + self.log.info( + u"\nTo initialize this nbextension in the browser every time" + " the notebook (or other app) loads:\n\n" + " jupyter nbextension enable {}{}{}{}\n".format( + self.extra_args[0] if self.python else "", + " --user" if self.user else "", + " --py" if self.python else "", + " --sys-prefix" if self.sys_prefix else "" + ) + ) + def start(self): + """Perform the App's function as configured""" if not self.extra_args: - for nbext in jupyter_path('nbextensions'): - if os.path.exists(nbext): - print("Notebook extensions in %s:" % nbext) - for ext in os.listdir(nbext): - print(u" %s" % ext) + sys.exit('Please specify an nbextension to install') else: try: self.install_extensions() except ArgumentConflict as e: - print(str(e), file=sys.stderr) - self.exit(1) + sys.exit(str(e)) -class EnableNBExtensionApp(JupyterApp): - name = "jupyter nbextension enable" +class UninstallNBExtensionApp(BaseNBExtensionApp): + """Entry point for uninstalling notebook extensions""" version = __version__ - description = "Configure an nbextension to be automatically loaded" + description = """Uninstall Jupyter notebook extensions - section = Unicode('notebook', config=True, - help=("Which config section to add the extension to. " - "'common' will affect all pages.") - ) + Usage + + jupyter nbextension uninstall path/url path/url/entrypoint + jupyter nbextension uninstall --py pythonPackageName + + This uninstalls an nbextension. + """ - aliases = {'section': 'EnableNBExtensionApp.section', - } + examples = """ + jupyter nbextension uninstall dest/dir dest/dir/extensionjs + jupyter nbextension uninstall --py extensionPyPackage + """ + + aliases = { + "prefix" : "UninstallNBExtensionApp.prefix", + "nbextensions" : "UninstallNBExtensionApp.nbextensions_dir", + "require": "UninstallNBExtensionApp.require", + } + + prefix = Unicode('', config=True, help="Installation prefix") + nbextensions_dir = Unicode('', config=True, help="Full path to nbextensions dir (probably use prefix or user)") + require = Unicode('', config=True, help="require.js module to load.") def _config_file_name_default(self): + """The default config file name.""" return 'jupyter_notebook_config' - def enable_nbextension(self, name): - # Local import to avoid circular import issue on Py 2 - from .services.config import ConfigManager - cm = ConfigManager(parent=self, config=self.config) - cm.update(self.section, {"load_extensions": {name: True}}) + def uninstall_extensions(self): + """Uninstall some nbextensions""" + kwargs = { + 'user': self.user, + 'sys_prefix': self.sys_prefix, + 'prefix': self.prefix, + 'nbextensions_dir': self.nbextensions_dir, + 'logger': self.log + } + + arg_count = 1 + if len(self.extra_args) > arg_count: + raise ValueError("only one nbextension allowed at a time. Call multiple times to uninstall multiple extensions.") + if len(self.extra_args) < arg_count: + raise ValueError("not enough arguments") + + if self.python: + uninstall_nbextension_python(self.extra_args[0], **kwargs) + else: + if self.require: + kwargs['require'] = self.require + uninstall_nbextension(self.extra_args[0], **kwargs) def start(self): if not self.extra_args: - sys.exit('No extensions specified') - elif len(self.extra_args) > 1: - sys.exit('Please specify one extension at a time') - - self.enable_nbextension(self.extra_args[0]) + sys.exit('Please specify an nbextension to uninstall') + else: + try: + self.uninstall_extensions() + except ArgumentConflict as e: + sys.exit(str(e)) -class DisableNBExtensionApp(JupyterApp): - name = "jupyter nbextension disable" +class ToggleNBExtensionApp(BaseNBExtensionApp): + """A base class for apps that enable/disable extensions""" + name = "jupyter nbextension enable/disable" version = __version__ - description = "Remove the configuration to automatically load an extension" + description = "Enable/disable an nbextension in configuration." section = Unicode('notebook', config=True, - help=("Which config section to remove the extension from. " - "This should match the one it was previously added to.") + help="""Which config section to add the extension to, 'common' will affect all pages.""" ) + user = Bool(True, config=True, help="Apply the configuration only for the current user (default)") - aliases = {'section': 'DisableNBExtensionApp.section', - } + aliases = {'section': 'ToggleNBExtensionApp.section'} + + _toggle_value = None def _config_file_name_default(self): + """The default config file name.""" return 'jupyter_notebook_config' - def disable_nbextension(self, name): - # Local import to avoid circular import issue on Py 2 - from .services.config import ConfigManager - cm = ConfigManager(parent=self, config=self.config) - if name not in cm.get(self.section).get('load_extensions', {}): - sys.exit('{} is not enabled in section {}'.format(name, self.section)) - # We're using a dict as a set - updating with None removes the key - cm.update(self.section, {"load_extensions": {name: None}}) + def toggle_nbextension_python(self, module): + """Toggle some extensions in an importable Python module. + + Returns a list of booleans indicating whether the state was changed as + requested. + + Parameters + ---------- + module : str + Importable Python module exposing the + magic-named `_jupyter_nbextension_paths` function + """ + toggle = (enable_nbextension_python if self._toggle_value + else disable_nbextension_python) + return toggle(module, + user=self.user, + sys_prefix=self.sys_prefix, + logger=self.log) + + def toggle_nbextension(self, require): + """Toggle some a named nbextension by require-able AMD module. + + Returns whether the state was changed as requested. + + Parameters + ---------- + require : str + require.js path used to load the nbextension + """ + toggle = (enable_nbextension if self._toggle_value + else disable_nbextension) + return toggle(self.section, require, + user=self.user, sys_prefix=self.sys_prefix, + logger=self.log) def start(self): if not self.extra_args: - sys.exit('No extensions specified') + sys.exit('Please specify an nbextension/package to enable or disable') elif len(self.extra_args) > 1: - sys.exit('Please specify one extension at a time') + sys.exit('Please specify one nbextension/package at a time') + if self.python: + self.toggle_nbextension_python(self.extra_args[0]) + else: + self.toggle_nbextension(self.extra_args[0]) + + +class EnableNBExtensionApp(ToggleNBExtensionApp): + """An App that enables nbextensions""" + name = "jupyter nbextension enable" + description = """ + Enable an nbextension in frontend configuration. + + Usage + jupyter nbextension enable [--system|--sys-prefix] + """ + _toggle_value = True + + +class DisableNBExtensionApp(ToggleNBExtensionApp): + """An App that disables nbextensions""" + name = "jupyter nbextension disable" + description = """ + Enable an nbextension in frontend configuration. + + Usage + jupyter nbextension disable [--system|--sys-prefix] + """ + _toggle_value = None + + +class ListNBExtensionsApp(BaseNBExtensionApp): + """An App that lists and validates nbextensions""" + name = "jupyter nbextension list" + version = __version__ + description = "List all nbextensions known by the configuration system" + + def list_nbextensions(self): + """List all the nbextensions""" + config_dirs = [os.path.join(p, 'nbconfig') for p in jupyter_config_path()] + + print("Known nbextensions:") + + for config_dir in config_dirs: + head = u' config dir: {}'.format(config_dir) + head_shown = False - self.disable_nbextension(self.extra_args[0]) + cm = BaseJSONConfigManager(parent=self, config_dir=config_dir) + for section in NBCONFIG_SECTIONS: + data = cm.get(section) + if 'load_extensions' in data: + if not head_shown: + # only show heading if there is an nbextension here + print(head) + head_shown = True + print(u' {} section'.format(section)) -class NBExtensionApp(JupyterApp): + for require, enabled in data['load_extensions'].items(): + print(u' {} {}'.format( + require, + GREEN_ENABLED if enabled else RED_DISABLED)) + if enabled: + validate_nbextension(require, logger=self.log) + + def start(self): + """Perform the App's functions as configured""" + self.list_nbextensions() + + +_examples = """ +jupyter nbextension list # list all configured nbextensions +jupyter nbextension install --py # install an nbextension from a Python package +jupyter nbextension enable --py # enable all nbextensions in a Python package +jupyter nbextension disable --py # disable all nbextensions in a Python package +jupyter nbextension uninstall --py # uninstall an nbextension in a Python package +""" + +class NBExtensionApp(BaseNBExtensionApp): + """Base jupyter nbextension command entry point""" name = "jupyter nbextension" version = __version__ description = "Work with Jupyter notebook extensions" + examples = _examples subcommands = dict( - install=(InstallNBExtensionApp, - """Install notebook extensions""" - ), - enable=(EnableNBExtensionApp, "Enable a notebook extension"), - disable=(DisableNBExtensionApp, "Disable a notebook extension"), + install=(InstallNBExtensionApp,"Install an nbextension"), + enable=(EnableNBExtensionApp, "Enable an nbextension"), + disable=(DisableNBExtensionApp, "Disable an nbextension"), + uninstall=(UninstallNBExtensionApp, "Uninstall an nbextension"), + list=(ListNBExtensionsApp, "List nbextensions") ) def start(self): + """Perform the App's functions as configured""" super(NBExtensionApp, self).start() # The above should have called a subcommand and raised NoStart; if we - # get here, it didn't, so we should print a message. + # get here, it didn't, so we should self.log.info a message. subcmds = ", ".join(sorted(self.subcommands)) sys.exit("Please supply at least one subcommand: %s" % subcmds) main = NBExtensionApp.launch_instance +#------------------------------------------------------------------------------ +# Private API +#------------------------------------------------------------------------------ + + +def _should_copy(src, dest, logger=None): + """Should a file be copied, if it doesn't exist, or is newer? + + Returns whether the file needs to be updated. + + Parameters + ---------- + + src : string + A path that should exist from which to copy a file + src : string + A path that might exist to which to copy a file + logger : Jupyter logger [optional] + Logger instance to use + """ + if not os.path.exists(dest): + return True + if os.stat(src).st_mtime - os.stat(dest).st_mtime > 1e-6: + # we add a fudge factor to work around a bug in python 2.x + # that was fixed in python 3.x: http://bugs.python.org/issue12904 + if logger: + logger.warn("Out of date: %s" % dest) + return True + if logger: + logger.info("Up to date: %s" % dest) + return False + + +def _maybe_copy(src, dest, logger=None): + """Copy a file if it needs updating. + + Parameters + ---------- + + src : string + A path that should exist from which to copy a file + src : string + A path that might exist to which to copy a file + logger : Jupyter logger [optional] + Logger instance to use + """ + if _should_copy(src, dest, logger=logger): + if logger: + logger.info("Copying: %s -> %s" % (src, dest)) + shutil.copy2(src, dest) + + +def _safe_is_tarfile(path): + """Safe version of is_tarfile, return False on IOError. + + Returns whether the file exists and is a tarfile. + + Parameters + ---------- + + path : string + A path that might not exist and or be a tarfile + """ + try: + return tarfile.is_tarfile(path) + except IOError: + return False + + +def _get_nbextension_dir(user=False, sys_prefix=False, prefix=None, nbextensions_dir=None): + """Return the nbextension directory specified + + Parameters + ---------- + + user : bool [default: False] + Get the user's .jupyter/nbextensions directory + sys_prefix : bool [default: False] + Get sys.prefix, i.e. ~/.envs/my-env/share/jupyter/nbextensions + prefix : str [optional] + Get custom prefix + nbextensions_dir : str [optional] + Get what you put in + """ + if sum(map(bool, [user, prefix, nbextensions_dir, sys_prefix])) > 1: + raise ArgumentConflict("cannot specify more than one of user, sys_prefix, prefix, or nbextensions_dir") + if user: + nbext = pjoin(jupyter_data_dir(), u'nbextensions') + elif sys_prefix: + nbext = pjoin(ENV_JUPYTER_PATH[0], u'nbextensions') + elif prefix: + nbext = pjoin(prefix, 'share', 'jupyter', 'nbextensions') + elif nbextensions_dir: + nbext = nbextensions_dir + else: + nbext = pjoin(SYSTEM_JUPYTER_PATH[0], 'nbextensions') + return nbext + + +def _nbextension_dirs(): + """The possible locations of nbextensions. + + Returns a list of known base extension locations + """ + return [ + pjoin(jupyter_data_dir(), u'nbextensions'), + pjoin(ENV_JUPYTER_PATH[0], u'nbextensions'), + pjoin(SYSTEM_JUPYTER_PATH[0], 'nbextensions') + ] + + +def _get_config_dir(user=False, sys_prefix=False): + """Get the location of config files for the current context + + Returns the string to the enviornment + + Parameters + ---------- + + user : bool [default: False] + Get the user's .jupyter config directory + sys_prefix : bool [default: False] + Get sys.prefix, i.e. ~/.envs/my-env/etc/jupyter + """ + user = False if sys_prefix else user + if user and sys_prefix: + raise ArgumentConflict("Cannot specify more than one of user or sys_prefix") + if user: + nbext = jupyter_config_dir() + elif sys_prefix: + nbext = ENV_CONFIG_PATH[0] + else: + nbext = SYSTEM_CONFIG_PATH[0] + return nbext + + +def _get_nbextension_metadata(module): + """Get the list of nbextension paths associated with a Python module. + + Returns a tuple of (the module, [{ + 'section': 'notebook', + 'src': 'mockextension', + 'dest': '_mockdestination', + 'require': '_mockdestination/index' + }]) + + Parameters + ---------- + + module : str + Importable Python module exposing the + magic-named `_jupyter_nbextension_paths` function + """ + m = import_item(module) + if not hasattr(m, '_jupyter_nbextension_paths'): + raise KeyError('The Python module {} is not a valid nbextension'.format(module)) + nbexts = m._jupyter_nbextension_paths() + return m, nbexts + + +def _read_config_data(user=False, sys_prefix=False): + """Get the config for the current context + + Returns the string to the enviornment + + Parameters + ---------- + + user : bool [default: False] + Get the user's .jupyter config directory + sys_prefix : bool [default: False] + Get sys.prefix, i.e. ~/.envs/my-env/etc/jupyter + """ + config_dir = _get_config_dir(user=user, sys_prefix=sys_prefix) + config_man = BaseJSONConfigManager(config_dir=config_dir) + return config_man.get('jupyter_notebook_config') + + +def _write_config_data(data, user=False, sys_prefix=False): + """Update the config for the current context + + Parameters + ---------- + data : object + An object which can be accepted by ConfigManager.update + user : bool [default: False] + Get the user's .jupyter config directory + sys_prefix : bool [default: False] + Get sys.prefix, i.e. ~/.envs/my-env/etc/jupyter + """ + config_dir = _get_config_dir(user=user, sys_prefix=sys_prefix) + config_man = BaseJSONConfigManager(config_dir=config_dir) + config_man.update('jupyter_notebook_config', data) + + if __name__ == '__main__': main() - diff --git a/notebook/notebookapp.py b/notebook/notebookapp.py index 98e6568ee2..185aa124d4 100644 --- a/notebook/notebookapp.py +++ b/notebook/notebookapp.py @@ -13,6 +13,7 @@ import io import json import logging +import mimetypes import os import random import re @@ -114,15 +115,6 @@ def load_handlers(name): return mod.default_handlers -class DeprecationHandler(IPythonHandler): - def get(self, url_path): - self.set_header("Content-Type", 'text/javascript') - self.finish(""" - console.warn('`/static/widgets/js` is deprecated. Use `nbextensions/widgets/widgets/js` instead.'); - define(['%s'], function(x) { return x; }); - """ % url_path_join('nbextensions', 'widgets', 'widgets', url_path.rstrip('.js'))) - self.log.warn('Deprecated widget Javascript path /static/widgets/js/*.js was used') - #----------------------------------------------------------------------------- # The Tornado web application #----------------------------------------------------------------------------- @@ -225,7 +217,6 @@ def init_handlers(self, settings): # Order matters. The first handler to match the URL will handle the request. handlers = [] - handlers.append((r'/deprecatedwidgets/(.*)', DeprecationHandler)) handlers.extend(load_handlers('tree.handlers')) handlers.extend([(r"/login", settings['login_handler_class'])]) handlers.extend([(r"/logout", settings['logout_handler_class'])]) @@ -244,16 +235,20 @@ def init_handlers(self, settings): handlers.extend(load_handlers('services.security.handlers')) # BEGIN HARDCODED WIDGETS HACK + # TODO: Remove on notebook 5.0 try: - import ipywidgets - handlers.append( - (r"/nbextensions/widgets/(.*)", FileFindHandler, { - 'path': ipywidgets.find_static_assets(), - 'no_cache_paths': ['/'], # don't cache anything in nbextensions - }), - ) + import widgetsnbextension except: - app_log.warn('ipywidgets package not installed. Widgets are unavailable.') + try: + import ipywidgets as widgets + handlers.append( + (r"/nbextensions/widgets/(.*)", FileFindHandler, { + 'path': widgets.find_static_assets(), + 'no_cache_paths': ['/'], # don't cache anything in nbextensions + }), + ) + except: + app_log.warning('Widgets are unavailable. Please install widgetsnbextension or ipywidgets 4.0') # END HARDCODED WIDGETS HACK handlers.append( @@ -554,6 +549,10 @@ def _webapp_settings_changed(self, name, old, new): help="Supply overrides for the tornado.web.Application that the " "Jupyter notebook uses.") + cookie_options = Dict(config=True, + help="Extra keyword arguments to pass to `set_secure_cookie`." + " See tornado's set_secure_cookie docs for details." + ) ssl_options = Dict(config=True, help="""Supply SSL options for the tornado HTTPServer. See the tornado docs for details.""") @@ -782,9 +781,18 @@ def _notebook_dir_changed(self, name, old, new): self.config.FileContentsManager.root_dir = new self.config.MappingKernelManager.root_dir = new + # TODO: Remove me in notebook 5.0 server_extensions = List(Unicode(), config=True, - help=("Python modules to load as notebook server extensions. " - "This is an experimental API, and may change in future releases.") + help=("DEPRECATED use the nbserver_extensions dict instead") + ) + def _server_extensions_changed(self, name, old, new): + self.log.warning("server_extensions is deprecated, use nbserver_extensions") + self.server_extensions = new + + nbserver_extensions = Dict({}, config=True, + help=("Dict of Python modules to load as notebook server extensions." + "Entry values can be used to enable and disable the loading of" + "the extensions.") ) reraise_server_extension_failures = Bool( @@ -871,6 +879,7 @@ def init_webapp(self): if self.allow_origin_pat: self.tornado_settings['allow_origin_pat'] = re.compile(self.allow_origin_pat) self.tornado_settings['allow_credentials'] = self.allow_credentials + self.tornado_settings['cookie_options'] = self.cookie_options # ensure default_url starts with base_url if not self.default_url.startswith(self.base_url): self.default_url = url_path_join(self.base_url, self.default_url) @@ -910,7 +919,7 @@ def init_webapp(self): self.http_server.listen(port, self.ip) except socket.error as e: if e.errno == errno.EADDRINUSE: - self.log.info('The port %i is already in use, trying another random port.' % port) + self.log.info('The port %i is already in use, trying another port.' % port) continue elif e.errno in (errno.EACCES, getattr(errno, 'WSAEACCES', errno.EACCES)): self.log.warn("Permission to listen on port %i denied" % port) @@ -1023,18 +1032,34 @@ def init_server_extensions(self): The extension API is experimental, and may change in future releases. """ + + # TODO: Remove me in notebook 5.0 for modulename in self.server_extensions: - try: - mod = importlib.import_module(modulename) - func = getattr(mod, 'load_jupyter_server_extension', None) - if func is not None: - func(self) - except Exception: - if self.reraise_server_extension_failures: - raise - self.log.warn("Error loading server extension %s", modulename, - exc_info=True) - + # Don't override disable state of the extension if it already exist + # in the new traitlet + if not modulename in self.nbserver_extensions: + self.nbserver_extensions[modulename] = True + + for modulename in self.nbserver_extensions: + if self.nbserver_extensions[modulename]: + try: + mod = importlib.import_module(modulename) + func = getattr(mod, 'load_jupyter_server_extension', None) + if func is not None: + func(self) + except Exception: + if self.reraise_server_extension_failures: + raise + self.log.warning("Error loading server extension %s", modulename, + exc_info=True) + + def init_mime_overrides(self): + # On some Windows machines, an application has registered an incorrect + # mimetype for CSS in the registry. Tornado uses this when serving + # .css files, causing browsers to reject the stylesheet. We know the + # mimetype always needs to be text/css, so we override it here. + mimetypes.add_type('text/css', '.css') + @catch_config_error def initialize(self, argv=None): super(NotebookApp, self).initialize(argv) @@ -1047,6 +1072,7 @@ def initialize(self, argv=None): self.init_terminals() self.init_signal() self.init_server_extensions() + self.init_mime_overrides() def cleanup_kernels(self): """Shutdown all kernels. @@ -1119,7 +1145,8 @@ def start(self): relpath = os.path.relpath(self.file_to_run, self.notebook_dir) uri = url_escape(url_path_join('notebooks', *relpath.split(os.sep))) else: - uri = self.default_url + # default_url contains base_url, but so does connection_url + uri = self.default_url[len(self.base_url):] if browser: b = lambda : browser.open(url_path_join(self.connection_url, uri), new=2) diff --git a/notebook/serverextensions.py b/notebook/serverextensions.py new file mode 100644 index 0000000000..1cc08aa331 --- /dev/null +++ b/notebook/serverextensions.py @@ -0,0 +1,341 @@ +# coding: utf-8 +"""Utilities for installing server extensions for the notebook""" + +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +from __future__ import print_function + +import importlib +import sys + + +from jupyter_core.paths import jupyter_config_path +from ._version import __version__ +from .nbextensions import ( + JupyterApp, BaseNBExtensionApp, _get_config_dir, + GREEN_ENABLED, RED_DISABLED, + GREEN_OK, RED_X, +) + +from traitlets import Bool +from traitlets.utils.importstring import import_item +from traitlets.config.manager import BaseJSONConfigManager + + +# ------------------------------------------------------------------------------ +# Public API +# ------------------------------------------------------------------------------ +class ArgumentConflict(ValueError): + pass + + +def toggle_serverextension_python(import_name, enabled=None, parent=None, + user=True, sys_prefix=False, logger=None): + """Toggle a server extension. + + By default, toggles the extension in the system-wide Jupyter configuration + location (e.g. /usr/local/etc/jupyter). + + Parameters + ---------- + + import_name : str + Importable Python module (dotted-notation) exposing the magic-named + `load_jupyter_server_extension` function + enabled : bool [default: None] + Toggle state for the extension. Set to None to toggle, True to enable, + and False to disable the extension. + parent : Configurable [default: None] + user : bool [default: True] + Toggle in the user's configuration location (e.g. ~/.jupyter). + sys_prefix : bool [default: False] + Toggle in the current Python environment's configuration location + (e.g. ~/.envs/my-env/etc/jupyter). Will override `user`. + logger : Jupyter logger [optional] + Logger instance to use + """ + user = False if sys_prefix else user + config_dir = _get_config_dir(user=user, sys_prefix=sys_prefix) + cm = BaseJSONConfigManager(parent=parent, config_dir=config_dir) + cfg = cm.get("jupyter_notebook_config") + server_extensions = ( + cfg.setdefault("NotebookApp", {}) + .setdefault("nbserver_extensions", {}) + ) + + old_enabled = server_extensions.get(import_name, None) + new_enabled = enabled if enabled is not None else not old_enabled + + if logger: + if new_enabled: + logger.info(u"Enabling: %s" % (import_name)) + else: + logger.info(u"Disabling: %s" % (import_name)) + + server_extensions[import_name] = new_enabled + + if logger: + logger.info(u"- Writing config: {}".format(config_dir)) + + cm.update("jupyter_notebook_config", cfg) + + if new_enabled: + validate_serverextension(import_name, logger) + + +def validate_serverextension(import_name, logger=None): + """Assess the health of an installed server extension + + Returns a list of validation warnings. + + Parameters + ---------- + + import_name : str + Importable Python module (dotted-notation) exposing the magic-named + `load_jupyter_server_extension` function + logger : Jupyter logger [optional] + Logger instance to use + """ + + warnings = [] + infos = [] + + func = None + + if logger: + logger.info(" - Validating...") + + try: + mod = importlib.import_module(import_name) + func = getattr(mod, 'load_jupyter_server_extension', None) + except Exception: + logger.warning("Error loading server extension %s", import_name) + + import_msg = u" {} is {} importable?" + if func is not None: + infos.append(import_msg.format(GREEN_OK, import_name)) + else: + warnings.append(import_msg.format(RED_X, import_name)) + + post_mortem = u" {} {} {}" + if logger: + if warnings: + [logger.info(info) for info in infos] + [logger.warn(warning) for warning in warnings] + else: + logger.info(post_mortem.format(import_name, "", GREEN_OK)) + + return warnings + + +# ---------------------------------------------------------------------- +# Applications +# ---------------------------------------------------------------------- + +flags = {} +flags.update(JupyterApp.flags) +flags.pop("y", None) +flags.pop("generate-config", None) +flags.update({ + "user" : ({ + "ToggleServerExtensionApp" : { + "user" : True, + }}, "Perform the operation for the current user" + ), + "system" : ({ + "ToggleServerExtensionApp" : { + "user" : False, + "sys_prefix": False, + }}, "Perform the operation system-wide" + ), + "sys-prefix" : ({ + "ToggleServerExtensionApp" : { + "sys_prefix" : True, + }}, "Use sys.prefix as the prefix for installing server extensions" + ), + "py" : ({ + "ToggleServerExtensionApp" : { + "python" : True, + }}, "Install from a Python package" + ), +}) +flags['python'] = flags['py'] + + +class ToggleServerExtensionApp(BaseNBExtensionApp): + """A base class for enabling/disabling extensions""" + name = "jupyter serverextension enable/disable" + description = "Enable/disable a server extension using frontend configuration files." + + aliases = {} + flags = flags + + user = Bool(True, config=True, help="Whether to do a user install") + sys_prefix = Bool(False, config=True, help="Use the sys.prefix as the prefix") + python = Bool(False, config=True, help="Install from a Python package") + + def toggle_server_extension(self, import_name): + """Change the status of a named server extension. + + Uses the value of `self._toggle_value`. + + Parameters + --------- + + import_name : str + Importable Python module (dotted-notation) exposing the magic-named + `load_jupyter_server_extension` function + """ + toggle_serverextension_python( + import_name, self._toggle_value, parent=self, user=self.user, + sys_prefix=self.sys_prefix, logger=self.log) + + def toggle_server_extension_python(self, package): + """Change the status of some server extensions in a Python package. + + Uses the value of `self._toggle_value`. + + Parameters + --------- + + package : str + Importable Python module exposing the + magic-named `_jupyter_server_extension_paths` function + """ + m, server_exts = _get_server_extension_metadata(package) + for server_ext in server_exts: + module = server_ext['module'] + self.toggle_server_extension(module) + + def start(self): + """Perform the App's actions as configured""" + if not self.extra_args: + sys.exit('Please specify a server extension/package to enable or disable') + for arg in self.extra_args: + if self.python: + self.toggle_server_extension_python(arg) + else: + self.toggle_server_extension(arg) + + +class EnableServerExtensionApp(ToggleServerExtensionApp): + """An App that enables (and validates) Server Extensions""" + name = "jupyter serverextension enable" + description = """ + Enable a serverextension in configuration. + + Usage + jupyter serverextension enable [--system|--sys-prefix] + """ + _toggle_value = True + + +class DisableServerExtensionApp(ToggleServerExtensionApp): + """An App that disables Server Extensions""" + name = "jupyter serverextension disable" + description = """ + Disable a serverextension in configuration. + + Usage + jupyter serverextension disable [--system|--sys-prefix] + """ + _toggle_value = False + + +class ListServerExtensionsApp(BaseNBExtensionApp): + """An App that lists (and validates) Server Extensions""" + name = "jupyter serverextension list" + version = __version__ + description = "List all server extensions known by the configuration system" + + def list_server_extensions(self): + """List all enabled and disabled server extensions, by config path + + Enabled extensions are validated, potentially generating warnings. + """ + config_dirs = jupyter_config_path() + for config_dir in config_dirs: + cm = BaseJSONConfigManager(parent=self, config_dir=config_dir) + data = cm.get("jupyter_notebook_config") + server_extensions = ( + data.setdefault("NotebookApp", {}) + .setdefault("nbserver_extensions", {}) + ) + if server_extensions: + print(u'config dir: {}'.format(config_dir)) + for import_name, enabled in server_extensions.items(): + print(u' {} {}'.format( + import_name, + GREEN_ENABLED if enabled else RED_DISABLED)) + validate_serverextension(import_name, self.log) + + def start(self): + """Perform the App's actions as configured""" + self.list_server_extensions() + + +_examples = """ +jupyter serverextension list # list all configured server extensions +jupyter serverextension enable --py # enable all server extensions in a Python package +jupyter serverextension disable --py # disable all server extensions in a Python package +""" + + +class ServerExtensionApp(BaseNBExtensionApp): + """Root level server extension app""" + name = "jupyter serverextension" + version = __version__ + description = "Work with Jupyter server extensions" + examples = _examples + + subcommands = dict( + enable=(EnableServerExtensionApp, "Enable an server extension"), + disable=(DisableServerExtensionApp, "Disable an server extension"), + list=(ListServerExtensionsApp, "List server extensions") + ) + + def start(self): + """Perform the App's actions as configured""" + super(ServerExtensionApp, self).start() + + # The above should have called a subcommand and raised NoStart; if we + # get here, it didn't, so we should self.log.info a message. + subcmds = ", ".join(sorted(self.subcommands)) + sys.exit("Please supply at least one subcommand: %s" % subcmds) + + +main = ServerExtensionApp.launch_instance + +# ------------------------------------------------------------------------------ +# Private API +# ------------------------------------------------------------------------------ + + +def _get_server_extension_metadata(module): + """Load server extension metadata from a module. + + Returns a tuple of ( + the package as loaded + a list of server extension specs: [ + { + "module": "mockextension" + } + ] + ) + + Parameters + ---------- + + module : str + Importable Python module exposing the + magic-named `_jupyter_server_extension_paths` function + """ + m = import_item(module) + if not hasattr(m, '_jupyter_server_extension_paths'): + raise KeyError(u'The Python module {} does not include any valid server extensions'.format(module)) + return m, m._jupyter_server_extension_paths() + +if __name__ == '__main__': + main() diff --git a/notebook/services/config/manager.py b/notebook/services/config/manager.py index 2399f86054..9604be8e1e 100644 --- a/notebook/services/config/manager.py +++ b/notebook/services/config/manager.py @@ -5,13 +5,46 @@ import os.path -from traitlets.config.manager import BaseJSONConfigManager -from jupyter_core.paths import jupyter_config_dir -from traitlets import Unicode +from traitlets.config.manager import BaseJSONConfigManager, recursive_update +from jupyter_core.paths import jupyter_config_dir, jupyter_config_path +from traitlets import Unicode, Instance, List +from traitlets.config import LoggingConfigurable -class ConfigManager(BaseJSONConfigManager): + +class ConfigManager(LoggingConfigurable): """Config Manager used for storing notebook frontend config""" - - config_dir = Unicode(config=True) - def _config_dir_default(self): + + # Public API + + def get(self, section_name): + """Get the config from all config sections.""" + config = {} + for p in self.read_config_path: + cm = BaseJSONConfigManager(config_dir=p) + recursive_update(config, cm.get(section_name)) + return config + + def set(self, section_name, data): + """Set the config only to the user's config.""" + return self.write_config_manager.set(section_name, data) + + def update(self, section_name, new_data): + """Update the config only to the user's config.""" + return self.write_config_manager.update(section_name, new_data) + + # Private API + + read_config_path = List(Unicode()) + def _read_config_path_default(self): + return [os.path.join(p, 'nbconfig') for p in jupyter_config_path()] + + write_config_dir = Unicode() + def _write_config_dir_default(self): return os.path.join(jupyter_config_dir(), 'nbconfig') + + write_config_manager = Instance(BaseJSONConfigManager) + def _write_config_manager_default(self): + return BaseJSONConfigManager(config_dir=self.write_config_dir) + + def _write_config_dir_changed(self, name, old, new): + self.write_config_manager = BaseJSONConfigManager(config_dir=self.write_config_dir) diff --git a/notebook/services/kernels/handlers.py b/notebook/services/kernels/handlers.py index aa5b319791..28458dbb31 100644 --- a/notebook/services/kernels/handlers.py +++ b/notebook/services/kernels/handlers.py @@ -16,7 +16,7 @@ from ipython_genutils.py3compat import cast_unicode from notebook.utils import url_path_join, url_escape -from ...base.handlers import IPythonHandler, APIHandler, json_errors +from ...base.handlers import APIHandler, json_errors from ...base.zmqhandlers import AuthenticatedZMQStreamHandler, deserialize_binary_message from jupyter_client import protocol_version as client_protocol_version @@ -97,6 +97,11 @@ def post(self, kernel_id, action): class ZMQChannelsHandler(AuthenticatedZMQStreamHandler): + # class-level registry of open sessions + # allows checking for conflict on session-id, + # which is used as a zmq identity and must be unique. + _open_sessions = {} + @property def kernel_info_timeout(self): return self.settings.get('kernel_info_timeout', 10) @@ -194,6 +199,8 @@ def initialize(self): self.kernel_id = None self.kernel_info_channel = None self._kernel_info_future = Future() + self._close_future = Future() + self.session_key = '' # Rate limiting code self._iopub_window_msg_count = 0 @@ -209,6 +216,8 @@ def initialize(self): def pre_get(self): # authenticate first super(ZMQChannelsHandler, self).pre_get() + # check session collision: + yield self._register_session() # then request kernel info, waiting up to a certain time before giving up. # We don't want to wait forever, because browsers don't take it well when # servers never respond to websocket connection requests. @@ -232,6 +241,21 @@ def get(self, kernel_id): self.kernel_id = cast_unicode(kernel_id, 'ascii') yield super(ZMQChannelsHandler, self).get(kernel_id=kernel_id) + @gen.coroutine + def _register_session(self): + """Ensure we aren't creating a duplicate session. + + If a previous identical session is still open, close it to avoid collisions. + This is likely due to a client reconnecting from a lost network connection, + where the socket on our side has not been cleaned up yet. + """ + self.session_key = '%s:%s' % (self.kernel_id, self.session.session) + stale_handler = self._open_sessions.get(self.session_key) + if stale_handler: + self.log.warning("Replacing stale connection: %s", self.session_key) + yield stale_handler.close() + self._open_sessions[self.session_key] = self + def open(self, kernel_id): super(ZMQChannelsHandler, self).open() try: @@ -348,8 +372,15 @@ def write_stderr(error_message): return super(ZMQChannelsHandler, self)._on_zmq_reply(stream, msg) + def close(self): + super(ZMQChannelsHandler, self).close() + return self._close_future def on_close(self): + self.log.debug("Websocket closed %s", self.session_key) + # unregister myself as an open session (only if it's really me) + if self._open_sessions.get(self.session_key) is self: + self._open_sessions.pop(self.session_key) km = self.kernel_manager if self.kernel_id in km: km.remove_restart_callback( @@ -370,6 +401,7 @@ def on_close(self): socket.close() self.channels = {} + self._close_future.set_result(None) def _send_status_message(self, status): msg = self.session.msg("status", diff --git a/notebook/services/sessions/handlers.py b/notebook/services/sessions/handlers.py index b7a7f11de4..a382241686 100644 --- a/notebook/services/sessions/handlers.py +++ b/notebook/services/sessions/handlers.py @@ -42,10 +42,13 @@ def post(self): path = model['notebook']['path'] except KeyError: raise web.HTTPError(400, "Missing field in JSON data: notebook.path") - try: - kernel_name = model['kernel']['name'] - except KeyError: - self.log.debug("No kernel name specified, using default kernel") + + kernel = model.get('kernel', {}) + kernel_name = kernel.get('name') or None + kernel_id = kernel.get('id') or None + + if not kernel_id and not kernel_name: + self.log.debug("No kernel specified, using default kernel") kernel_name = None # Check to see if session exists @@ -55,7 +58,8 @@ def post(self): else: try: model = yield gen.maybe_future( - sm.create_session(path=path, kernel_name=kernel_name)) + sm.create_session(path=path, kernel_name=kernel_name, + kernel_id=kernel_id)) except NoSuchKernel: msg = ("The '%s' kernel is not available. Please pick another " "suitable kernel instead, or install that kernel." % kernel_name) @@ -86,19 +90,47 @@ def get(self, session_id): @json_errors @gen.coroutine def patch(self, session_id): - # Currently, this handler is strictly for renaming notebooks + """Patch updates sessions: + + - notebook.path updates session to track renamed notebooks + - kernel.name starts a new kernel with a given kernelspec + """ sm = self.session_manager + km = self.kernel_manager model = self.get_json_body() if model is None: raise web.HTTPError(400, "No JSON data provided") + + # get the previous session model + before = yield gen.maybe_future(sm.get_session(session_id=session_id)) + changes = {} if 'notebook' in model: notebook = model['notebook'] - if 'path' in notebook: + if notebook.get('path') is not None: changes['path'] = notebook['path'] + if 'kernel' in model: + # Kernel id takes precedence over name. + if model['kernel'].get('id') is not None: + kernel_id = model['kernel']['id'] + if kernel_id not in km: + raise web.HTTPError(400, "No such kernel: %s" % kernel_id) + changes['kernel_id'] = kernel_id + elif model['kernel'].get('name') is not None: + kernel_name = model['kernel']['name'] + kernel_id = yield sm.start_kernel_for_session( + session_id, kernel_name=kernel_name, path=before['notebook']['path']) + changes['kernel_id'] = kernel_id yield gen.maybe_future(sm.update_session(session_id, **changes)) model = yield gen.maybe_future(sm.get_session(session_id=session_id)) + + if model['kernel']['id'] != before['kernel']['id']: + # kernel_id changed because we got a new kernel + # shutdown the old one + yield gen.maybe_future( + km.shutdown_kernel(before['kernel']['id']) + ) self.finish(json.dumps(model, default=date_default)) @web.authenticated diff --git a/notebook/services/sessions/sessionmanager.py b/notebook/services/sessions/sessionmanager.py index 1259404083..f0554bf740 100644 --- a/notebook/services/sessions/sessionmanager.py +++ b/notebook/services/sessions/sessionmanager.py @@ -4,7 +4,12 @@ # Distributed under the terms of the Modified BSD License. import uuid -import sqlite3 + +try: + import sqlite3 +except ImportError: + # fallback on pysqlite2 if Python was build without sqlite + from pysqlite2 import dbapi2 as sqlite3 from tornado import gen, web @@ -62,22 +67,33 @@ def session_exists(self, path): def new_session_id(self): "Create a uuid for a new session" return unicode_type(uuid.uuid4()) - + @gen.coroutine - def create_session(self, path=None, kernel_name=None): + def create_session(self, path=None, kernel_name=None, kernel_id=None): """Creates a session and returns its model""" session_id = self.new_session_id() - # allow nbm to specify kernels cwd - kernel_path = self.contents_manager.get_kernel_path(path=path) - kernel_id = yield gen.maybe_future( - self.kernel_manager.start_kernel(path=kernel_path, kernel_name=kernel_name) - ) + if kernel_id is not None and kernel_id in self.kernel_manager: + pass + else: + kernel_id = yield self.start_kernel_for_session(session_id, path, + kernel_name) result = yield gen.maybe_future( self.save_session(session_id, path=path, kernel_id=kernel_id) ) # py2-compat raise gen.Return(result) - + + @gen.coroutine + def start_kernel_for_session(self, session_id, path, kernel_name): + """Start a new kernel for a given session.""" + # allow contents manager to specify kernels cwd + kernel_path = self.contents_manager.get_kernel_path(path=path) + kernel_id = yield gen.maybe_future( + self.kernel_manager.start_kernel(path=kernel_path, kernel_name=kernel_name) + ) + # py2-compat + raise gen.Return(kernel_id) + def save_session(self, session_id, path=None, kernel_id=None): """Saves the items for the session with the given session_id @@ -211,9 +227,9 @@ def list_sessions(self): pass return result + @gen.coroutine def delete_session(self, session_id): """Deletes the row in the session database with given session_id""" - # Check that session exists before deleting session = self.get_session(session_id=session_id) - self.kernel_manager.shutdown_kernel(session['kernel']['id']) + yield gen.maybe_future(self.kernel_manager.shutdown_kernel(session['kernel']['id'])) self.cursor.execute("DELETE FROM session WHERE session_id=?", (session_id,)) diff --git a/notebook/services/sessions/tests/test_sessionmanager.py b/notebook/services/sessions/tests/test_sessionmanager.py index 0b31d97a4e..14bda028b5 100644 --- a/notebook/services/sessions/tests/test_sessionmanager.py +++ b/notebook/services/sessions/tests/test_sessionmanager.py @@ -175,6 +175,8 @@ def test_bad_delete_session(self): # try to delete a session that doesn't exist ~ raise error sm = self.sm self.create_session(path='/path/to/test.ipynb', kernel_name='python') - self.assertRaises(TypeError, sm.delete_session, bad_kwarg='23424') # Bad keyword - self.assertRaises(web.HTTPError, sm.delete_session, session_id='23424') # nonexistant + with self.assertRaises(TypeError): + self.loop.run_sync(lambda : sm.delete_session(bad_kwarg='23424')) # Bad keyword + with self.assertRaises(web.HTTPError): + self.loop.run_sync(lambda : sm.delete_session(session_id='23424')) # nonexistent diff --git a/notebook/services/sessions/tests/test_sessions_api.py b/notebook/services/sessions/tests/test_sessions_api.py index 1a2dc80030..08cb381029 100644 --- a/notebook/services/sessions/tests/test_sessions_api.py +++ b/notebook/services/sessions/tests/test_sessions_api.py @@ -39,15 +39,25 @@ def list(self): def get(self, id): return self._req('GET', id) - def create(self, path, kernel_name='python'): + def create(self, path, kernel_name='python', kernel_id=None): body = json.dumps({'notebook': {'path':path}, - 'kernel': {'name': kernel_name}}) + 'kernel': {'name': kernel_name, + 'id': kernel_id}}) return self._req('POST', '', body) - def modify(self, id, path): + def modify_path(self, id, path): body = json.dumps({'notebook': {'path':path}}) return self._req('PATCH', id, body) + def modify_kernel_name(self, id, kernel_name): + body = json.dumps({'kernel': {'name': kernel_name}}) + return self._req('PATCH', id, body) + + def modify_kernel_id(self, id, kernel_id): + # Also send a dummy name to show that id takes precedence. + body = json.dumps({'kernel': {'id': kernel_id, 'name': 'foo'}}) + return self._req('PATCH', id, body) + def delete(self, id): return self._req('DELETE', id) @@ -102,6 +112,28 @@ def test_create(self): got = self.sess_api.get(sid).json() self.assertEqual(got, newsession) + def test_create_with_kernel_id(self): + # create a new kernel + r = requests.post(url_path_join(self.base_url(), 'api/kernels')) + r.raise_for_status() + kernel = r.json() + + resp = self.sess_api.create('foo/nb1.ipynb', kernel_id=kernel['id']) + self.assertEqual(resp.status_code, 201) + newsession = resp.json() + self.assertIn('id', newsession) + self.assertEqual(newsession['notebook']['path'], 'foo/nb1.ipynb') + self.assertEqual(newsession['kernel']['id'], kernel['id']) + self.assertEqual(resp.headers['Location'], self.url_prefix + 'api/sessions/{0}'.format(newsession['id'])) + + sessions = self.sess_api.list().json() + self.assertEqual(sessions, [newsession]) + + # Retrieve it + sid = newsession['id'] + got = self.sess_api.get(sid).json() + self.assertEqual(got, newsession) + def test_delete(self): newsession = self.sess_api.create('foo/nb1.ipynb').json() sid = newsession['id'] @@ -115,10 +147,47 @@ def test_delete(self): with assert_http_error(404): self.sess_api.get(sid) - def test_modify(self): + def test_modify_path(self): newsession = self.sess_api.create('foo/nb1.ipynb').json() sid = newsession['id'] - changed = self.sess_api.modify(sid, 'nb2.ipynb').json() + changed = self.sess_api.modify_path(sid, 'nb2.ipynb').json() self.assertEqual(changed['id'], sid) self.assertEqual(changed['notebook']['path'], 'nb2.ipynb') + + def test_modify_kernel_name(self): + before = self.sess_api.create('foo/nb1.ipynb').json() + sid = before['id'] + + after = self.sess_api.modify_kernel_name(sid, before['kernel']['name']).json() + self.assertEqual(after['id'], sid) + self.assertEqual(after['notebook'], before['notebook']) + self.assertNotEqual(after['kernel']['id'], before['kernel']['id']) + + # check kernel list, to be sure previous kernel was cleaned up + r = requests.get(url_path_join(self.base_url(), 'api/kernels')) + r.raise_for_status() + kernel_list = r.json() + self.assertEqual(kernel_list, [after['kernel']]) + + def test_modify_kernel_id(self): + before = self.sess_api.create('foo/nb1.ipynb').json() + sid = before['id'] + + # create a new kernel + r = requests.post(url_path_join(self.base_url(), 'api/kernels')) + r.raise_for_status() + kernel = r.json() + + # Attach our session to the existing kernel + after = self.sess_api.modify_kernel_id(sid, kernel['id']).json() + self.assertEqual(after['id'], sid) + self.assertEqual(after['notebook'], before['notebook']) + self.assertNotEqual(after['kernel']['id'], before['kernel']['id']) + self.assertEqual(after['kernel']['id'], kernel['id']) + + # check kernel list, to be sure previous kernel was cleaned up + r = requests.get(url_path_join(self.base_url(), 'api/kernels')) + r.raise_for_status() + kernel_list = r.json() + self.assertEqual(kernel_list, [kernel]) diff --git a/notebook/static/base/js/namespace.js b/notebook/static/base/js/namespace.js index e6c87b31d2..ee2f4f8284 100644 --- a/notebook/static/base/js/namespace.js +++ b/notebook/static/base/js/namespace.js @@ -73,7 +73,7 @@ define(function(){ // tree jglobal('SessionList','tree/js/sessionlist'); - Jupyter.version = "4.2.0.dev"; + Jupyter.version = "4.2.1"; Jupyter._target = '_blank'; return Jupyter; }); diff --git a/notebook/static/base/js/utils.js b/notebook/static/base/js/utils.js index 8775a8f794..884c209214 100644 --- a/notebook/static/base/js/utils.js +++ b/notebook/static/base/js/utils.js @@ -13,6 +13,16 @@ define([ // keep track of which extensions have been loaded already var extensions_loaded = []; + /** + * Whether or not an extension has been loaded + * @param {string} extension - name of the extension + * @return {boolean} true if loaded already + */ + var is_loaded = function(extension) { + var ext_path = "nbextensions/" + extension; + return extensions_loaded.indexOf(ext_path) >= 0; + }; + /** * Load a single extension. * @param {string} extension - extension path. @@ -21,17 +31,17 @@ define([ var load_extension = function (extension) { return new Promise(function(resolve, reject) { var ext_path = "nbextensions/" + extension; - require([ext_path], function(module) { - try { - if (extensions_loaded.indexOf(ext_path) < 0) { - console.log("Loading extension: " + extension); - module.load_ipython_extension(); - extensions_loaded.push(ext_path); - } - else{ - console.log("Loaded extension already: " + extension); + requirejs([ext_path], function(module) { + if (!is_loaded(extension)) { + console.log("Loading extension: " + extension); + if (module.load_ipython_extension) { + Promise.resolve(module.load_ipython_extension()).then(function() { + resolve(module); + }).catch(reject); } - } finally { + extensions_loaded.push(ext_path); + } else { + console.log("Loaded extension already: " + extension); resolve(module); } }, function(err) { @@ -46,23 +56,38 @@ define([ * @return {Promise} that resolves to a list of loaded module handles. */ var load_extensions = function () { + console.log("load_extensions", arguments); return Promise.all(Array.prototype.map.call(arguments, load_extension)).catch(function(err) { console.error("Failed to load extension" + (err.requireModules.length>1?'s':'') + ":", err.requireModules, err); }); }; + /** + * Return a list of extensions that should be active + * The config for nbextensions comes in as a dict where keys are + * nbextensions paths and the values are a bool indicating if it + * should be active. This returns a list of nbextension paths + * where the value is true + */ + function filter_extensions(nbext_config) { + var active = []; + Object.keys(nbext_config).forEach(function (nbext) { + if (nbext_config[nbext]) {active.push(nbext);} + }); + return active; + } + /** * Wait for a config section to load, and then load the extensions specified * in a 'load_extensions' key inside it. */ function load_extensions_from_config(section) { - section.loaded.then(function() { + return section.loaded.then(function() { if (section.data.load_extensions) { - var nbextension_paths = Object.getOwnPropertyNames( - section.data.load_extensions); - load_extensions.apply(this, nbextension_paths); + var active = filter_extensions(section.data.load_extensions); + return load_extensions.apply(this, active); } - }); + }).catch(utils.reject('Could not load nbextensions from ' + section.section_name + ' config file')); } //============================================================================ @@ -826,8 +851,10 @@ define([ }; var utils = { + is_loaded: is_loaded, load_extension: load_extension, load_extensions: load_extensions, + filter_extensions: filter_extensions, load_extensions_from_config: load_extensions_from_config, regex_split : regex_split, uuid : uuid, diff --git a/notebook/static/edit/js/menubar.js b/notebook/static/edit/js/menubar.js index 73952d4db5..0509016b88 100644 --- a/notebook/static/edit/js/menubar.js +++ b/notebook/static/edit/js/menubar.js @@ -114,6 +114,11 @@ define([ }); // View + + this.element.find('#toggle_header').click(function (){ + $("#header-container").toggle(); + }); + this.element.find('#menu-line-numbers').click(function () { var current = editor.codemirror.getOption('lineNumbers'); var value = Boolean(1-current); diff --git a/notebook/static/notebook/js/about.js b/notebook/static/notebook/js/about.js index 50ef7ad3d1..b28285f292 100644 --- a/notebook/static/notebook/js/about.js +++ b/notebook/static/notebook/js/about.js @@ -9,19 +9,27 @@ require([ 'use strict'; $('#notebook_about').click(function () { // use underscore template to auto html escape - var text = 'You are using Jupyter notebook.

'; - text = text + 'The version of the notebook server is '; - text = text + _.template('<%- version %>')({ version: sys_info.notebook_version }); - if (sys_info.commit_hash) { - text = text + _.template('-<%- hash %>')({ hash: sys_info.commit_hash }); + if (sys_info) { + var text = 'You are using Jupyter notebook.

'; + text = text + 'The version of the notebook server is '; + text = text + _.template('<%- version %>')({ version: sys_info.notebook_version }); + if (sys_info.commit_hash) { + text = text + _.template('-<%- hash %>')({ hash: sys_info.commit_hash }); + } + text = text + _.template(' and is running on:
Python <%- pyver %>
')({ + pyver: sys_info.sys_version }); + var kinfo = $('
').attr('id', '#about-kinfo').text('Waiting for kernel to be available...'); + var body = $('
'); + body.append($('

').text('Server Information:')); + body.append($('

').html(text)); + body.append($('

').text('Current Kernel Information:')); + body.append(kinfo); + } else { + var text = 'Could not access sys_info variable for version information.'; + var body = $('
'); + body.append($('

').text('Cannot find sys_info!')); + body.append($('

').html(text)); } - text = text + _.template(' and is running on:

Python <%- pyver %>
')({ pyver: sys_info.sys_version }); - var kinfo = $('
').attr('id', '#about-kinfo').text('Waiting for kernel to be available...'); - var body = $('
'); - body.append($('

').text('Server Information:')); - body.append($('

').html(text)); - body.append($('

').text('Current Kernel Information:')); - body.append(kinfo); dialog.modal({ title: 'About Jupyter Notebook', body: body, diff --git a/notebook/static/notebook/js/codecell.js b/notebook/static/notebook/js/codecell.js index 59807b0d8b..df0958dfb6 100644 --- a/notebook/static/notebook/js/codecell.js +++ b/notebook/static/notebook/js/codecell.js @@ -305,24 +305,21 @@ define([ * @method execute */ CodeCell.prototype.execute = function (stop_on_error) { - if (!this.kernel || !this.kernel.is_connected()) { - console.log("Can't execute, kernel is not connected."); + if (!this.kernel) { + console.log("Can't execute cell since kernel is not set."); return; } - this.output_area.clear_output(false, true); - if (stop_on_error === undefined) { stop_on_error = true; } + this.output_area.clear_output(false, true); var old_msg_id = this.last_msg_id; - if (old_msg_id) { this.kernel.clear_callbacks_for_msg(old_msg_id); - if (old_msg_id) { - delete CodeCell.msg_cells[old_msg_id]; - } + delete CodeCell.msg_cells[old_msg_id]; + this.last_msg_id = null; } if (this.get_text().trim().length === 0) { // nothing to do diff --git a/notebook/static/notebook/js/main.js b/notebook/static/notebook/js/main.js index 4f5cbce6bf..8bff36788a 100644 --- a/notebook/static/notebook/js/main.js +++ b/notebook/static/notebook/js/main.js @@ -54,12 +54,6 @@ require([ ) { "use strict"; - // BEGIN HARDCODED WIDGETS HACK - utils.load_extension('widgets/notebook/js/extension').catch(function () { - console.warn('ipywidgets package not installed. Widgets are not available.'); - }); - // END HARDCODED WIDGETS HACK - // compat with old IPython, remove for IPython > 3.0 window.CodeMirror = CodeMirror; @@ -177,8 +171,28 @@ require([ configurable: false }); - utils.load_extensions_from_config(config_section); - utils.load_extensions_from_config(common_config); + // Now actually load nbextensions from config + Promise.all([ + utils.load_extensions_from_config(config_section), + utils.load_extensions_from_config(common_config), + ]) + .catch(function(error) { + console.error('Could not load nbextensions from user config files', error); + }) + // BEGIN HARDCODED WIDGETS HACK + .then(function() { + if (!utils.is_loaded('jupyter-js-widgets/extension')) { + // Fallback to the ipywidgets extension + utils.load_extension('widgets/notebook/js/extension').catch(function () { + console.warn('Widgets are not available. Please install widgetsnbextension or ipywidgets 4.0'); + }); + } + }) + .catch(function(error) { + console.error('Could not load ipywidgets', error); + }); + // END HARDCODED WIDGETS HACK + notebook.load_notebook(common_options.notebook_path); }); diff --git a/notebook/static/notebook/js/notebook.js b/notebook/static/notebook/js/notebook.js index 4127df67bb..ab420074cb 100644 --- a/notebook/static/notebook/js/notebook.js +++ b/notebook/static/notebook/js/notebook.js @@ -67,6 +67,8 @@ define(function (require) { this.last_modified = null; // debug 484 this._last_modified = 'init'; + // Firefox workaround + this._ff_beforeunload_fired = false; // Create default scroll manager. this.scroll_manager = new scrollmanager.ScrollManager(this); @@ -111,7 +113,8 @@ define(function (require) { } }, function (err) { console.log("No CodeMirror mode: " + lang); - callback(err, code); + console.log("Require CodeMirror mode error: " + err); + callback(null, code); }); } }); @@ -314,6 +317,17 @@ define(function (require) { if (kill_kernel) { that.session.delete(); } + if ( utils.browser[0] === "Firefox") { + // Workaround ancient Firefox bug showing beforeunload twice: https://bugzilla.mozilla.org/show_bug.cgi?id=531199 + if (that._ff_beforeunload_fired) { + return; // don't show twice on FF + } + that._ff_beforeunload_fired = true; + // unset flag immediately after dialog is dismissed + setTimeout(function () { + that._ff_beforeunload_fired = false; + }, 1); + } // if we are autosaving, trigger an autosave on nav-away. // still warn, because if we don't the autosave may fail. if (that.dirty) { diff --git a/notebook/static/notebook/js/notificationarea.js b/notebook/static/notebook/js/notificationarea.js index f331f93778..36e2e55ad6 100644 --- a/notebook/static/notebook/js/notificationarea.js +++ b/notebook/static/notebook/js/notificationarea.js @@ -139,8 +139,7 @@ define([ if (info.attempt === 1) { var msg = "A connection to the notebook server could not be established." + - " The notebook will continue trying to reconnect, but" + - " until it does, you will NOT be able to run code. Check your" + + " The notebook will continue trying to reconnect. Check your" + " network connection or notebook server configuration."; dialog.kernel_modal({ diff --git a/notebook/static/notebook/js/pager.js b/notebook/static/notebook/js/pager.js index 260f7b4719..cb9bf3207c 100644 --- a/notebook/static/notebook/js/pager.js +++ b/notebook/static/notebook/js/pager.js @@ -161,6 +161,14 @@ define([ this.pager_element.find(".container").append($('
').html(utils.fixCarriageReturn(utils.fixConsole(text))));
     };
 
+    Pager.prototype.append = function (htm) {
+        /**
+         * The only user content injected with this HTML call is escaped by
+         * the fixConsole() method.
+         */
+        this.pager_element.find(".container").append(htm);
+    };
+
 
     Pager.prototype._resize = function() {
         /**
diff --git a/notebook/static/services/kernels/kernel.js b/notebook/static/services/kernels/kernel.js
index 91c975b809..8600a0c34e 100644
--- a/notebook/static/services/kernels/kernel.js
+++ b/notebook/static/services/kernels/kernel.js
@@ -63,6 +63,8 @@ define([
         this._autorestart_attempt = 0;
         this._reconnect_attempt = 0;
         this.reconnect_limit = 7;
+
+        this._pending_messages = [];
     };
 
     /**
@@ -337,7 +339,7 @@ define([
          * @function reconnect
          */
         if (this.is_connected()) {
-            return;
+            this.stop_channels();
         }
         this._reconnect_attempt = this._reconnect_attempt + 1;
         this.events.trigger('kernel_reconnecting.Kernel', {
@@ -409,6 +411,15 @@ define([
          * @function _kernel_connected
          */
         this.events.trigger('kernel_connected.Kernel', {kernel: this});
+
+        // Send pending messages. We shift the message off the queue
+        // after the message is sent so that if there is an exception,
+        // the message is still pending.
+        while (this._pending_messages.length > 0) {
+          this.ws.send(this._pending_messages[0]);
+          this._pending_messages.shift();
+        }
+
         // get kernel info so we know what state the kernel is in
         var that = this;
         this.kernel_info(function (reply) {
@@ -524,8 +535,13 @@ define([
 
         this.events.trigger('kernel_disconnected.Kernel', {kernel: this});
         if (error) {
-            console.log('WebSocket connection failed: ', ws_url);
-            this.events.trigger('kernel_connection_failed.Kernel', {kernel: this, ws_url: ws_url, attempt: this._reconnect_attempt});
+            console.log('WebSocket connection failed: ', ws_url, error);
+            this.events.trigger('kernel_connection_failed.Kernel', {
+                kernel: this,
+                ws_url: ws_url,
+                attempt: this._reconnect_attempt,
+                error: error,
+            });
         }
         this._schedule_reconnect();
     };
@@ -602,19 +618,34 @@ define([
         return (this.ws === null);
     };
     
+    Kernel.prototype._send = function(msg) {
+      /**
+       * Send a message (if the kernel is connected) or queue the message for future delivery
+       *
+       * Pending messages will automatically be sent when a kernel becomes connected.
+       *
+       * @function _send
+       * @param msg
+       */
+      if (this.is_connected()) {
+            this.ws.send(msg);
+        } else {
+            this._pending_messages.push(msg);
+        }
+    }
+
     Kernel.prototype.send_shell_message = function (msg_type, content, callbacks, metadata, buffers) {
         /**
          * Send a message on the Kernel's shell channel
          *
+         * If the kernel is not connected, the message will be buffered.
+         *
          * @function send_shell_message
          */
-        if (!this.is_connected()) {
-            throw new Error("kernel is not connected");
-        }
         var msg = this._get_msg(msg_type, content, metadata, buffers);
         msg.channel = 'shell';
-        this.ws.send(serialize.serialize(msg));
         this.set_callbacks_for_msg(msg.header.msg_id, callbacks);
+        this._send(serialize.serialize(msg));
         return msg.header.msg_id;
     };
 
@@ -777,16 +808,13 @@ define([
      * @function send_input_reply
      */
     Kernel.prototype.send_input_reply = function (input) {
-        if (!this.is_connected()) {
-            throw new Error("kernel is not connected");
-        }
         var content = {
             value : input
         };
         this.events.trigger('input_reply.Kernel', {kernel: this, content: content});
         var msg = this._get_msg("input_reply", content);
         msg.channel = 'stdin';
-        this.ws.send(serialize.serialize(msg));
+        this._send(serialize.serialize(msg));
         return msg.header.msg_id;
     };
 
diff --git a/notebook/static/terminal/js/main.js b/notebook/static/terminal/js/main.js
index 84ec2f5995..23210c055e 100644
--- a/notebook/static/terminal/js/main.js
+++ b/notebook/static/terminal/js/main.js
@@ -20,10 +20,12 @@ require([
     "use strict";
     page = new page.Page();
 
+    var config = new configmod.ConfigSection('terminal',
+                                     {base_url: utils.get_body_data('baseUrl')});
+    config.load();
     var common_config = new configmod.ConfigSection('common', 
                                     {base_url: utils.get_body_data('baseUrl')});
     common_config.load();
-
     // Test size: 25x80
     var termRowHeight = function(){ return 1.00 * $("#dummy-screen")[0].offsetHeight / 25;};
         // 1.02 here arrived at by trial and error to make the spacing look right
@@ -34,7 +36,7 @@ require([
     var ws_url = location.protocol.replace('http', 'ws') + "//" + location.host
                                     + base_url + ws_path;
     
-    var header = $("#header")[0]
+    var header = $("#header")[0];
     function calculate_size() {
         var height = $(window).height() - header.offsetHeight;
         var width = $('#terminado-container').width();
@@ -51,6 +53,7 @@ require([
     
     page.show_site();
     
+    utils.load_extensions_from_config(config);
     utils.load_extensions_from_config(common_config);
     
     window.onresize = function() { 
diff --git a/notebook/static/terminal/js/terminado.js b/notebook/static/terminal/js/terminado.js
index 5192f713ff..61dda1a004 100644
--- a/notebook/static/terminal/js/terminado.js
+++ b/notebook/static/terminal/js/terminado.js
@@ -1,4 +1,6 @@
-define ([], function() {
+define ([
+    'termjs',
+], function(Terminal) {
     "use strict";
     function make_terminal(element, size, ws_url) {
         var ws = new WebSocket(ws_url);
diff --git a/notebook/static/tree/js/notebooklist.js b/notebook/static/tree/js/notebooklist.js
index 4cf108d01e..ba5ca31f43 100644
--- a/notebook/static/tree/js/notebooklist.js
+++ b/notebook/static/tree/js/notebooklist.js
@@ -555,6 +555,7 @@ define([
         var uri_prefix = NotebookList.uri_prefixes[model.type];
         if (model.type === 'file' &&
             model.mimetype && model.mimetype.substr(0,5) !== 'text/'
+            && !model.mimetype.endsWith('javascript')
         ) {
             // send text/unidentified files to editor, others go to raw viewer
             uri_prefix = 'files';
diff --git a/notebook/templates/edit.html b/notebook/templates/edit.html
index f02a927128..b83988d5f0 100644
--- a/notebook/templates/edit.html
+++ b/notebook/templates/edit.html
@@ -65,7 +65,9 @@
             
             
  • View
               
             
    • 
             
  • Language
diff --git a/notebook/templates/notebook.html b/notebook/templates/notebook.html
index 197bd9ccd1..879021291b 100644
--- a/notebook/templates/notebook.html
+++ b/notebook/templates/notebook.html
@@ -105,7 +105,7 @@
                         
    • 
                         
  • Download as
                             
      
-                                
    • IPython Notebook (.ipynb)
      • 
+                                
    • Notebook (.ipynb)
      • 
                                 
    • Script
      • 
                                 
    • HTML (.html)
      • 
                                 
    • Markdown (.md)
      • 
@@ -148,7 +148,7 @@
                 
    • View
                     
                 
      • 
diff --git a/notebook/templates/page.html b/notebook/templates/page.html
index f9cd56a976..3ee4ef4fea 100644
--- a/notebook/templates/page.html
+++ b/notebook/templates/page.html
@@ -4,7 +4,7 @@
 
     
 
-    {% block title %}IPython Notebook{% endblock %}
+    {% block title %}Jupyter Notebook{% endblock %}
     {% block favicon %}{% endblock %}
     
     
@@ -27,7 +27,6 @@
             'auth/js/main': 'auth/js/main.min',
             custom : '{{ base_url }}custom',
             nbextensions : '{{ base_url }}nbextensions',
-            widgets : '{{ base_url }}deprecatedwidgets',
             kernelspecs : '{{ base_url }}kernelspecs',
             underscore : 'components/underscore/underscore-min',
             backbone : 'components/backbone/backbone-min',
@@ -37,7 +36,7 @@
             'jquery-ui': 'components/jquery-ui/ui/minified/jquery-ui.min',
             moment: 'components/moment/moment',
             codemirror: 'components/codemirror',
-            termjs: 'components/term.js/src/term',
+            termjs: 'components/xterm.js/src/xterm',
             typeahead: 'components/jquery-typeahead/dist/jquery.typeahead'
           },
 	  map: { // for backward compatibility
@@ -80,6 +79,22 @@
               }
           }
       });
+
+      define("bootstrap", function () {
+          return window.$;
+      });
+
+      define("jquery", function () {
+          return window.$;
+      });
+
+      define("jqueryui", function () {
+          return window.$;
+      });
+
+      define("jquery-ui", function () {
+          return window.$;
+      });
     
 
     {% block meta %}
@@ -91,7 +106,7 @@
 
 
diff --git a/notebook/templates/terminal.html b/notebook/templates/terminal.html
index 230b78118c..fd4f7db811 100644
--- a/notebook/templates/terminal.html
+++ b/notebook/templates/terminal.html
@@ -15,6 +15,7 @@
 {{super()}}
 
 
+
 {% endblock %}
 
 {% block site %}
diff --git a/notebook/tests/mockextension/index.js b/notebook/tests/mockextension/index.js
new file mode 100644
index 0000000000..0475609d40
--- /dev/null
+++ b/notebook/tests/mockextension/index.js
@@ -0,0 +1 @@
+console.log('z');
diff --git a/notebook/tests/notebook/buffering.js b/notebook/tests/notebook/buffering.js
new file mode 100644
index 0000000000..34f3224886
--- /dev/null
+++ b/notebook/tests/notebook/buffering.js
@@ -0,0 +1,56 @@
+//
+// Test buffering for execution requests.
+//
+casper.notebook_test(function () {
+    this.then(function() {
+      // make sure there are at least three cells for the tests below.
+      this.append_cell();
+      this.append_cell();
+      this.append_cell();
+    })
+
+    this.thenEvaluate(function () {
+        IPython.notebook.kernel.stop_channels();
+        var cell = IPython.notebook.get_cell(0);
+        cell.set_text('a=10; print(a)');
+        IPython.notebook.execute_cells([0]);
+        IPython.notebook.kernel.reconnect(1);
+    });
+
+    this.wait_for_output(0);
+
+    this.then(function () {
+        var result = this.get_output_cell(0);
+        this.test.assertEquals(result.text, '10\n', 'kernels buffer messages if connection is down');
+    });
+
+    this.thenEvaluate(function () {
+        var cell = IPython.notebook.get_cell(0);
+        var cellplus = IPython.notebook.get_cell(1);
+        var cellprint = IPython.notebook.get_cell(2);
+        cell.set_text('k=1');
+        cellplus.set_text('k+=1');
+        cellprint.set_text('k*=2')
+
+        IPython.notebook.kernel.stop_channels();
+
+        // Repeated execution of cell queued up in the kernel executes
+        // each execution request in order.
+        IPython.notebook.execute_cells([0]);
+        IPython.notebook.execute_cells([2]);
+        IPython.notebook.execute_cells([1]);
+        IPython.notebook.execute_cells([1]);
+        IPython.notebook.execute_cells([1]);
+        cellprint.set_text('print(k)')
+        IPython.notebook.execute_cells([2]);
+
+        IPython.notebook.kernel.reconnect(1);
+    });
+
+    this.wait_for_output(2);
+
+    this.then(function () {
+        var result = this.get_output_cell(2);
+        this.test.assertEquals(result.text, '5\n', 'kernels send buffered messages in order');
+    });
+});
diff --git a/notebook/tests/notebook/markdown.js b/notebook/tests/notebook/markdown.js
index fdd1ee9ee9..ae769ea5cf 100644
--- a/notebook/tests/notebook/markdown.js
+++ b/notebook/tests/notebook/markdown.js
@@ -60,4 +60,46 @@ casper.notebook_test(function () {
         var hashes = new Array(level + 1).join('#');
         this.test.assertEquals(level_text, hashes + ' ' + text, 'markdown set_heading_level ' + level);
     }
+
+    // Test markdown code blocks
+
+
+    function md_render_test (codeblock, result, message) {
+        // make a cell and trigger render
+        casper.thenEvaluate(function (text) {
+            var cell = Jupyter.notebook.insert_cell_at_bottom('markdown');
+            cell.set_text(text);
+            // signal window._rendered when cell render completes
+            window._rendered = null;
+            cell.events.one("rendered.MarkdownCell", function (event, data) {
+                window._rendered = data.cell.get_rendered();
+            });
+            cell.render();
+        }, {text: codeblock});
+        // wait for render to complete
+        casper.waitFor(function () {
+            return casper.evaluate(function () {
+                return window._rendered;
+            });
+        });
+        // test after waiting
+        casper.then(function () {
+            // get rendered result
+            var output = casper.evaluate(function () {
+                var rendered = window._rendered;
+                delete window._rendered;
+                return rendered;
+            });
+            // perform test
+            this.test.assertEquals(output.trim(), result, message);
+        });
+    };
+
+    var codeblock = '```\nx = 1\n```'
+    var result = '
      x = 1\n
      '
+    md_render_test(codeblock, result, 'Markdown code block no language');
+
+    codeblock = '```aaaa\nx = 1\n```'
+    result = '
      x = 1\n
      '
+    md_render_test(codeblock, result, 'Markdown code block unknown language');
 });
diff --git a/notebook/tests/test_nbextensions.py b/notebook/tests/test_nbextensions.py
index 7feea2d8c9..86fb25a071 100644
--- a/notebook/tests/test_nbextensions.py
+++ b/notebook/tests/test_nbextensions.py
@@ -6,12 +6,12 @@
 
 import glob
 import os
-import re
 import sys
 import tarfile
 import zipfile
 from io import BytesIO, StringIO
 from os.path import basename, join as pjoin
+from traitlets.tests.utils import check_help_all_output
 from unittest import TestCase
 
 try:
@@ -23,7 +23,14 @@
 from ipython_genutils import py3compat
 from ipython_genutils.tempdir import TemporaryDirectory
 from notebook import nbextensions
-from notebook.nbextensions import install_nbextension, check_nbextension
+from notebook.nbextensions import (install_nbextension, check_nbextension,
+    enable_nbextension, disable_nbextension,
+    install_nbextension_python, uninstall_nbextension_python,
+    enable_nbextension_python, disable_nbextension_python, _get_config_dir,
+    validate_nbextension, validate_nbextension_python
+)
+
+from traitlets.config.manager import BaseJSONConfigManager
 
 
 def touch(file, mtime=None):
@@ -38,6 +45,15 @@ def touch(file, mtime=None):
         os.utime(file, (atime, mtime))
     return os.stat(file).st_mtime
 
+
+def test_help_output():
+    check_help_all_output('notebook.nbextensions')
+    check_help_all_output('notebook.nbextensions', ['enable'])
+    check_help_all_output('notebook.nbextensions', ['disable'])
+    check_help_all_output('notebook.nbextensions', ['install'])
+    check_help_all_output('notebook.nbextensions', ['uninstall'])
+
+
 class TestInstallNBExtension(TestCase):
     
     def tempdir(self):
@@ -47,7 +63,7 @@ def tempdir(self):
 
     def setUp(self):
         self.tempdirs = []
-        src = self.src = self.tempdir()
+        self.src = self.tempdir()
         self.files = files = [
             pjoin(u'ƒile'),
             pjoin(u'∂ir', u'ƒile1'),
@@ -60,19 +76,24 @@ def setUp(self):
                 os.makedirs(parent)
             touch(fullpath)
         
-        self.data_dir = self.tempdir()
-        self.patch_data = patch.object(nbextensions,
-            'jupyter_data_dir', lambda : self.data_dir)
-        self.patch_data.start()
-        self.system_path = [self.tempdir()]
-        self.system_nbext = pjoin(self.system_path[0], 'nbextensions')
-        self.patch_system_data = patch.object(nbextensions,
+        self.test_dir = self.tempdir()
+        self.data_dir = os.path.join(self.test_dir, 'data')
+        self.config_dir = os.path.join(self.test_dir, 'config')
+        self.system_data_dir = os.path.join(self.test_dir, 'system_data')
+        self.system_path = [self.system_data_dir]
+        self.system_nbext = os.path.join(self.system_data_dir, 'nbextensions')
+        self.patch_env = patch.dict('os.environ', {
+            'JUPYTER_CONFIG_DIR': self.config_dir,
+            'JUPYTER_DATA_DIR': self.data_dir,
+        })
+        self.patch_env.start()
+        self.patch_system_path = patch.object(nbextensions,
             'SYSTEM_JUPYTER_PATH', self.system_path)
-        self.patch_system_data.start()
+        self.patch_system_path.start()
     
     def tearDown(self):
-        self.patch_data.stop()
-        self.patch_system_data.stop()
+        self.patch_env.stop()
+        self.patch_system_path.stop()
 
     def assert_dir_exists(self, path):
         if not os.path.exists(path):
@@ -104,18 +125,20 @@ def assert_not_installed(self, relative_path, user=False):
     def test_create_data_dir(self):
         """install_nbextension when data_dir doesn't exist"""
         with TemporaryDirectory() as td:
-            self.data_dir = pjoin(td, u'jupyter_data')
-            install_nbextension(self.src, user=True)
-            self.assert_dir_exists(self.data_dir)
-            for file in self.files:
-                self.assert_installed(
-                    pjoin(basename(self.src), file),
-                    user=True,
-                )
+            data_dir = os.path.join(td, self.data_dir)
+            with patch.dict('os.environ', {
+                'JUPYTER_DATA_DIR': data_dir,
+            }):
+                install_nbextension(self.src, user=True)
+                self.assert_dir_exists(data_dir)
+                for file in self.files:
+                    self.assert_installed(
+                        pjoin(basename(self.src), file),
+                        user=True,
+                    )
     
     def test_create_nbextensions_user(self):
         with TemporaryDirectory() as td:
-            self.data_dir = pjoin(td, u'jupyter_data')
             install_nbextension(self.src, user=True)
             self.assert_installed(
                 pjoin(basename(self.src), u'ƒile'),
@@ -197,7 +220,7 @@ def test_update_file(self):
             install_nbextension(src)
             self.assert_installed(fname)
             dest = pjoin(self.system_nbext, fname)
-            old_mtime = os.stat(dest).st_mtime
+            os.stat(dest).st_mtime
             with open(src, 'w') as f:
                 f.write('overwrite')
             touch(src, mtime + 10)
@@ -225,7 +248,7 @@ def test_quiet(self):
         stderr = StringIO()
         with patch.object(sys, 'stdout', stdout), \
              patch.object(sys, 'stderr', stderr):
-            install_nbextension(self.src, verbose=0)
+            install_nbextension(self.src)
         self.assertEqual(stdout.getvalue(), '')
         self.assertEqual(stderr.getvalue(), '')
     
@@ -343,3 +366,130 @@ def test_install_destination_bad(self):
         
             with self.assertRaises(ValueError):
                 install_nbextension(zsrc, destination='foo')
+
+    def test_nbextension_enable(self):
+        with TemporaryDirectory() as d:
+            f = u'ƒ.js'
+            src = pjoin(d, f)
+            touch(src)
+            install_nbextension(src, user=True)
+            enable_nbextension(section='notebook', require=u'ƒ')
+
+        config_dir = os.path.join(_get_config_dir(user=True), 'nbconfig')
+        cm = BaseJSONConfigManager(config_dir=config_dir)
+        enabled = cm.get('notebook').get('load_extensions', {}).get(u'ƒ', False)
+        assert enabled
+
+    def test_nbextension_disable(self):
+        self.test_nbextension_enable()
+        disable_nbextension(section='notebook', require=u'ƒ')
+
+        config_dir = os.path.join(_get_config_dir(user=True), 'nbconfig')
+        cm = BaseJSONConfigManager(config_dir=config_dir)
+        enabled = cm.get('notebook').get('load_extensions', {}).get(u'ƒ', False)
+        assert not enabled
+
+
+    def _mock_extension_spec_meta(self, section='notebook'):
+        return {
+            'section': section,
+            'src': 'mockextension',
+            'dest': '_mockdestination',
+            'require': '_mockdestination/index'
+        }
+
+    def _inject_mock_extension(self, section='notebook'):
+        outer_file = __file__
+
+        meta = self._mock_extension_spec_meta(section)
+
+        class mock():
+            __file__ = outer_file
+
+            @staticmethod
+            def _jupyter_nbextension_paths():
+                return [meta]
+
+        import sys
+        sys.modules['mockextension'] = mock
+
+    def test_nbextensionpy_files(self):
+        self._inject_mock_extension()
+        install_nbextension_python('mockextension')
+
+        assert check_nbextension('_mockdestination/index.js')
+        assert check_nbextension(['_mockdestination/index.js'])
+
+    def test_nbextensionpy_user_files(self):
+        self._inject_mock_extension()
+        install_nbextension_python('mockextension', user=True)
+
+        assert check_nbextension('_mockdestination/index.js', user=True)
+        assert check_nbextension(['_mockdestination/index.js'], user=True)
+
+    def test_nbextensionpy_uninstall_files(self):
+        self._inject_mock_extension()
+        install_nbextension_python('mockextension', user=True)
+        uninstall_nbextension_python('mockextension', user=True)
+
+        assert not check_nbextension('_mockdestination/index.js')
+        assert not check_nbextension(['_mockdestination/index.js'])
+
+    def test_nbextensionpy_enable(self):
+        self._inject_mock_extension('notebook')
+        install_nbextension_python('mockextension', user=True)
+        enable_nbextension_python('mockextension')
+
+        config_dir = os.path.join(_get_config_dir(user=True), 'nbconfig')
+        cm = BaseJSONConfigManager(config_dir=config_dir)
+        enabled = cm.get('notebook').get('load_extensions', {}).get('_mockdestination/index', False)
+        assert enabled
+
+    def test_nbextensionpy_disable(self):
+        self._inject_mock_extension('notebook')
+        install_nbextension_python('mockextension', user=True)
+        enable_nbextension_python('mockextension')
+        disable_nbextension_python('mockextension', user=True)
+
+        config_dir = os.path.join(_get_config_dir(user=True), 'nbconfig')
+        cm = BaseJSONConfigManager(config_dir=config_dir)
+        enabled = cm.get('notebook').get('load_extensions', {}).get('_mockdestination/index', False)
+        assert not enabled
+
+    def test_nbextensionpy_validate(self):
+        self._inject_mock_extension('notebook')
+
+        paths = install_nbextension_python('mockextension', user=True)
+        enable_nbextension_python('mockextension')
+
+        meta = self._mock_extension_spec_meta()
+        warnings = validate_nbextension_python(meta, paths[0])
+        self.assertEqual([], warnings, warnings)
+
+    def test_nbextensionpy_validate_bad(self):
+        # Break the metadata (correct file will still be copied)
+        self._inject_mock_extension('notebook')
+
+        paths = install_nbextension_python('mockextension', user=True)
+
+        enable_nbextension_python('mockextension')
+
+        meta = self._mock_extension_spec_meta()
+        meta.update(require="bad-require")
+
+        warnings = validate_nbextension_python(meta, paths[0])
+        self.assertNotEqual([], warnings, warnings)
+
+    def test_nbextension_validate(self):
+        # Break the metadata (correct file will still be copied)
+        self._inject_mock_extension('notebook')
+
+        install_nbextension_python('mockextension', user=True)
+        enable_nbextension_python('mockextension')
+
+        warnings = validate_nbextension("_mockdestination/index")
+        self.assertEqual([], warnings, warnings)
+
+    def test_nbextension_validate_bad(self):
+        warnings = validate_nbextension("this-doesn't-exist")
+        self.assertNotEqual([], warnings, warnings)
diff --git a/notebook/tests/test_serverextensions.py b/notebook/tests/test_serverextensions.py
new file mode 100644
index 0000000000..ee505938df
--- /dev/null
+++ b/notebook/tests/test_serverextensions.py
@@ -0,0 +1,89 @@
+import os
+from unittest import TestCase
+try:
+    from unittest.mock import patch
+except ImportError:
+    from mock import patch # py2
+
+from ipython_genutils.tempdir import TemporaryDirectory
+from ipython_genutils import py3compat
+
+from traitlets.config.manager import BaseJSONConfigManager
+from traitlets.tests.utils import check_help_all_output
+
+from notebook.serverextensions import toggle_serverextension_python
+from notebook import nbextensions
+from notebook.nbextensions import _get_config_dir
+
+
+def test_help_output():
+    check_help_all_output('notebook.serverextensions')
+    check_help_all_output('notebook.serverextensions', ['enable'])
+    check_help_all_output('notebook.serverextensions', ['disable'])
+    check_help_all_output('notebook.serverextensions', ['install'])
+    check_help_all_output('notebook.serverextensions', ['uninstall'])
+
+
+class TestInstallServerExtension(TestCase):
+
+    def tempdir(self):
+        td = TemporaryDirectory()
+        self.tempdirs.append(td)
+        return py3compat.cast_unicode(td.name)
+
+    def setUp(self):
+        self.tempdirs = []
+
+        self.test_dir = self.tempdir()
+        self.data_dir = os.path.join(self.test_dir, 'data')
+        self.config_dir = os.path.join(self.test_dir, 'config')
+        self.system_data_dir = os.path.join(self.test_dir, 'system_data')
+        self.system_path = [self.system_data_dir]
+
+        self.patch_env = patch.dict('os.environ', {
+            'JUPYTER_CONFIG_DIR': self.config_dir,
+            'JUPYTER_DATA_DIR': self.data_dir,
+        })
+        self.patch_env.start()
+        self.patch_system_path = patch.object(nbextensions,
+            'SYSTEM_JUPYTER_PATH', self.system_path)
+        self.patch_system_path.start()
+
+    def tearDown(self):
+        self.patch_env.stop()
+        self.patch_system_path.stop()
+
+    def _inject_mock_extension(self):
+        outer_file = __file__
+
+        class mock():
+            __file__ = outer_file
+
+            @staticmethod
+            def _jupyter_server_extension_paths():
+                return [{
+                    'module': '_mockdestination/index'
+                }]
+
+        import sys
+        sys.modules['mockextension'] = mock
+
+    def _get_config(self, user=True):
+        cm = BaseJSONConfigManager(config_dir=_get_config_dir(user))
+        data = cm.get("jupyter_notebook_config")
+        return data.get("NotebookApp", {}).get("nbserver_extensions", {})
+
+    def test_enable(self):
+        self._inject_mock_extension()
+        toggle_serverextension_python('mockextension', True)
+
+        config = self._get_config()
+        assert config['mockextension']
+
+    def test_disable(self):
+        self._inject_mock_extension()
+        toggle_serverextension_python('mockextension', True)
+        toggle_serverextension_python('mockextension', False)
+
+        config = self._get_config()
+        assert not config['mockextension']
diff --git a/notebook/utils.py b/notebook/utils.py
index ee9c6acebe..92a2410ccc 100644
--- a/notebook/utils.py
+++ b/notebook/utils.py
@@ -102,12 +102,18 @@ def is_hidden(abs_path, abs_root=''):
         return True
     
     # check that dirs can be listed
-    # may fail on Windows junctions or non-user-readable dirs
     if os.path.isdir(abs_path):
-        try:
-            os.listdir(abs_path)
-        except OSError:
-            return True
+        if sys.platform == 'win32':
+            # can't trust os.access on Windows because it seems to always return True
+            try:
+                os.stat(abs_path)
+            except OSError:
+                # stat may fail on Windows junctions or non-user-readable dirs
+                return True
+        else:
+            # use x-access, not actual listing, in case of slow/large listings
+            if not os.access(abs_path, os.X_OK | os.R_OK):
+                return True
     
     # check UF_HIDDEN on any location up to root
     path = abs_path
diff --git a/scripts/jupyter-serverextension b/scripts/jupyter-serverextension
new file mode 100644
index 0000000000..79b0b38e88
--- /dev/null
+++ b/scripts/jupyter-serverextension
@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+
+from notebook.serverextensions import main
+
+if __name__ == '__main__':
+    main()
diff --git a/setup.py b/setup.py
index cf3178637d..34e409ecda 100755
--- a/setup.py
+++ b/setup.py
@@ -182,6 +182,7 @@
         'console_scripts': [
             'jupyter-notebook = notebook.notebookapp:main',
             'jupyter-nbextension = notebook.nbextensions:main',
+            'jupyter-serverextension = notebook.serverextensions:main',
         ]
     }
     setup_args.pop('scripts', None)
diff --git a/setupbase.py b/setupbase.py
index 62d4e6324d..e722449661 100644
--- a/setupbase.py
+++ b/setupbase.py
@@ -142,7 +142,8 @@ def find_package_data():
         pjoin(components, "underscore", "underscore-min.js"),
         pjoin(components, "moment", "moment.js"),
         pjoin(components, "moment", "min", "moment.min.js"),
-        pjoin(components, "term.js", "src", "term.js"),
+        pjoin(components, "xterm.js", "src", "xterm.js"),
+        pjoin(components, "xterm.js", "src", "xterm.css"),
         pjoin(components, "text-encoding", "lib", "encoding.js"),
     ])
 
diff --git a/tools/build-main.js b/tools/build-main.js
index 1ae0a64a18..9f6e8539e8 100644
--- a/tools/build-main.js
+++ b/tools/build-main.js
@@ -21,7 +21,7 @@ var rjs_config = {
     "jquery-ui": 'components/jquery-ui/ui/minified/jquery-ui.min',
     moment: 'components/moment/moment',
     codemirror: 'components/codemirror',
-    termjs: 'components/term.js/src/term',
+    termjs: 'components/xterm.js/src/xterm',
     typeahead: 'components/jquery-typeahead/dist/jquery.typeahead',
     contents: 'empty:'
   },