docs/source/user_guide/tutorial_intro.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    ".. _pytest_notebook_by_example:\n",
    "\n",
    "# pytest-notebook by example"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    ".. note::\n",
    "\n",
    "    This tutorial is a notebook, that can be downloaded to run locally :download:`tutorial_intro.ipynb`,\n",
    "    and was converted to documentation with [ipypublish](https://ipypublish.readthedocs.io)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Python API"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The principal component of `pytest-notebook` is the\n",
    ":py:class:`~pytest_notebook.nb_regression.NBRegressionFixture` class,\n",
    "which is an [attrs](http://www.attrs.org) class, whose parameters can be instatiated or set via attibutes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import importlib_resources\n",
    "from pytest_notebook import example_nbs\n",
    "from pytest_notebook.nb_regression import NBRegressionFixture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "NBRegressionFixture(exec_notebook=True, exec_cwd=None, exec_allow_errors=False, exec_timeout=50, coverage=False, cov_config=None, cov_source=None, cov_merge=None, post_processors=('coalesce_streams',), process_resources={}, diff_replace=(), diff_ignore=('/cells/*/outputs/*/traceback',), diff_use_color=True, diff_color_words=False, force_regen=False)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fixture = NBRegressionFixture(exec_timeout=50)\n",
    "fixture.diff_color_words = False\n",
    "fixture"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The main method is :py:meth:`~pytest_notebook.nb_regression.NBRegressionFixture.check`, which executes a notebook and compares its initial and final contents."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true,
    "tags": [
     "raises-exception"
    ]
   },
   "outputs": [
    {
     "ename": "NBRegressionError",
     "evalue": "\n--- expected\n+++ obtained\n\u001b[34m## replaced /cells/1/execution_count:\u001b[0m\n\u001b[31m-  2\n\u001b[32m+  1\n\n\u001b[0m\u001b[34m## modified /cells/2/outputs/0/text:\u001b[0m\n\u001b[36m@@ -1,2 +1,2 @@\u001b[m\n hallo1\u001b[m\n\u001b[31m-hallo3\u001b[m\n\u001b[32m+\u001b[m\u001b[32mhallo2\u001b[m\n\n\u001b[0m",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNBRegressionError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-23ab8b792066>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mimportlib_resources\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexample_nbs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"example1.ipynb\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0mfixture\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcheck\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/GitHub/pytest-notebook/pytest_notebook/nb_regression.py\u001b[0m in \u001b[0;36mcheck\u001b[0;34m(self, path, raise_errors)\u001b[0m\n\u001b[1;32m    367\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mregen_exc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    368\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0mfiltered_diff\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 369\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mNBRegressionError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdiff_string\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    370\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    371\u001b[0m         return NBRegressionResult(\n",
      "\u001b[0;31mNBRegressionError\u001b[0m: \n--- expected\n+++ obtained\n\u001b[34m## replaced /cells/1/execution_count:\u001b[0m\n\u001b[31m-  2\n\u001b[32m+  1\n\n\u001b[0m\u001b[34m## modified /cells/2/outputs/0/text:\u001b[0m\n\u001b[36m@@ -1,2 +1,2 @@\u001b[m\n hallo1\u001b[m\n\u001b[31m-hallo3\u001b[m\n\u001b[32m+\u001b[m\u001b[32mhallo2\u001b[m\n\n\u001b[0m"
     ]
    }
   ],
   "source": [
    "with importlib_resources.path(example_nbs, \"example1.ipynb\") as path:\n",
    "    fixture.check(str(path))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To return the results, without raising an exception, use ``raise_errors=False``. This returns a :py:class:`~pytest_notebook.nb_regression.NBRegressionResult` instance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "NBRegressionResult(diff_full_length=1,diff_filtered_length=1)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with importlib_resources.path(example_nbs, \"example1.ipynb\") as path:\n",
    "    result = fixture.check(str(path), raise_errors=False)\n",
    "\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "--- expected\n",
      "+++ obtained\n",
      "\u001b[34m## replaced /cells/1/execution_count:\u001b[0m\n",
      "\u001b[31m-  2\n",
      "\u001b[32m+  1\n",
      "\n",
      "\u001b[0m\u001b[34m## modified /cells/2/outputs/0/text:\u001b[0m\n",
      "\u001b[36m@@ -1,2 +1,2 @@\u001b[m\n",
      " hallo1\u001b[m\n",
      "\u001b[31m-hallo3\u001b[m\n",
      "\u001b[32m+\u001b[m\u001b[32mhallo2\u001b[m\n",
      "\n",
      "\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "print(result.diff_string)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The diff of the notebooks is returned as a list of [nbdime](https://nbdime.readthedocs.io) `DiffEntry` objects."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'op': 'patch',\n",
       "  'key': 'cells',\n",
       "  'diff': [{'op': 'patch',\n",
       "    'key': 1,\n",
       "    'diff': [{'op': 'replace', 'key': 'execution_count', 'value': 1}]},\n",
       "   {'op': 'patch',\n",
       "    'key': 2,\n",
       "    'diff': [{'op': 'patch',\n",
       "      'key': 'outputs',\n",
       "      'diff': [{'op': 'patch',\n",
       "        'key': 0,\n",
       "        'diff': [{'op': 'patch',\n",
       "          'key': 'text',\n",
       "          'diff': [{'op': 'patch',\n",
       "            'key': 1,\n",
       "            'diff': [{'op': 'addrange', 'key': 5, 'valuelist': '2'},\n",
       "             {'op': 'removerange', 'key': 5, 'length': 1}]}]}]}]}]}]}]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.diff_filtered"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The notebooks are returned as :py:class:`nbformat.NotebookNode` instances."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'cells': [{'cell_type': 'markdown',\n",
       "   'metadata': {},\n",
       "   'source': '# Markdown Cell'},\n",
       "  {'cell_type': 'code',\n",
       "   'execution_count': 1,\n",
       "   'metadata': {},\n",
       "   'outputs': [],\n",
       "   'source': 'from IPython import display'},\n",
       "  {'cell_type': 'code',\n",
       "   'execution_count': 2,\n",
       "   'metadata': {},\n",
       "   'outputs': [{'output_type': 'stream',\n",
       "     'name': 'stdout',\n",
       "     'text': 'hallo1\\nhallo2\\n'}],\n",
       "   'source': \"print('hallo1')\\nprint('hallo2')\"}],\n",
       " 'metadata': {'kernelspec': {'display_name': 'Python 3',\n",
       "   'language': 'python',\n",
       "   'name': 'python3'},\n",
       "  'language_info': {'name': 'python',\n",
       "   'version': '3.6.7',\n",
       "   'mimetype': 'text/x-python',\n",
       "   'codemirror_mode': {'name': 'ipython', 'version': 3},\n",
       "   'pygments_lexer': 'ipython3',\n",
       "   'nbconvert_exporter': 'python',\n",
       "   'file_extension': '.py'}},\n",
       " 'nbformat': 4,\n",
       " 'nbformat_minor': 2}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.nb_final"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## pytest fixture"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    ".. seealso::\n",
    "\n",
    "    :py:mod:`pytest_notebook.ipy_magic`,\n",
    "    for the notebook magic used to run pytest in a Jupyter Notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext pytest_notebook.ipy_magic"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A :py:class:`~pytest_notebook.nb_regression.NBRegressionFixture` instance can accessed *via* the :py:func:`~pytest_notebook.plugin.nb_regression` fixture.\n",
    "This instance will be instatiated with parameters dictated by arguments parsed from the pytest command-line and configuration file(s).\n",
    "\n",
    ".. note::\n",
    "\n",
    "    pytest-notebook command-line and configuration file parameter names\n",
    "    are the same as for ``NBRegressionFixture``, but with the prefix ``nb_``.\n",
    "    \n",
    "    The command-line parameter takes precedence over the configuration file one."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m============================= test session starts ==============================\u001b[0m\n",
      "platform darwin -- Python 3.6.7, pytest-5.0.1, py-1.8.0, pluggy-0.12.0 -- //anaconda/envs/pytest_nb/bin/python\n",
      "cachedir: .pytest_cache\n",
      "rootdir: /private/var/folders/dm/b2qnkb_n3r72slmpxlfmcjvm00lbnd/T/tmppyh8kyts, inifile: pytest.ini\n",
      "plugins: cov-2.7.1, datadir-1.3.0, regressions-1.0.5, notebook-0.5.1\n",
      "\u001b[1mcollecting ... \u001b[0mcollected 1 item\n",
      "\n",
      "test_ipycell.py::test_notebook \u001b[31mFAILED\u001b[0m\u001b[36m                                    [100%]\u001b[0m\n",
      "\n",
      "=================================== FAILURES ===================================\n",
      "\u001b[31m\u001b[1m________________________________ test_notebook _________________________________\u001b[0m\n",
      "\n",
      "nb_regression = NBRegressionFixture(exec_notebook=True, exec_cwd='/Users/cjs14/GitHub/pytest-notebook/pytest_notebook/example_nbs', ex...place=(), diff_ignore=('/cells/*/outputs/*/traceback',), diff_use_color=True, diff_color_words=True, force_regen=False)\n",
      "\n",
      "\u001b[1m    def test_notebook(nb_regression):\u001b[0m\n",
      "\u001b[1m        with importlib_resources.path(example_nbs, \"example1.ipynb\") as path:\u001b[0m\n",
      "\u001b[1m>           nb_regression.check(str(path))\u001b[0m\n",
      "\u001b[1m\u001b[31mE           pytest_notebook.nb_regression.NBRegressionError: \u001b[0m\n",
      "\u001b[1m\u001b[31mE           --- expected\u001b[0m\n",
      "\u001b[1m\u001b[31mE           +++ obtained\u001b[0m\n",
      "\u001b[1m\u001b[31mE           \u001b[34m## replaced /cells/1/execution_count:\u001b[0m\u001b[0m\n",
      "\u001b[1m\u001b[31mE           \u001b[31m-  2\u001b[0m\n",
      "\u001b[1m\u001b[31mE           \u001b[32m+  1\u001b[0m\n",
      "\u001b[1m\u001b[31mE           \u001b[0m\n",
      "\u001b[1m\u001b[31mE           \u001b[0m\u001b[34m## modified /cells/2/outputs/0/text:\u001b[0m\u001b[0m\n",
      "\u001b[1m\u001b[31mE           \u001b[36m@@ -1,2 +1,2 @@\u001b[m\u001b[0m\n",
      "\u001b[1m\u001b[31mE           hallo1\u001b[m\u001b[0m\n",
      "\u001b[1m\u001b[31mE           \u001b[31mhallo3\u001b[m\u001b[32mhallo2\u001b[m\u001b[0m\n",
      "\u001b[1m\u001b[31mE           \u001b[0m\n",
      "\u001b[1m\u001b[31mE           \u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[1m\u001b[31mtest_ipycell.py\u001b[0m:8: NBRegressionError\n",
      "\u001b[31m\u001b[1m===================== 1 failed, 1 warnings in 2.42 seconds =====================\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "%%pytest -v --color=yes --disable-warnings --nb-exec-timeout 50\n",
    "\n",
    "---\n",
    "[pytest]\n",
    "nb_diff_color_words = True\n",
    "---\n",
    "\n",
    "import importlib_resources\n",
    "from pytest_notebook import example_nbs\n",
    "\n",
    "def test_notebook(nb_regression):\n",
    "    with importlib_resources.path(example_nbs, \"example1.ipynb\") as path:\n",
    "        nb_regression.check(str(path))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## pytest file collection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    ":py:meth:`~pytest_notebook.nb_regression.NBRegressionFixture.check` can be run automatically on all notebooks using the pytest collection mechanism.\n",
    "To activate this feature, set `--nb-test-files` on the command-line, or `nb_test_files = True` in the configuration file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "notebook1_content = importlib_resources.read_text(example_nbs, \"example1_pass.ipynb\")\n",
    "notebook2_content = importlib_resources.read_text(example_nbs, \"example1.ipynb\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m============================= test session starts ==============================\u001b[0m\n",
      "platform darwin -- Python 3.6.7, pytest-5.0.1, py-1.8.0, pluggy-0.12.0 -- //anaconda/envs/pytest_nb/bin/python\n",
      "cachedir: .pytest_cache\n",
      "rootdir: /private/var/folders/dm/b2qnkb_n3r72slmpxlfmcjvm00lbnd/T/tmpvs4w3kxu\n",
      "plugins: cov-2.7.1, datadir-1.3.0, regressions-1.0.5, notebook-0.5.1\n",
      "\u001b[1mcollecting ... \u001b[0mcollected 2 items\n",
      "\n",
      "test_notebook1.ipynb::nbregression(test_notebook1) \u001b[32mPASSED\u001b[0m\u001b[36m                [ 50%]\u001b[0m\n",
      "test_notebook2.ipynb::nbregression(test_notebook2) \u001b[31mFAILED\u001b[0m\u001b[36m                [100%]\u001b[0m\n",
      "\n",
      "=================================== FAILURES ===================================\n",
      "\u001b[31m\u001b[1m____________________ notebook: nbregression(test_notebook2) ____________________\u001b[0m\n",
      "pytest_notebook.nb_regression.NBRegressionError: \n",
      "--- expected\n",
      "+++ obtained\n",
      "\u001b[34m## replaced /cells/1/execution_count:\u001b[0m\n",
      "\u001b[31m-  2\n",
      "\u001b[32m+  1\n",
      "\n",
      "\u001b[0m\u001b[34m## modified /cells/2/outputs/0/text:\u001b[0m\n",
      "\u001b[36m@@ -1,2 +1,2 @@\u001b[m\n",
      " hallo1\u001b[m\n",
      "\u001b[31m-hallo3\u001b[m\n",
      "\u001b[32m+\u001b[m\u001b[32mhallo2\u001b[m\n",
      "\n",
      "\u001b[0m\n",
      "\u001b[31m\u001b[1m================ 1 failed, 1 passed, 1 warnings in 3.64 seconds ================\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "%%pytest -v --color=yes --disable-warnings --nb-test-files\n",
    "\n",
    "***\n",
    "(notebook1_content, \"test_notebook1.ipynb\")\n",
    "(notebook2_content, \"test_notebook2.ipynb\")\n",
    "***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m============================= test session starts ==============================\u001b[0m\n",
      "platform darwin -- Python 3.6.7, pytest-5.0.1, py-1.8.0, pluggy-0.12.0 -- //anaconda/envs/pytest_nb/bin/python\n",
      "cachedir: .pytest_cache\n",
      "rootdir: /private/var/folders/dm/b2qnkb_n3r72slmpxlfmcjvm00lbnd/T/tmphhv85_77, inifile: pytest.ini\n",
      "plugins: cov-2.7.1, datadir-1.3.0, regressions-1.0.5, notebook-0.5.1\n",
      "\u001b[1mcollecting ... \u001b[0mcollected 2 items\n",
      "\n",
      "test_notebook1.ipynb::nbregression(test_notebook1) \u001b[32mPASSED\u001b[0m\u001b[36m                [ 50%]\u001b[0m\n",
      "test_notebook2.ipynb::nbregression(test_notebook2) \u001b[31mFAILED\u001b[0m\u001b[36m                [100%]\u001b[0m\n",
      "\n",
      "=================================== FAILURES ===================================\n",
      "\u001b[31m\u001b[1m____________________ notebook: nbregression(test_notebook2) ____________________\u001b[0m\n",
      "pytest_notebook.nb_regression.NBRegressionError: \n",
      "--- expected\n",
      "+++ obtained\n",
      "\u001b[34m## replaced /cells/1/execution_count:\u001b[0m\n",
      "\u001b[31m-  2\n",
      "\u001b[32m+  1\n",
      "\n",
      "\u001b[0m\u001b[34m## modified /cells/2/outputs/0/text:\u001b[0m\n",
      "\u001b[36m@@ -1,2 +1,2 @@\u001b[m\n",
      " hallo1\u001b[m\n",
      "\u001b[31m-hallo3\u001b[m\n",
      "\u001b[32m+\u001b[m\u001b[32mhallo2\u001b[m\n",
      "\n",
      "\u001b[0m\n",
      "\u001b[31m\u001b[1m================ 1 failed, 1 passed, 1 warnings in 3.76 seconds ================\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "%%pytest -v --color=yes --disable-warnings\n",
    "\n",
    "---\n",
    "[pytest]\n",
    "nb_test_files = True\n",
    "---\n",
    "\n",
    "***\n",
    "(notebook1_content, \"test_notebook1.ipynb\")\n",
    "(notebook2_content, \"test_notebook2.ipynb\")\n",
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To restrict the notebook files pytest collects, one or more filename pattern matches can also be set (see :py:mod:`fnmatch`)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m============================= test session starts ==============================\u001b[0m\n",
      "platform darwin -- Python 3.6.7, pytest-5.0.1, py-1.8.0, pluggy-0.12.0 -- //anaconda/envs/pytest_nb/bin/python\n",
      "cachedir: .pytest_cache\n",
      "rootdir: /private/var/folders/dm/b2qnkb_n3r72slmpxlfmcjvm00lbnd/T/tmpd18whmsj, inifile: pytest.ini\n",
      "plugins: cov-2.7.1, datadir-1.3.0, regressions-1.0.5, notebook-0.5.1\n",
      "\u001b[1mcollecting ... \u001b[0mcollected 1 item\n",
      "\n",
      "test_notebook1.ipynb::nbregression(test_notebook1) \u001b[32mPASSED\u001b[0m\u001b[36m                [100%]\u001b[0m\n",
      "\n",
      "\u001b[33m\u001b[1m===================== 1 passed, 1 warnings in 2.31 seconds =====================\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "%%pytest -v --color=yes --disable-warnings\n",
    "\n",
    "---\n",
    "[pytest]\n",
    "nb_test_files = True\n",
    "nb_file_fnmatch = test_*.ipynb tutorial_*.ipynb\n",
    "---\n",
    "\n",
    "***\n",
    "(notebook1_content, \"test_notebook1.ipynb\")\n",
    "(notebook2_content, \"other_notebook2.ipynb\")\n",
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Live Logging of Cell Execution"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If you wish to view the progress of the notebook execution, you can use the [pytest live-logging](https://docs.pytest.org/en/latest/logging.html#live-logs) functionality:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m============================= test session starts ==============================\u001b[0m\n",
      "platform darwin -- Python 3.6.7, pytest-5.0.1, py-1.8.0, pluggy-0.12.0 -- //anaconda/envs/pytest_nb/bin/python\n",
      "cachedir: .pytest_cache\n",
      "rootdir: /private/var/folders/dm/b2qnkb_n3r72slmpxlfmcjvm00lbnd/T/tmpzax121qh\n",
      "plugins: cov-2.7.1, datadir-1.3.0, regressions-1.0.5, notebook-0.5.1\n",
      "\u001b[1mcollecting ... \u001b[0mcollected 1 item\n",
      "\n",
      "test_notebook1.ipynb::nbregression(test_notebook1) \n",
      "\u001b[1m-------------------------------- live log call ---------------------------------\u001b[0m\n",
      "\u001b[32mINFO    \u001b[0m pytest_notebook.execution:execution.py:142 About to execute notebook with 3 cells\n",
      "\u001b[32mINFO    \u001b[0m pytest_notebook.execution:execution.py:148 Executing notebook with kernel: python3\n",
      "\u001b[32mINFO    \u001b[0m pytest_notebook.execution:execution.py:168 Executing cell 1\n",
      "\u001b[32mINFO    \u001b[0m pytest_notebook.execution:execution.py:168 Executing cell 2\n",
      "\u001b[32mPASSED\u001b[0m\u001b[36m                                                                   [100%]\u001b[0m\n",
      "\n",
      "\u001b[33m\u001b[1m===================== 1 passed, 1 warnings in 2.40 seconds =====================\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "%%pytest -v --color=yes --disable-warnings --nb-test-files --log-cli-level=info\n",
    "\n",
    "***\n",
    "(notebook1_content, \"test_notebook1.ipynb\")\n",
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    ".. _regen_notebooks:\n",
    "\n",
    "## Regenerating Notebooks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Failing notebooks can be regenerated by setting `--nb-force-regen`.\n",
    "This will overwrite failing notebooks with the output from the notebook execution.\n",
    "\n",
    ".. note::\n",
    "\n",
    "    Notebooks will not be regenerated if they raise any unexpected exceptions,\n",
    "    during execution.\n",
    "    \n",
    "    This approach to regeneration mimics [pytest-regressions](https://pytest-regressions.readthedocs.io)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m============================= test session starts ==============================\u001b[0m\n",
      "platform darwin -- Python 3.6.7, pytest-5.0.1, py-1.8.0, pluggy-0.12.0 -- //anaconda/envs/pytest_nb/bin/python\n",
      "cachedir: .pytest_cache\n",
      "NB force regen: True\n",
      "rootdir: /private/var/folders/dm/b2qnkb_n3r72slmpxlfmcjvm00lbnd/T/tmp7n8y_fg0, inifile: pytest.ini\n",
      "plugins: cov-2.7.1, datadir-1.3.0, regressions-1.0.5, notebook-0.5.1\n",
      "\u001b[1mcollecting ... \u001b[0mcollected 2 items\n",
      "\n",
      "test_notebook1.ipynb::nbregression(test_notebook1) \u001b[32mPASSED\u001b[0m\u001b[36m                [ 50%]\u001b[0m\n",
      "test_notebook2.ipynb::nbregression(test_notebook2) \u001b[31mFAILED\u001b[0m\u001b[36m                [100%]\u001b[0m\n",
      "\n",
      "=================================== FAILURES ===================================\n",
      "\u001b[31m\u001b[1m____________________ notebook: nbregression(test_notebook2) ____________________\u001b[0m\n",
      "pytest_notebook.nb_regression.NBRegressionError: Files differ and --nb-force-regen set, regenerating file at:\n",
      "- /private/var/folders/dm/b2qnkb_n3r72slmpxlfmcjvm00lbnd/T/tmp7n8y_fg0/test_notebook2.ipynb\n",
      "----------------------------- Captured stderr call -----------------------------\n",
      "Diff before regeneration:\n",
      "\n",
      "--- expected\n",
      "+++ obtained\n",
      "\u001b[34m## replaced /cells/1/execution_count:\u001b[0m\n",
      "\u001b[31m-  2\n",
      "\u001b[32m+  1\n",
      "\n",
      "\u001b[0m\u001b[34m## modified /cells/2/outputs/0/text:\u001b[0m\n",
      "\u001b[36m@@ -1,2 +1,2 @@\u001b[m\n",
      " hallo1\u001b[m\n",
      "\u001b[31m-hallo3\u001b[m\n",
      "\u001b[32m+\u001b[m\u001b[32mhallo2\u001b[m\n",
      "\n",
      "\u001b[0m\n",
      "\u001b[31m\u001b[1m================ 1 failed, 1 passed, 1 warnings in 3.67 seconds ================\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "%%pytest -v --color=yes --disable-warnings --nb-force-regen\n",
    "\n",
    "---\n",
    "[pytest]\n",
    "nb_test_files = True\n",
    "nb_force_regen = True\n",
    "---\n",
    "\n",
    "***\n",
    "(notebook1_content, \"test_notebook1.ipynb\")\n",
    "(notebook2_content, \"test_notebook2.ipynb\")\n",
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The regenration can be observed, if we run two tests on the same notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m============================= test session starts ==============================\u001b[0m\n",
      "platform darwin -- Python 3.6.7, pytest-5.0.1, py-1.8.0, pluggy-0.12.0 -- //anaconda/envs/pytest_nb/bin/python\n",
      "cachedir: .pytest_cache\n",
      "rootdir: /private/var/folders/dm/b2qnkb_n3r72slmpxlfmcjvm00lbnd/T/tmpeb7sjuqh\n",
      "plugins: cov-2.7.1, datadir-1.3.0, regressions-1.0.5, notebook-0.5.1\n",
      "\u001b[1mcollecting ... \u001b[0mcollected 2 items\n",
      "\n",
      "test_ipycell.py::test_notebook1 \u001b[31mFAILED\u001b[0m\u001b[36m                                   [ 50%]\u001b[0m\n",
      "test_ipycell.py::test_notebook2 \u001b[32mPASSED\u001b[0m\u001b[36m                                   [100%]\u001b[0m\n",
      "\n",
      "=================================== FAILURES ===================================\n",
      "\u001b[31m\u001b[1m________________________________ test_notebook1 ________________________________\u001b[0m\n",
      "\n",
      "nb_regression = NBRegressionFixture(exec_notebook=True, exec_cwd='/var/folders/dm/b2qnkb_n3r72slmpxlfmcjvm00lbnd/T', exec_allow_errors...place=(), diff_ignore=('/cells/*/outputs/*/traceback',), diff_use_color=True, diff_color_words=False, force_regen=True)\n",
      "notebook = '/var/folders/dm/b2qnkb_n3r72slmpxlfmcjvm00lbnd/T/tmp64orgem_.ipynb'\n",
      "\n",
      "\u001b[1m    def test_notebook1(nb_regression, notebook):\u001b[0m\n",
      "\u001b[1m        nb_regression.force_regen = True\u001b[0m\n",
      "\u001b[1m>       nb_regression.check(notebook)\u001b[0m\n",
      "\u001b[1m\u001b[31mE       pytest_notebook.nb_regression.NBRegressionError: Files differ and --nb-force-regen set, regenerating file at:\u001b[0m\n",
      "\u001b[1m\u001b[31mE       - /var/folders/dm/b2qnkb_n3r72slmpxlfmcjvm00lbnd/T/tmp64orgem_.ipynb\u001b[0m\n",
      "\n",
      "\u001b[1m\u001b[31mtest_ipycell.py\u001b[0m:20: NBRegressionError\n",
      "----------------------------- Captured stderr call -----------------------------\n",
      "Diff before regeneration:\n",
      "\n",
      "--- expected\n",
      "+++ obtained\n",
      "\u001b[34m## replaced /cells/1/execution_count:\u001b[0m\n",
      "\u001b[31m-  2\n",
      "\u001b[32m+  1\n",
      "\n",
      "\u001b[0m\u001b[34m## modified /cells/2/outputs/0/text:\u001b[0m\n",
      "\u001b[36m@@ -1,2 +1,2 @@\u001b[m\n",
      " hallo1\u001b[m\n",
      "\u001b[31m-hallo3\u001b[m\n",
      "\u001b[32m+\u001b[m\u001b[32mhallo2\u001b[m\n",
      "\n",
      "\u001b[0m\n",
      "\u001b[31m\u001b[1m================ 1 failed, 1 passed, 1 warnings in 3.76 seconds ================\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "%%pytest -v --color=yes --disable-warnings\n",
    "\n",
    "import os, tempfile\n",
    "import importlib_resources\n",
    "import pytest\n",
    "from pytest_notebook import example_nbs\n",
    "\n",
    "@pytest.fixture(scope=\"module\")\n",
    "def notebook():\n",
    "    tmphandle, tmppath = tempfile.mkstemp(suffix=\".ipynb\")\n",
    "    with open(tmppath, \"w\") as handle:\n",
    "        handle.write(\n",
    "            importlib_resources.read_text(example_nbs, \"example1.ipynb\")\n",
    "        )\n",
    "    yield tmppath\n",
    "    os.remove(tmppath)\n",
    "    \n",
    "\n",
    "def test_notebook1(nb_regression, notebook):\n",
    "    nb_regression.force_regen = True\n",
    "    nb_regression.check(notebook)\n",
    "\n",
    "def test_notebook2(nb_regression, notebook):\n",
    "    nb_regression.check(notebook)"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Tags",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}