#13 Support database plugins / rework of entrypoints

.circleci/config.yml

@@ -5,7 +5,7 @@ defaults: &defaults
       CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
       CIRCLE_TEST_REPORTS: /tmp/circleci-test-results
       # CODECOV_TOKEN: b0d35139-0a75-427a-907b-2c78a762f8f0
-      VERSION: 0.0.2
+      VERSION: 0.1.0
       PANDOC_RELEASES_URL: https://github.com/jgm/pandoc/releases
       YARN_STATIC_DIR: notebooker/web/static/
       IMAGE_NAME: mangroup/notebooker
@@ -25,25 +25,38 @@ defaults: &defaults
         name: Restore Yarn Package Cache
         keys:
           - yarn-packages-{{ checksum "notebooker/web/static/yarn.lock" }}
+    - run:
+        name: Version checks
+        command: |
+            grep -q $VERSION notebooker/_version.py || (echo "ERROR: Version number not found in notebooker/_version.py: $VERSION"; exit 1)
+            grep -q $VERSION CHANGELOG.md || (echo "ERROR: Version number not found in CHANGES.md: $VERSION"; exit 1)
+            grep -q $VERSION docs/conf.py || (echo "ERROR: Version number not found in docs/source/conf.py: $VERSION"; exit 1)
+            grep -q $VERSION notebooker/web/static/package.json || (echo "ERROR: Version number not found in package.json: $VERSION"; exit 1)
+    - run:
+        name: Install MongoDB
+        command: |
+            # run "cat /etc/os-release" to view information about the OS
+            # good article on how to install mongo, https://docs.mongodb.com/manual/tutorial/install-mongodb-on-ubuntu/
+
+            cat /etc/os-release
+            set -x
+            wget -qO - https://www.mongodb.org/static/pgp/server-4.2.asc | sudo apt-key add -
+            sudo apt-get install gnupg
+            wget -qO - https://www.mongodb.org/static/pgp/server-4.2.asc | sudo apt-key add -
+            echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.2 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.2.list
+            sudo apt-get update
+            sudo ln -s /bin/true /bin/systemctl
+            sudo apt-get install -y mongodb-org=4.2.11 mongodb-org-server=4.2.11 mongodb-org-shell=4.2.11 mongodb-org-mongos=4.2.11 mongodb-org-tools=4.2.11
     - run:
         name: Install JS Dependencies
         command: |
-          pushd $YARN_STATIC_DIR
-          yarn install --frozen-lockfile
+            pushd $YARN_STATIC_DIR
+            yarn install --frozen-lockfile
     - save_cache:
         name: Save Yarn Package Cache
         key: yarn-packages-{{ checksum "notebooker/web/static/yarn.lock" }}
         paths:
-          - ~/.cache/yarn
-    - run:
-        name: Install MongoDB
-        command: |
-          # run "cat /etc/os-release" to view information about the OS
-          # this article really helped with this madness: https://linuxize.com/post/how-to-install-mongodb-on-debian-9/
-          sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4
-          echo "deb http://repo.mongodb.org/apt/debian stretch/mongodb-org/4.0 main" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.0.list
-          sudo apt-get update
-          sudo apt-get install -y mongodb-org
+            - ~/.cache/yarn
     - run:
         name: Lint & Format JS Code
         command: |
@@ -86,12 +99,15 @@ defaults: &defaults
     - run:
        name: Run all tests
        command: |
+         set -x
          . ci/bin/activate
+         ls -la /bin | grep mongo
+         which mongod
          pip install -e .[prometheus,test]
          python -m ipykernel install --user --name=notebooker_kernel
          pip install -r ./notebooker/notebook_templates_example/notebook_requirements.txt
          mkdir test-results
-         pytest --junitxml=test-results/junit.xml
+         py.test -svvvvv --junitxml=test-results/junit.xml
          # bash <(curl -s https://codecov.io/bash) -c -F python
     - run:
         name: Build Sphinx Documentation
@@ -109,10 +125,6 @@ defaults: &defaults
           . ci/bin/activate
           pip install docutils
           pip install Pygments
-          grep -q $VERSION notebooker/_version.py || (echo "ERROR: Version number not found in notebooker/_version.py: $VERSION"; exit 1)
-          grep -q $VERSION CHANGELOG.md || (echo "ERROR: Version number not found in CHANGES.md: $VERSION"; exit 1)
-          grep -q $VERSION docs/conf.py || (echo "ERROR: Version number not found in docs/source/conf.py: $VERSION"; exit 1)
-          grep -q $VERSION notebooker/web/static/package.json || (echo "ERROR: Version number not found in package.json: $VERSION"; exit 1)
           python setup.py --long-description > ../README.rst
           cat ../README.rst | rst2html.py 1> ../README.html 2> ../log
           cp ../README.rst /tmp/circleci-artifacts
@@ -138,13 +150,19 @@ defaults: &defaults
         path: test-results
 version: 2
 jobs:
-  build:
-    working_directory: ~/notebooker
+  build_3_6:
+    working_directory: ~/notebooker_3_6
+    docker:
+    - image: cimg/python:3.6-node
+    <<: *defaults
+  build_3_7:
+    working_directory: ~/notebooker_3_7
     docker:
-    - image: circleci/python:3.6-stretch-node-browsers
+    - image: cimg/python:3.7-node
     <<: *defaults
 workflows:
   version: 2
   build_all:
     jobs:
-      - build
+      - build_3_6
+      - build_3_7

CHANGELOG.md

@@ -1,3 +1,12 @@
+0.1.0 (2020-11-30)
+------------------
+Support for database plugins and tidying up configuration to be consistent across the board.
+
+**Breaking changes**
+* 3 primary entrypoints have been consolidated under one - notebooker-cli, e.g. `notebooker-cli start-webapp` and `notebooker-cli execute-notebook`. Run notebooker-cli --help for more info. 
+* In config, PY_TEMPLATE_DIR has been renamed to PY_TEMPLATE_BASE_DIR
+* In config, GIT_REPO_TEMPLATE_DIR has been renamed to PY_TEMPLATE_SUBDIR
+
 0.0.2 (2020-10-25)
 ------------------
 Bugfixes & cleanup

docker/docker-compose.yaml

@@ -17,9 +17,9 @@ services:
       MONGO_HOST: mongodb:27017
       # this should be something like "notebooker" but this simplifies the compose file
       DATABASE_NAME: admin
-      RESULT_COLLECTION_NAME: notebook_results
+      RESULT_COLLECTION_NAME: NOTEBOOK_OUTPUT
 
-      PY_TEMPLATE_DIR: /var/run/template_repo
+      PY_TEMPLATE_BASE_DIR: /var/run/template_repo
     volumes:
       - git-repo:/var/run/template_repo
     command: ["notebooker_webapp"]

docs/conf.py

@@ -23,7 +23,7 @@
 author = "Man Group Quant Tech"
 
 # The full version, including alpha/beta/rc tags
-release = "0.0.2"
+release = "0.1.0"
 
 
 # -- General configuration ---------------------------------------------------

docs/report_execution.rst

@@ -19,6 +19,8 @@ Executing a Notebook
 There are two primary ways to do this: either through the webapp or through the entrypoint. Both
 of these methods will rely on a `notebooker_kernel` being available in the current ipykernel environment.
 
+For more information on the entrypoint, please run: `notebooker-cli execute-notebook --help`
+
 Technologies
 ------------
 Notebooker leverages multiple open-source technologies but in particular, it heavily makes use of some

docs/setup.rst

@@ -60,7 +60,7 @@ NB: mongo should be running as above for these steps to work!
 
 .. code:: bash
 
-    $ MONGO_HOST=localhost:27017 MONGO_USER=jon MONGO_PASSWORD=hello PORT=11828 notebooker_webapp
+    $ notebooker-cli --mongo-host localhost:27017 --mongo-user jon --mongo-password hello start-webapp --port 11828
 
 4. Open the link that is printed in your web browser.
 
@@ -117,7 +117,7 @@ NB: mongo should be running as above for these steps to work!
 
 .. code:: bash
 
-    $ MONGO_HOST=localhost:27017 MONGO_USER=jon MONGO_PASSWORD=hello PORT=11828 notebooker_webapp
+    $ notebooker-cli --mongo-host localhost:27017 --mongo-user jon --mongo-password hello start-webapp --port 11828
 
 
 7. Open the link that is printed in your web browser.

docs/templates.rst

@@ -25,8 +25,8 @@ installed, should be added to that folder.
 
 For Notebooker to use a your checked-out repository, set two environment variables:
 
-* Set :code:`PY_TEMPLATE_DIR` to the checked-out repository
-* Set :code:`GIT_REPO_TEMPLATE_DIR` to the subdirectory within your git repo which contains the templates
+* Set :code:`PY_TEMPLATE_BASE_DIR` to the checked-out repository
+* Set :code:`PY_TEMPLATE_SUBDIR` to the subdirectory within your git repo which contains the templates
 
 Adding parameters
 -----------------

notebooker/__init__.py

@@ -1 +1,3 @@
 from ._version import __version__
+
+__import__("pkg_resources").declare_namespace(__name__)

notebooker/_entrypoints.py

@@ -0,0 +1,172 @@
+import os
+import uuid
+
+import click
+
+from notebooker.constants import DEFAULT_SERIALIZER
+from notebooker.execute_notebook import execute_notebook_entrypoint
+from notebooker.serialization import SERIALIZER_TO_CLI_OPTIONS
+from notebooker.settings import BaseConfig, WebappConfig
+from notebooker.snapshot import snap_latest_successful_notebooks
+from notebooker.web.app import main
+
+
+class NotebookerEntrypoint(click.Group):
+    def parse_args(self, ctx, args):
+        try:
+            serializer_arg = args.index("--serializer-cls")
+            serializer = args[serializer_arg + 1]
+        except ValueError:
+            serializer = DEFAULT_SERIALIZER
+        self.params += SERIALIZER_TO_CLI_OPTIONS[serializer].params
+
+        return super().parse_args(ctx, args)
+
+
+pass_config = click.make_pass_decorator(BaseConfig)
+
+
+def filesystem_default_value(dirname):
+    return os.path.join(os.path.expanduser("~"), ".notebooker", dirname, str(uuid.uuid4()))
+
+
+@click.group(cls=NotebookerEntrypoint)
+@click.option("--notebook-kernel-name", default=None, help="The name of the kernel which is running our notebook code.")
+@click.option(
+    "--output-base-dir",
+    default=filesystem_default_value("output"),
+    help="The base directory to which we will save our notebook output temporarily. Required by Papermill.",
+)
+@click.option(
+    "--template-base-dir",
+    default=filesystem_default_value("templates"),
+    help="The base directory to which we will save our notebook templates which have been converted "
+    "from .py to .ipynb.",
+)
+@click.option(
+    "--py-template-base-dir",
+    default=None,
+    help="The base directory of the git repository which holds the notebook templates as .py files. "
+    "If not specified, this will default to the sample directory within notebooker.",
+)
+@click.option(
+    "--py-template-subdir",
+    default=None,
+    help="The subdirectory of the git repository which contains only notebook templates.",
+)
+@click.option(
+    "--notebooker-disable-git",
+    default=False,
+    is_flag=True,
+    help="If selected, notebooker will not try to pull the latest version of python templates from git.",
+)
+@click.option(
+    "--serializer-cls",
+    default=DEFAULT_SERIALIZER,
+    help="The serializer class through which we will save the notebook result.",
+)
+@click.pass_context
+def base_notebooker(
+    ctx,
+    notebook_kernel_name,
+    output_base_dir,
+    template_base_dir,
+    py_template_base_dir,
+    py_template_subdir,
+    notebooker_disable_git,
+    serializer_cls,
+    **serializer_args,
+):
+    config = BaseConfig(
+        SERIALIZER_CLS=serializer_cls,
+        SERIALIZER_CONFIG=serializer_args,
+        NOTEBOOK_KERNEL_NAME=notebook_kernel_name,
+        OUTPUT_DIR=output_base_dir,
+        TEMPLATE_DIR=template_base_dir,
+        PY_TEMPLATE_BASE_DIR=py_template_base_dir,
+        PY_TEMPLATE_SUBDIR=py_template_subdir,
+        NOTEBOOKER_DISABLE_GIT=notebooker_disable_git,
+    )
+    ctx.obj = config
+
+
+@base_notebooker.command()
+@click.option("--port", default=11828)
+@click.option("--logging-level", default="INFO")
+@click.option("--debug", default=False)
+@click.option("--base-cache-dir", default=filesystem_default_value("webcache"))
+@pass_config
+def start_webapp(config: BaseConfig, port, logging_level, debug, base_cache_dir):
+    web_config = WebappConfig.copy_existing(config)
+    web_config.PORT = port
+    web_config.LOGGING_LEVEL = logging_level
+    web_config.DEBUG = debug
+    web_config.CACHE_DIR = base_cache_dir
+    return main(web_config)
+
+
+@base_notebooker.command()
+@click.option("--report-name", help="The name of the template to execute, relative to the template directory.")
+@click.option(
+    "--overrides-as-json", default="{}", help="The parameters to inject into the notebook template, in JSON format."
+)
+@click.option(
+    "--iterate-override-values-of",
+    default="",
+    help="For the key/values in the overrides, set this to the value of one of the keys to run reports for "
+    "each of its values.",
+)
+@click.option("--report-title", default="", help="A custom title for this notebook. The default is the report_name.")
+@click.option("--n-retries", default=3, help="The number of times to retry when executing this notebook.")
+@click.option(
+    "--job-id",
+    default=str(uuid.uuid4()),
+    help="The unique job ID for this notebook. Can be non-unique, but note that you will overwrite history.",
+)
+@click.option("--mailto", default="", help="A comma-separated list of email addresses which will receive results.")
+@click.option("--pdf-output/--no-pdf-output", default=True, help="Whether we generate PDF output or not.")
+@click.option(
+    "--prepare-notebook-only",
+    is_flag=True,
+    help='Used for debugging and testing. Whether to actually execute the notebook or just "prepare" it.',
+)
+@pass_config
+def execute_notebook(
+    config: BaseConfig,
+    report_name,
+    overrides_as_json,
+    iterate_override_values_of,
+    report_title,
+    n_retries,
+    job_id,
+    mailto,
+    pdf_output,
+    prepare_notebook_only,
+):
+    if report_name is None:
+        raise ValueError("Error! Please provide a --report-name.")
+    return execute_notebook_entrypoint(
+        config,
+        report_name,
+        overrides_as_json,
+        iterate_override_values_of,
+        report_title,
+        n_retries,
+        job_id,
+        mailto,
+        pdf_output,
+        prepare_notebook_only,
+    )
+
+
+@base_notebooker.command()
+@click.option(
+    "--report-name", required=True, help="The name of the template to retrieve, relative to the template directory."
+)
+@pass_config
+def snapshot_latest_successful_notebooks(config: BaseConfig, report_name):
+    snap_latest_successful_notebooks(config, report_name)
+
+
+if __name__ == "__main__":
+    base_notebooker()

notebooker/_version.py

@@ -1 +1 @@
-__version__ = "0.0.2"
+__version__ = "0.1.0"

notebooker/constants.py

@@ -8,13 +8,17 @@
 
 SUBMISSION_TIMEOUT = 3
 RUNNING_TIMEOUT = 60
-NOTEBOOKER_TEMPLATE_GIT_URL = os.getenv("NOTEBOOKER_TEMPLATE_GIT_URL")
-NOTEBOOKER_DISABLE_GIT = os.getenv("NOTEBOOKER_DISABLE_GIT")
 CANCEL_MESSAGE = "The webapp shut down while this job was running. Please resubmit with the same parameters."
 TEMPLATE_DIR_SEPARATOR = "^"
+DEFAULT_SERIALIZER = "PyMongoResultSerializer"
 logger = logging.getLogger(__name__)
 
 
+DEFAULT_DATABASE_NAME = "notebooker"
+DEFAULT_MONGO_HOST = "localhost"
+DEFAULT_RESULT_COLLECTION_NAME = "NOTEBOOK_OUTPUT"
+
+
 def kernel_spec():
     return {
         "display_name": os.getenv("NOTEBOOK_KERNEL_NAME", "notebooker_kernel"),
@@ -23,9 +27,9 @@ def kernel_spec():
     }
 
 
-def python_template_dir() -> Optional[str]:
-    if os.getenv("PY_TEMPLATE_DIR"):
-        return os.path.join(os.environ["PY_TEMPLATE_DIR"], os.environ.get("GIT_REPO_TEMPLATE_DIR", ""))
+def python_template_dir(py_template_base_dir, py_template_subdir) -> Optional[str]:
+    if py_template_base_dir:
+        return os.path.join(py_template_base_dir, py_template_subdir or "")
     return None