Skip to content

Commit

Permalink
Automate notebooks with papermill and nbstripout
Browse files Browse the repository at this point in the history
  • Loading branch information
@jacebrowning
jacebrowning committed May 23, 2019
1 parent ca16ce8 commit 64f6908
Show file tree
Hide file tree
Showing 9 changed files with 482 additions and 162 deletions.
11 changes: 10 additions & 1 deletion Makefile
View file
Expand Up @@ -24,6 +24,7 @@ watch: install ## Continuously run all CI tasks when files chanage

.PHONY: demo
demo: install
poetry run nbstripout notebooks/*.ipynb
poetry run jupyter notebook --notebook-dir=notebooks --browser=firefox

# SYSTEM DEPENDENCIES #########################################################
Expand Down Expand Up @@ -88,7 +89,7 @@ test-profile: install
MKDOCS_INDEX := site/index.html

.PHONY: docs
docs: mkdocs uml ## Generate documentation and UML
docs: mkdocs uml papermill ## Generate documentation and UML

.PHONY: mkdocs
mkdocs: install $(MKDOCS_INDEX)
Expand All @@ -114,6 +115,14 @@ docs/*.png: $(MODULES)
- mv -f classes_$(PACKAGE).png docs/classes.png
- mv -f packages_$(PACKAGE).png docs/packages.png

.PHONY: papermill
papermill: install
@ cd notebooks; for filename in *.ipynb; do \
poetry run papermill $$filename $$filename; \
done
git config filter.nbstripout.extrakeys 'metadata.papermill cell.metadata.papermill'
poetry run nbstripout --keep-output notebooks/*.ipynb

# RELEASE #####################################################################

DIST_FILES := dist/*.tar.gz dist/*.whl
Expand Down
79 changes: 55 additions & 24 deletions notebooks/file_inference.ipynb
View file
Expand Up @@ -2,15 +2,19 @@
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"metadata": {
"tags": []
},
"source": [
"# Simple"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stdout",
Expand All @@ -33,14 +37,16 @@
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO: datafiles.managers: Loading 'sample' object from 'files/sample.yml'\n"
"INFO: datafiles.mappers: Loading 'sample' object from 'files/sample.yml'\n"
]
},
{
Expand All @@ -49,27 +55,31 @@
"['Alice', 'Bob']"
]
},
"execution_count": 2,
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from datafiles import auto\n",
"\n",
"sample = auto('files/sample.yml')\n",
"\n",
"sample.names"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO: datafiles.managers: Saving 'sample' object to 'files/sample.yml'\n"
"INFO: datafiles.mappers: Saving 'sample' object to 'files/sample.yml'\n"
]
}
],
Expand All @@ -79,8 +89,10 @@
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stdout",
Expand All @@ -104,15 +116,19 @@
},
{
"cell_type": "markdown",
"metadata": {},
"metadata": {
"tags": []
},
"source": [
"# Complex"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stdout",
Expand Down Expand Up @@ -164,14 +180,16 @@
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO: datafiles.managers: Loading 'travis' object from 'files/travis.yml'\n"
"INFO: datafiles.mappers: Loading 'travis' object from 'files/travis.yml'\n"
]
}
],
Expand All @@ -183,14 +201,16 @@
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO: datafiles.managers: Saving 'travis' object to 'files/travis.yml'\n"
"INFO: datafiles.mappers: Saving 'travis' object to 'files/travis.yml'\n"
]
}
],
Expand All @@ -200,8 +220,10 @@
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stdout",
Expand Down Expand Up @@ -240,6 +262,15 @@
"\n",
"cat files/travis.yml"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": []
}
],
"metadata": {
Expand All @@ -258,7 +289,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.1"
"version": "3.7.2"
}
},
"nbformat": 4,
Expand Down
68 changes: 45 additions & 23 deletions notebooks/format_options.ipynb
View file
Expand Up @@ -2,8 +2,10 @@
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"%%sh\n",
Expand All @@ -13,8 +15,10 @@
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"from dataclasses import dataclass\n",
Expand All @@ -36,21 +40,25 @@
},
{
"cell_type": "markdown",
"metadata": {},
"metadata": {
"tags": []
},
"source": [
"# JSON"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO: datafiles.managers: Saving 'Sample' object to 'files/format_options.json'\n"
"INFO: datafiles.mappers: Saving 'Sample' object to 'files/format_options.json'\n"
]
}
],
Expand All @@ -66,8 +74,10 @@
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stdout",
Expand Down Expand Up @@ -101,21 +111,25 @@
},
{
"cell_type": "markdown",
"metadata": {},
"metadata": {
"tags": []
},
"source": [
"# TOML"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO: datafiles.managers: Saving 'Sample' object to 'files/format_options.toml'\n"
"INFO: datafiles.mappers: Saving 'Sample' object to 'files/format_options.toml'\n"
]
}
],
Expand All @@ -131,8 +145,10 @@
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stdout",
Expand Down Expand Up @@ -162,21 +178,25 @@
},
{
"cell_type": "markdown",
"metadata": {},
"metadata": {
"tags": []
},
"source": [
"# YAML"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO: datafiles.managers: Saving 'Sample' object to 'files/format_options.yml'\n"
"INFO: datafiles.mappers: Saving 'Sample' object to 'files/format_options.yml'\n"
]
}
],
Expand All @@ -192,8 +212,10 @@
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stdout",
Expand Down Expand Up @@ -234,7 +256,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.1"
"version": "3.7.2"
}
},
"nbformat": 4,
Expand Down

0 comments on commit 64f6908

Please sign in to comment.