Merge pull request #764 from mindsdb/jupyter_take_2

Revamp Notebooks, test docs

.github/workflows/doc_build.yml

@@ -5,6 +5,8 @@ on:
     branches:
       - staging
       - separate_doc_branch
+      - jupyter_actions
+
 jobs:
   doc_build:
     runs-on: ubuntu-latest
@@ -18,15 +20,22 @@ jobs:
       with:
         python-version: 3.9
 
-    - name: build docs
+    - name: install all dependencies
       run: |
         sudo apt install pandoc
         python -m pip install --upgrade pip
         pip install --no-cache-dir -e .
-        pip install install 'Sphinx==4.1.2' 'sphinx-autoapi==1.8.4' 'sphinx-autodoc-typehints==1.12.0' 'sphinx-code-include==1.1.1' 'sphinx-rtd-theme==0.5.2' 'sphinxcontrib-applehelp==1.0.2' 'sphinxcontrib-devhelp==1.0.2' 'sphinxcontrib-htmlhelp==2.0.0' 'sphinxcontrib-jsmath==1.0.1' 'sphinxcontrib-napoleon==0.7' 'sphinxcontrib-qthelp==1.0.3' 'sphinxcontrib-serializinghtml==1.1.5' autoapi nbsphinx myst_parser pandoc jupyter
+        pip install install 'Sphinx==4.1.2' 'sphinx-autoapi==1.8.4' 'sphinx-autodoc-typehints==1.12.0' 'sphinx-code-include==1.1.1' 'sphinx-rtd-theme==0.5.2' 'sphinxcontrib-applehelp==1.0.2' 'sphinxcontrib-devhelp==1.0.2' 'sphinxcontrib-htmlhelp==2.0.0' 'sphinxcontrib-jsmath==1.0.1' 'sphinxcontrib-napoleon==0.7' 'sphinxcontrib-qthelp==1.0.3' 'sphinxcontrib-serializinghtml==1.1.5' autoapi nbsphinx myst_parser pandoc jupyter matplotlib imblearn fsspec
+    - name: Re-run notebooks
+      run: |
+        find . -iname '*.ipynb' -exec jupyter nbconvert --to notebook --inplace --execute {} \;  > out.txt 2>&1
+        cat out.txt
+        cat out.txt | grep -zvqi exception && echo 'no errors detected' || exit
+        cat out.txt | grep -zvqi error && echo 'no errors detected' || exit
+    - name: Make the docs
+      run: |
         cd docssrc && make github
-    
-
+
     - name: Deploy to another branch
       uses: s0/git-publish-subdir-action@develop
       env:

.github/workflows/ligthtwood.yml

@@ -7,6 +7,7 @@ on:
       - stable
       - staging
 
+
 jobs:
   test:
     runs-on: ${{ matrix.os }}
@@ -49,6 +50,7 @@ jobs:
         python -m flake8 .
     - name: Test with unittest
       run: |
+        # Run all the "standard" tests
         python -m unittest discover tests
 
   deploy:

.gitignore

@@ -59,4 +59,11 @@ AI2.json
 
 assert.sh
 docssrc/build
-docssrc/build/*
+docssrc/build/*
+docssrc/source/tutorials/*.html
+docssrc/source/tutorials/*.pickle
+docssrc/source/tutorials/*.py
+docs
+docs/*
+*.zip
+docs/*

docssrc/source/tutorials/README.md

@@ -0,0 +1,18 @@
+## How to make a tutorial notebook?
+
+We use some of our tutorial notebooks as unit-tests to ensure that our pipeline is up-to-date, and to keep our examples relevant. 
+
+In order to preserve our (and the reader's) sanity these need to behave so that they can:
+1. Run via the CI tools
+2. Execute locally by a user
+
+To make things easier, the Lightwood team has proposed a general set of rules for tutorials:
+
+1. If you are using an external dataset, please ensure there is a URL that links to it (i.e.: load it in a dataframe using `pd.read_csv('{link}')`). Exceptions can be made for custom data types if the download dataset link is provided. We try to avoid hosting large datasets via Github, but please contact us if you believe it should be in our benchmarking suite.
+2. Show any **custom code within the notebook**. If you need to export it to a file (e.g. in order to load it as a lightwood module), use `%%writefile my_file.py` at the top of the jupyter codeblock, this will write the code into a file.
+3. Please do not save any extra files within the notebook (`.json` files may be ok); if your tutorial really requires saving extra files, please contact us and we can help.
+4. Please edit json-ai within the notebook as opposed to externally (i.e. generate a default, then make changes based on the key you need). You can show the difference between default and custom json-ai via a `print` statement.
+5. The notebook must not have any code metadata, otherwise github actions will fail to run them (in the json representation, grep for `kernel` and you will find the global `metadata` key, set that to `{}`)
+
+
+If your tutorial is anything more than a single `.ipynb` notebook and some accompanying .png or .jpg files, it may be rejected automatically. We would be more than happy to work with you to help adapt them to fit our automated notebooks.