Upgrade pydoc-markdown

.github/utils/generate_json_schema.py

@@ -8,7 +8,6 @@
 import haystack.nodes
 import pydantic.schema
 from fastapi.dependencies.utils import get_typed_signature
-from github import Github
 from pydantic import BaseConfig, BaseSettings, Required, SecretStr, create_model
 from pydantic.fields import ModelField
 from pydantic.schema import SkipField, TypeModelOrEnum, TypeModelSet, encode_default
@@ -217,6 +216,8 @@ def generate_json_schema():
 
 
 def main():
+    from github import Github
+
     generate_json_schema()
     logging.basicConfig(level=logging.INFO)
     settings = Settings()

.github/workflows/linux_ci.yml

@@ -3,12 +3,93 @@ name: Linux CI
 on:
   workflow_dispatch:
   push:
-    branches: [ master ]
-  pull_request:
-    branches: [ master ]
 
 jobs:
+
+  code_and_docs_updates:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token
+          #fetch-depth: 0 # otherwise, you will fail to push refs to dest repo
+
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - name: Install dependencies
+        run: |
+          pip install --upgrade pip
+          pip install black[jupyter] "pydoc-markdown>=4,<5" mkdocs jupytercontrib watchdog==1.0.2
+
+      # Apply black on the entire codebase
+      - name: Blacken
+        run: python3 -m black .
+
+      # Convert the Jupyter notebooks into markdown tutorials
+      - name: Generate Tutorials
+        run: |
+          cd docs/_src/tutorials/tutorials/
+          python3 convert_ipynb.py
+
+      # Generate markdown files from the docstrings with pydoc-markdown
+      - name: Generate Docstrings
+        run: |
+          set -e   # Fails on any error in the following loop
+          cd docs/_src/api/api/
+          for file in ../pydoc/* ; do
+            echo "Processing" $file
+            pydoc-markdown "$file"
+          done
+
+      # Generates the OpenAPI specs file to be used on the documentation website
+      - name: Generate OpenAPI Specs
+        run: |
+          pip install rest_api/
+          cd docs/_src/api/openapi/
+          python generate_openapi_specs.py
+
+      # Generates a new JSON schema for the pipeline YAML validation
+      - name: Generate JSON schema for pipelines
+        run: python ./.github/utils/generate_json_schema.py
+
+      # Commit the files to GitHub
+      - name: Commit files
+        run: |
+          git status
+          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+
+          echo "============= Documentation changes ============="
+          git add docs/_src/api/api/
+          git commit -m " -> Update Documentation" -a || echo "No documentation changes to commit"
+
+          echo "============= API Specs changes: ============="
+          git add docs/_src/api/openapi/
+          git commit -m "Update OpenAPI specs" -a || echo "No API specification changes to commit"
+
+          echo "============= JSON schema changes: ============="
+          git add json-schemas/
+          git commit -m "Update JSON schema" -a || echo "No JSON schema changes to commit"
+
+          echo "============= Code Style (Black): ============="
+          git add .
+          git commit -m "Apply Black" -a || echo "No code changes to commit"
+
+          echo "============= Summary: ============="
+          git status
+
+      - name: Push changes
+        uses: ad-m/github-push-action@master
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          branch: ${{ github.ref }}
+
   type-check:
+    needs: code_and_docs_updates
     runs-on: ubuntu-20.04
     steps:
 
@@ -23,7 +104,6 @@ jobs:
       - name: Test with mypy
         run: mypy haystack
 
-
   build-cache:
     needs: type-check
     runs-on: ubuntu-20.04