initial commit of pipeline

openMetadataInitiative · Jun 29, 2023 · 300c842 · 300c842
1 parent 06af472
commit 300c842
Show file tree

Hide file tree

Showing 24 changed files with 374 additions and 306 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -0,0 +1,35 @@
+# MIT licensed
+name: openMINDS_documentation_build_pipeline
+
+on:
+  push:
+      branches:
+        - pipeline
+  workflow_dispatch: # This triggers the workflow when a webhook is received
+
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Checkout Repository
+      uses: actions/checkout@v3
+
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.11
+
+    - name: Checkout documentation branch
+      uses: actions/checkout@v3
+      with:
+        ref: main
+        path: /tmp/upload
+        token: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Run build
+      run: |
+        pip install -r requirements.txt
+        python build.py
+
diff --git a/.gitignore b/.gitignore
@@ -1,156 +1,3 @@
-
-# Created by https://www.toptal.com/developers/gitignore/api/python,jupyternotebooks
-# Edit at https://www.toptal.com/developers/gitignore?templates=python,jupyternotebooks
-
-### JupyterNotebooks ###
-# gitignore template for Jupyter Notebooks
-# website: http://jupyter.org/
-
-.ipynb_checkpoints
-*/.ipynb_checkpoints/*
-
-# IPython
-profile_default/
-ipython_config.py
-
-# Remove previous ipynb_checkpoints
-#   git rm -r .ipynb_checkpoints/
-
-### Python ###
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-cover/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-.pybuilder/
-target/
-
-# Jupyter Notebook
-
-# IPython
-
-# pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-
-# pytype static type analyzer
-.pytype/
-
-# Cython debug symbols
-cython_debug/
-
-# End of https://www.toptal.com/developers/gitignore/api/python,jupyternotebooks
+sources/**
+target/**
+.idea/**
diff --git a/build.py b/build.py
@@ -0,0 +1,37 @@
+import shutil
+
+from pipeline.schema import SchemaDocBuilder
+from pipeline.toctree import create_toc_tree_page
+from pipeline.utils import clone_sources, SchemaLoader, copy_static_structures, GitPusher
+
+print("***************************************")
+print(f"Triggering the generation of documentation for openMINDS")
+print("***************************************")
+
+# Step 1 - clone central repository in main branch to get the latest sources
+clone_sources()
+schema_loader = SchemaLoader()
+shutil.rmtree("target")
+
+git_pusher = GitPusher()
+
+for schema_version in schema_loader.get_schema_versions():
+
+    # Step 2 - building static structure
+    copy_static_structures(schema_version)
+
+    # Step 3 - find all involved schemas for the current version
+    schemas = schema_loader.find_schemas(schema_version)
+
+    for schema in schemas:
+        # Step 4 - build documentation for version specific schema
+        SchemaDocBuilder(schema, schema_loader.schemas_sources).build()
+
+    # Step 5 - building toctrees (please note that the implementation requires the schemas to be built already)
+    create_toc_tree_page(schema_version, ["specifications"])
+
+    # Step 6 - checkout version branch in relative "upload" directory (provided by GitHub pipeline), copy files and push
+    git_pusher.upload(schema_version)
+
+
+    # create_specifications(schema_version)
diff --git a/docs/_static/custom.css b/docs/_static/custom.css
diff --git a/docs/_static/custom.js b/docs/_static/custom.js
diff --git a/docs/_static/dark_openMINDS-logo.png b/docs/_static/dark_openMINDS-logo.png
diff --git a/docs/_static/light_openMINDS-logo.png b/docs/_static/light_openMINDS-logo.png
diff --git a/docs/conf.py b/docs/conf.py
diff --git a/docs/shared/getting_started.rst b/docs/shared/getting_started.rst
diff --git a/docs/shared/getting_started/matlab.rst b/docs/shared/getting_started/matlab.rst
diff --git a/docs/shared/getting_started/python.rst b/docs/shared/getting_started/python.rst
diff --git a/docs/shared/how_to_contribute.rst b/docs/shared/how_to_contribute.rst
diff --git a/docs/shared/how_to_contribute/code_of_conduct.rst b/docs/shared/how_to_contribute/code_of_conduct.rst
diff --git a/docs/shared/how_to_contribute/extensions.rst b/docs/shared/how_to_contribute/extensions.rst
diff --git a/docs/shared/how_to_contribute/instances.rst b/docs/shared/how_to_contribute/instances.rst
diff --git a/docs/shared/how_to_contribute/schemas.rst b/docs/shared/how_to_contribute/schemas.rst