Document and streamline procedure to generate process listings from o…

…peneo-processes repo

related to #390

docs/development.rst

@@ -366,3 +366,24 @@ For development, what you need to do is:
 
     # Now install the openeo code in editable mode.
     pip install -e .[dev]
+
+
+
+Update of generated files
+==========================
+
+Some parts of the openEO Python Client Library source code are
+generated/compiled from upstream sources (e.g. official openEO specifications).
+Because updates are not often required,
+it's just a semi-manual procedure (to run from the project root):
+
+.. code-block:: console
+
+    # Update the sub-repositories (like git submodules, but optional)
+    python specs/update-subrepos.py
+
+    # Update `openeo/processes.py` from specifications in openeo-processes repository
+    python openeo/internal/processes/generator.py  specs/openeo-processes specs/openeo-processes/proposals --output openeo/processes.py
+
+    # Update the openEO process mapping documentation page
+    python docs/process_mapping.py > docs/process_mapping.rst

docs/process_mapping.rst

@@ -2,7 +2,7 @@
 ..
     !Warning! This is an auto-generated file.
     Do not edit directly.
-    Generated from: ['process_mapping.py']
+    Generated from: ['docs/process_mapping.py']
 
 .. _openeo_process_mapping:
 
@@ -63,11 +63,11 @@ method or function in the openEO Python Client Library.
     *   - `filter_temporal <https://processes.openeo.org/#filter_temporal>`_
         - :py:meth:`DataCube.filter_temporal() <openeo.rest.datacube.DataCube.filter_temporal>`
     *   - `fit_class_random_forest <https://processes.openeo.org/#fit_class_random_forest>`_
-        - :py:meth:`DataCube.fit_class_random_forest() <openeo.rest.datacube.DataCube.fit_class_random_forest>`
+        - :py:meth:`VectorCube.fit_class_random_forest() <openeo.rest.vectorcube.VectorCube.fit_class_random_forest>`
     *   - `fit_curve <https://processes.openeo.org/#fit_curve>`_
         - :py:meth:`DataCube.fit_curve() <openeo.rest.datacube.DataCube.fit_curve>`
     *   - `fit_regr_random_forest <https://processes.openeo.org/#fit_regr_random_forest>`_
-        - :py:meth:`DataCube.fit_regr_random_forest() <openeo.rest.datacube.DataCube.fit_regr_random_forest>`
+        - :py:meth:`VectorCube.fit_regr_random_forest() <openeo.rest.vectorcube.VectorCube.fit_regr_random_forest>`
     *   - `flatten_dimensions <https://processes.openeo.org/#flatten_dimensions>`_
         - :py:meth:`DataCube.flatten_dimensions() <openeo.rest.datacube.DataCube.flatten_dimensions>`
     *   - `ge <https://processes.openeo.org/#ge>`_
@@ -143,4 +143,4 @@ method or function in the openEO Python Client Library.
     *   - `unflatten_dimension <https://processes.openeo.org/#unflatten_dimension>`_
         - :py:meth:`DataCube.unflatten_dimension() <openeo.rest.datacube.DataCube.unflatten_dimension>`
 
-:subscript:`(Table autogenerated on 2022-11-14)`
+:subscript:`(Table autogenerated on 2023-03-15)`

openeo/internal/processes/generator.py

@@ -1,4 +1,5 @@
 import argparse
+import datetime
 import keyword
 import sys
 import textwrap
@@ -207,13 +208,17 @@ def __gt__(self, other) -> 'ProcessBuilder':
             # Used command line arguments:
             #    {cli}
         """.format(cli=" ".join(argv))))
+    output.write(f"# Generated on {datetime.date.today().isoformat()}\n")
+
     output.write(oo_src)
-    output.write(fun_src)
+    output.write(fun_src.rstrip() + "\n")
 
 
 def main():
-    # Usage example (from project root, assuming the `openeo-process` repo is checked out as well):
-    #     python openeo/internal/processes/generator.py  ../openeo-processes  --output openeo/processes.py
+    # Usage example (from project root):
+    #   # Update subrepos (with process specs)
+    #   python specs/update-subrepos.py
+    #   python openeo/internal/processes/generator.py  specs/openeo-processes specs/openeo-processes/proposals --output openeo/processes.py
 
     argv = sys.argv
     arg_parser = argparse.ArgumentParser()

openeo/processes.py

@@ -2,7 +2,8 @@
 # Do not edit this file directly.
 # It is automatically generated.
 # Used command line arguments:
-#    openeo/internal/processes/generator.py ../openeo-processes/ ../openeo-processes/proposals/ --output openeo/processes.py
+#    openeo/internal/processes/generator.py specs/openeo-processes specs/openeo-processes/proposals --output openeo/processes.py
+# Generated on 2023-03-15
 
 import builtins
 from openeo.internal.processes.builder import ProcessBuilderBase, UNSET

specs/.gitignore

@@ -0,0 +1 @@
+openeo-processes/

specs/update-subrepos.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+"""
+Script to manage/update specification sub-repositories,
+in the style of git submodules, but without requiring
+them for all development situations
+because they are only necessary for a small subset of
+maintenance tasks.
+
+This script should be runnable without non-stdlib dependencies.
+"""
+
+# TODO: windows support?
+
+import collections
+import contextlib
+import logging
+import logging.config
+import os
+import subprocess
+import sys
+from pathlib import Path
+from typing import List
+
+
+_log = logging.getLogger(__name__)
+_ROOT_DIR = Path(__file__).parent
+
+SubRepo = collections.namedtuple("SubRepo", ["url", "rev", "path"])
+
+
+def main():
+    logging.basicConfig(
+        level=logging.INFO, format=">>> {message}", style="{", stream=sys.stdout
+    )
+    subrepos = [
+        SubRepo(
+            url="https://github.com/Open-EO/openeo-processes.git",
+            rev="ca9e31094b863233d88459b6cf2a37416bc90d4e",
+            path="openeo-processes",
+        )
+    ]
+
+    for subrepo in subrepos:
+        ensure_subrepo(subrepo)
+
+    run_command(["ls", "-al", _ROOT_DIR])
+
+
+def run_command(cmd: List[str], *, return_stdout: bool = False, **kwargs):
+    _log.info(f"Running {cmd} ({kwargs=})")
+    if not return_stdout:
+        return subprocess.check_call(args=cmd, **kwargs)
+    else:
+        return subprocess.check_output(args=cmd, **kwargs)
+
+
+@contextlib.contextmanager
+def in_cwd(path: Path):
+    orig_cwd = Path.cwd()
+    try:
+        _log.info(f"Changing working directory to {path}")
+        os.chdir(path)
+        yield
+    finally:
+        _log.info(f"Resetting working directory to {path}")
+        os.chdir(orig_cwd)
+
+
+def ensure_subrepo(subrepo: SubRepo):
+    path = _ROOT_DIR / subrepo.path
+    _log.info(f"Ensuring subrepo {subrepo}")
+    if not path.exists():
+        run_command(["git", "clone", subrepo.url, str(path)])
+        run_command(["git", "config", "advice.detachedHead", "false"], cwd=path)
+
+    if path.is_dir() and (path / ".git").is_dir():
+        _log.info(f"{path} already looks like a git repo")
+    else:
+        raise RuntimeError(f"{path} exists but does not look like a git repo")
+
+    # Checkout to desired revision
+    run_command(["git", "checkout", subrepo.rev], cwd=path)
+    run_command(["git", "log", "-1"], cwd=path)
+    run_command(["git", "submodule", "update", "--init", "--recursive"], cwd=path)
+
+    # Check that repo is not dirty
+    dirty = run_command(
+        ["git", "status", "--short", "--untracked-files=all", "--ignored"],
+        cwd=path,
+        return_stdout=True,
+    )
+    if dirty.strip():
+        raise RuntimeError(f"Repo {path} seems dirty: {dirty}")
+
+
+if __name__ == "__main__":
+    main()