From 0a8e824dd86203bef45213df01cbadd33d3cc324 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 4 Sep 2023 19:08:15 +0200 Subject: [PATCH] feat: auto-mount edx-platform python requirements These changes make to possible to run: tutor mounts add /path/to/my-xblock The xblock directory with then be auto-magically bind-mounted in the "openedx" image at build time, and the lms*/cms* containers at run time. This makes it effectively possible to work as a developer on edx-platform requirements. We take the opportunity to move some openedx-specific code to a dedicated module. Close https://github.com/openedx/wg-developer-experience/issues/177 --- ...64751_regis_mount_edx_platform_packages.md | 1 + docs/tutorials/edx-platform.rst | 130 +++++++++++++++++ docs/tutorials/index.rst | 1 + tutor/commands/compose.py | 32 ----- tutor/commands/images.py | 14 -- tutor/hooks/catalog.py | 21 +++ tutor/plugins/__init__.py | 2 +- tutor/plugins/openedx.py | 131 ++++++++++++++++++ tutor/templates/build/openedx/Dockerfile | 12 ++ 9 files changed, 297 insertions(+), 47 deletions(-) create mode 100644 changelog.d/20231009_164751_regis_mount_edx_platform_packages.md create mode 100644 docs/tutorials/edx-platform.rst create mode 100644 tutor/plugins/openedx.py diff --git a/changelog.d/20231009_164751_regis_mount_edx_platform_packages.md b/changelog.d/20231009_164751_regis_mount_edx_platform_packages.md new file mode 100644 index 00000000000..7fdfa364322 --- /dev/null +++ b/changelog.d/20231009_164751_regis_mount_edx_platform_packages.md @@ -0,0 +1 @@ +- [Feature] Make it easy to work on edx-platform Python requirements with `tutor mounts add /path/to/my/package`. (by @regisb) diff --git a/docs/tutorials/edx-platform.rst b/docs/tutorials/edx-platform.rst new file mode 100644 index 00000000000..480cdfa5657 --- /dev/null +++ b/docs/tutorials/edx-platform.rst @@ -0,0 +1,130 @@ +.. _edx_platform: + +Working on edx-platform as a developer +====================================== + +Tutor supports running in development with ``tutor dev`` commands. Developers frequently need to work on a fork of some repository. The question then becomes: how to make their changes available within the "openedx" Docker container? + +For instance, when troubleshooting an issue in `edx-platform `__, we would like to make some changes to a local fork of that repository, and then apply these changes immediately in the "lms" and the "cms" containers (but also "lms-worker", "cms-worker", etc.) + +Similarly, when developing a custom XBlock, we would like to hot-reload any change we make to the XBlock source code within the containers. + +Tutor provides a simple solution to these questions. In both cases, the solution takes the form of a ``tutor mounts add ...`` command. + +Working on the "edx-platform" repository +---------------------------------------- + +Download the code from the upstream repository:: + + cd /my/workspace/edx-plaform + git clone https://github.com/openedx/edx-platform . + +Check out the right version of the upstream repository. If you are working on the `current "zebulon" release `__ of Open edX, then you should checkout the corresponding branch:: + + # "zebulon" is an example. You should put the actual release name here. + # I.e: aspen, birch, cypress, etc. + git checkout open-release/zebulon.master + +On the other hand, if you are working on the Tutor :ref:`"nightly" ` branch then you should checkout the master branch:: + + git checkout master + +Then, mount the edx-platform repository with Tutor:: + + tutor mounts add /my/workspace/edx-plaform + +This command does a few "magical" things 🧙 behind the scenes: + +1. Mount the edx-platform repository in the image at build-time. This means that when you run ``tutor images build openedx``, your custom repository will be used instead of the upstream. In particular, any change you've made to the installed requirements, static assets, etc. will be taken into account. +2. Mount the edx-platform repository at run time. Thus, when you run ``tutor dev start``, any change you make to the edx-platform repository will be hot-reloaded. + +You can get a glimpse of how these auto-mounts work by running ``tutor mounts list``. It should output something similar to the following:: + + $ tutor mounts list + - name: /home/data/regis/projets/overhang/repos/edx/edx-platform + build_mounts: + - image: openedx + context: edx-platform + - image: openedx-dev + context: edx-platform + compose_mounts: + - service: lms + container_path: /openedx/edx-platform + - service: cms + container_path: /openedx/edx-platform + - service: lms-worker + container_path: /openedx/edx-platform + - service: cms-worker + container_path: /openedx/edx-platform + - service: lms-job + container_path: /openedx/edx-platform + - service: cms-job + container_path: /openedx/edx-platform + +Working on edx-platform Python dependencies +------------------------------------------- + +Quite often, developers don't want to work on edx-platform directly, but on a dependency of edx-platform. For instance: an XBlock. This works the same way as above. Let's take the example of the `"edx-ora2" `__ package, for open response assessments. First, clone the Python package:: + + cd /my/workspace/edx-ora2 + git clone https://github.com/openedx/edx-ora2 . + +Then, check out the right version of the package. This is the version that is indicated in the ``edx-platform/requirements/edx/base.txt``. Be careful that the version that is currently in use in your version of edx-platform is **not necessarily the latest version**:: + + git checkout + +Then, mount this repository:: + + tutor mounts add /my/workspace/edx-ora2 + +Verify that your repository is properly bind-mounted by running ``tutor mounts list``:: + + $ tutor mounts list + - name: /my/workspace/edx-ora2 + build_mounts: + - image: openedx + context: req-edx-ora2 + - image: openedx-dev + context: req-edx-ora2 + compose_mounts: + - service: lms + container_path: /openedx/requirements/edx-ora2 + - service: cms + container_path: /openedx/requirements/edx-ora2 + - service: lms-worker + container_path: /openedx/requirements/edx-ora2 + - service: cms-worker + container_path: /openedx/requirements/edx-ora2 + - service: lms-job + container_path: /openedx/requirements/edx-ora2 + - service: cms-job + container_path: /openedx/requirements/edx-ora2 + +It is quite possible that your package is not automatically recognized and bind-mounted by Tutor. In such a case, you will need to create a :ref:`Tutor plugin ` that implements the :py:data:`tutor.hooks.Filters.EDX_PLATFORM_PYTHON_PACKAGES` patch:: + + from tutor import hooks + hooks.Filters.EDX_PLATFORM_PYTHON_PACKAGES.add_item("my-package") + +After you implement and enable that plugin, ``tutor mounts list`` should display your directory among the bind-mounted directories. + +You should then re-build the "openedx" Docker image to pick up your changes:: + + tutor images build openedx-dev + +Then, whenever you run ``tutor dev start``, the "lms" and "cms" container should automatically hot-reload your changes. + +To push your changes in production, you should do the same with ``tutor local`` and the "openedx" image:: + + tutor images build openedx + tutor local start -d + +Do I have to re-build the "openedx" Docker image after every change? +-------------------------------------------------------------------- + +No, you don't. Re-building the "openedx" Docker image may take a while, and you don't want to run this command every time you make a change to your local repositories. Because your host directory is bind-mounted in the containers at runtime, your changes will be automatically applied to the container. If you run ``tutor dev`` commands, then your changes will be automatically picked up. + +If you run ``tutor local`` commands (for instance: when debugging a production instance) then your changes will *not* be automatically picked up. In such a case you should manually restart the containers:: + + tutor local restart lms cms lms-worker cms-worker + +Re-building the "openedx" image should only be necessary when you want to push your changes to a Docker registry, then pull them on a remote server. diff --git a/docs/tutorials/index.rst b/docs/tutorials/index.rst index d5128a3f3cc..cef5c070e53 100644 --- a/docs/tutorials/index.rst +++ b/docs/tutorials/index.rst @@ -9,6 +9,7 @@ Open edX customization plugin theming + edx-platform edx-platform-settings google-smtp nightly diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index 0f19e3a5e3d..ca44c1594a5 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -426,38 +426,6 @@ def dc_command( context.job_runner(config).docker_compose(command, *args) -@hooks.Filters.COMPOSE_MOUNTS.add() -def _mount_edx_platform( - volumes: list[tuple[str, str]], name: str -) -> list[tuple[str, str]]: - """ - When mounting edx-platform with `tutor mounts add /path/to/edx-platform`, - bind-mount the host repo in the lms/cms containers. - """ - if name == "edx-platform": - path = "/openedx/edx-platform" - volumes += [ - ("lms", path), - ("cms", path), - ("lms-worker", path), - ("cms-worker", path), - ("lms-job", path), - ("cms-job", path), - ] - return volumes - - -@hooks.Filters.APP_PUBLIC_HOSTS.add() -def _edx_platform_public_hosts( - hosts: list[str], context_name: t.Literal["local", "dev"] -) -> list[str]: - if context_name == "dev": - hosts += ["{{ LMS_HOST }}:8000", "{{ CMS_HOST }}:8001"] - else: - hosts += ["{{ LMS_HOST }}", "{{ CMS_HOST }}"] - return hosts - - hooks.Filters.ENV_TEMPLATE_VARIABLES.add_item(("iter_mounts", bindmount.iter_mounts)) diff --git a/tutor/commands/images.py b/tutor/commands/images.py index d21e2801cab..059c7bd377e 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -274,20 +274,6 @@ def get_image_build_contexts(config: Config) -> dict[str, list[tuple[str, str]]] return build_contexts -@hooks.Filters.IMAGES_BUILD_MOUNTS.add() -def _mount_edx_platform( - volumes: list[tuple[str, str]], path: str -) -> list[tuple[str, str]]: - """ - Automatically add an edx-platform repo from the host to the build context whenever - it is added to the `MOUNTS` setting. - """ - if os.path.basename(path) == "edx-platform": - volumes.append(("openedx", "edx-platform")) - volumes.append(("openedx-dev", "edx-platform")) - return volumes - - @click.command(short_help="Pull images from the Docker registry") @click.argument("image_names", metavar="image", type=PullImageNameParam(), nargs=-1) @click.pass_obj diff --git a/tutor/hooks/catalog.py b/tutor/hooks/catalog.py index 66ed97a4ca9..7163b4665cd 100644 --- a/tutor/hooks/catalog.py +++ b/tutor/hooks/catalog.py @@ -258,6 +258,26 @@ def your_filter_callback(some_data): #: arguments. Note that these arguments do not include the leading ``docker`` command. DOCKER_BUILD_COMMAND: Filter[list[str], []] = Filter() + #: List of python package names that can be potentially installed next to edx-platform. + #: Whenever a user runs: ``tutor mounts add /path/to/name`` "name" will be matched to + #: the regular expressions in this filter. If it matches, then the directory will be + #: automatically bind-mounted in the "openedx" Docker image at build time and run + #: time. They will be mounted in ``/openedx/requirements/``. Then, ``pip install + #: -e .`` will be run in this directory at build-time. And the same host directory + #: will be bind-mounted in that location at run time. This allows users to + #: transparently work on edx-platform dependencies. + #: + #: By default, xblocks and some common packages are already present in this + #: filter. Add your own edx-platform dependencies to this filter to make it easier for + #: users to work on edx-platform dependencies. + #: + #: See the list of all edx-platform base requirements here: + #: https://github.com/openedx/edx-platform/blob/master/requirements/edx/base.txt + #: + #: :parameter list[str] names: Add here simple names ("my-package") or regular + #: expressions. + EDX_PLATFORM_PYTHON_PACKAGES: Filter[list[str], []] = Filter() + #: List of patches that should be inserted in a given location of the templates. #: #: :parameter list[tuple[str, str]] patches: pairs of (name, content) tuples. Use this @@ -338,6 +358,7 @@ def your_filter_callback(some_data): #: - ``is_buildkit_enabled``: a boolean function that indicates whether BuildKit is available on the host. #: - ``iter_values_named``: a function to iterate on variables that start or end with a given string. #: - ``iter_mounts``: a function that yields compose-compatible bind-mounts for any given service. + #: - ``iter_mounted_edx_platform_python_requirements``: iterate on bind-mounted edx-platform python package names. #: - ``patch``: a function to incorporate extra content into a template. #: #: :parameter filters: list of (name, value) tuples. diff --git a/tutor/plugins/__init__.py b/tutor/plugins/__init__.py index c5e88d20b58..608421ade4f 100644 --- a/tutor/plugins/__init__.py +++ b/tutor/plugins/__init__.py @@ -10,7 +10,7 @@ from tutor.types import Config, get_typed # Import modules to trigger hook creation -from . import v0, v1 +from . import openedx, v0, v1 # Cache of plugin patches, for efficiency ENV_PATCHES_DICT: dict[str, list[str]] = {} diff --git a/tutor/plugins/openedx.py b/tutor/plugins/openedx.py new file mode 100644 index 00000000000..cc9237e75f6 --- /dev/null +++ b/tutor/plugins/openedx.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +import os +import re +import typing as t + +from tutor import bindmount +from tutor import hooks + + +@hooks.Filters.APP_PUBLIC_HOSTS.add() +def _edx_platform_public_hosts( + hosts: list[str], context_name: t.Literal["local", "dev"] +) -> list[str]: + if context_name == "dev": + hosts += ["{{ LMS_HOST }}:8000", "{{ CMS_HOST }}:8001"] + else: + hosts += ["{{ LMS_HOST }}", "{{ CMS_HOST }}"] + return hosts + + +@hooks.Filters.IMAGES_BUILD_MOUNTS.add() +def _mount_edx_platform_build( + volumes: list[tuple[str, str]], path: str +) -> list[tuple[str, str]]: + """ + Automatically add an edx-platform repo from the host to the build context whenever + it is added to the `MOUNTS` setting. + """ + if os.path.basename(path) == "edx-platform": + volumes += [ + ("openedx", "edx-platform"), + ("openedx-dev", "edx-platform"), + ] + return volumes + + +@hooks.Filters.COMPOSE_MOUNTS.add() +def _mount_edx_platform_compose( + volumes: list[tuple[str, str]], name: str +) -> list[tuple[str, str]]: + """ + When mounting edx-platform with `tutor mounts add /path/to/edx-platform`, + bind-mount the host repo in the lms/cms containers. + """ + if name == "edx-platform": + path = "/openedx/edx-platform" + volumes += [ + ("lms", path), + ("cms", path), + ("lms-worker", path), + ("cms-worker", path), + ("lms-job", path), + ("cms-job", path), + ] + return volumes + + +# Auto-magically bind-mount xblock directories and some common dependencies. +hooks.Filters.EDX_PLATFORM_PYTHON_PACKAGES.add_items( + [ + r".*[xX][bB]lock.*", + "edx-enterprise", + "edx-ora2", + "edx-search", + r"platform-plugin-.*", + ] +) + + +def iter_mounted_edx_platform_python_requirements(mounts: list[str]) -> t.Iterator[str]: + """ + Parse the list of mounted directories and yield the directory names that are for + edx-platform python requirements. Names are yielded in alphabetical order. + """ + names: set[str] = set() + for mount in mounts: + for _service, host_path, _container_path in bindmount.parse_mount(mount): + name = os.path.basename(host_path) + for regex in hooks.Filters.EDX_PLATFORM_PYTHON_PACKAGES.iterate(): + if re.match(regex, name): + names.add(name) + break + + yield from sorted(names) + + +hooks.Filters.ENV_TEMPLATE_VARIABLES.add_item( + ( + "iter_mounted_edx_platform_python_requirements", + iter_mounted_edx_platform_python_requirements, + ) +) + + +@hooks.Filters.IMAGES_BUILD_MOUNTS.add() +def _mount_edx_platform_python_requirements_build( + volumes: list[tuple[str, str]], path: str +) -> list[tuple[str, str]]: + """ + Automatically bind-mount edx-platform Python requirements at build-time. + """ + name = os.path.basename(path) + for regex in hooks.Filters.EDX_PLATFORM_PYTHON_PACKAGES.iterate(): + if re.match(regex, name): + volumes.append(("openedx", f"req-{name}")) + volumes.append(("openedx-dev", f"req-{name}")) + break + return volumes + + +@hooks.Filters.COMPOSE_MOUNTS.add() +def _mount_edx_platform_python_requirements_compose( + volumes: list[tuple[str, str]], name: str +) -> list[tuple[str, str]]: + """ + Automatically bind-mount edx-platform Python requirements at runtime. + """ + for regex in hooks.Filters.EDX_PLATFORM_PYTHON_PACKAGES.iterate(): + if re.match(regex, name): + # Bind-mount requirement + path = f"/openedx/requirements/{name}" + volumes += [ + ("lms", path), + ("cms", path), + ("lms-worker", path), + ("cms-worker", path), + ("lms-job", path), + ("cms-job", path), + ] + return volumes diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 7c41bf9f9c7..d5124d67ba3 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -61,6 +61,12 @@ RUN git config --global user.email "tutor@overhang.io" \ FROM scratch as edx-platform COPY --from=code /openedx/edx-platform / + +{# Create empty layers for all bind-mounted edx-platform python requirements #} +{% for req in iter_mounted_edx_platform_python_requirements(MOUNTS) %} +FROM scratch as req-{{ req }} +{% endfor %} + ###### Download extra locales to /openedx/locale/contrib/locale FROM minimal as locales ARG OPENEDX_I18N_VERSION={{ OPENEDX_COMMON_VERSION }} @@ -116,6 +122,12 @@ RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip, {% for extra_requirements in OPENEDX_EXTRA_PIP_REQUIREMENTS %}RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install '{{ extra_requirements }}' {% endfor %} +{% for req in iter_mounted_edx_platform_python_requirements(MOUNTS) %} +# Install local copy of {{ req }} +COPY --from=req-{{ req }} --chown=app:app / /openedx/requirements/{{ req }} +RUN pip install -e /openedx/requirements/{{ req }} +{% endfor %} + ###### Install nodejs with nodeenv in /openedx/nodeenv FROM python as nodejs-requirements ENV PATH /openedx/nodeenv/bin:/openedx/venv/bin:${PATH}