diff --git a/tutor/commands/jobs.py b/tutor/commands/jobs.py index 562e78d4549..f31cef08a44 100644 --- a/tutor/commands/jobs.py +++ b/tutor/commands/jobs.py @@ -42,15 +42,6 @@ def _add_core_init_tasks() -> None: ("mysql", env.read_core_template_file("jobs", "init", "mysql.sh")) ) with hooks.Contexts.app("lms").enter(): - hooks.Filters.CLI_DO_INIT_TASKS.add_item( - ( - "lms", - env.read_core_template_file("jobs", "init", "mounted-edx-platform.sh"), - ), - # If edx-platform is mounted, then we may need to perform some setup - # before other initialization scripts can be run. - priority=priorities.HIGH, - ) hooks.Filters.CLI_DO_INIT_TASKS.add_item( ("lms", env.read_core_template_file("jobs", "init", "lms.sh")) ) diff --git a/tutor/commands/mounts.py b/tutor/commands/mounts.py index c97075521b4..e43ff39a895 100644 --- a/tutor/commands/mounts.py +++ b/tutor/commands/mounts.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +from collections import defaultdict import click import yaml @@ -10,7 +11,13 @@ from tutor import exceptions, fmt, hooks from tutor.commands.config import save as config_save from tutor.commands.context import Context +from tutor.commands.images import ( + find_images_to_build, + find_remote_image_tags, + ImageNotFoundError, +) from tutor.commands.params import ConfigLoaderParam +from tutor.utils import execute as execute_shell class MountParamType(ConfigLoaderParam): @@ -73,8 +80,9 @@ def mounts_list(context: Context) -> None: @click.command(name="add") @click.argument("mounts", metavar="mount", type=click.Path(), nargs=-1) +@click.option("-p", "--populate", is_flag=True, help="Populate mount after adding it") @click.pass_context -def mounts_add(context: click.Context, mounts: list[str]) -> None: +def mounts_add(context: click.Context, mounts: list[str], populate: bool) -> None: """ Add a bind-mounted folder @@ -98,6 +106,8 @@ def mounts_add(context: click.Context, mounts: list[str]) -> None: explicit form. """ new_mounts = [] + implicit_mounts = [] + for mount in mounts: if not bindmount.parse_explicit_mount(mount): # Path is implicit: check that this path is valid @@ -105,11 +115,15 @@ def mounts_add(context: click.Context, mounts: list[str]) -> None: mount = os.path.abspath(os.path.expanduser(mount)) if not os.path.exists(mount): raise exceptions.TutorError(f"Path {mount} does not exist on the host") + implicit_mounts.append(mount) new_mounts.append(mount) fmt.echo_info(f"Adding bind-mount: {mount}") context.invoke(config_save, append_vars=[("MOUNTS", mount) for mount in new_mounts]) + if populate: + context.invoke(mounts_populate, mounts=implicit_mounts) + @click.command(name="remove") @click.argument("mounts", metavar="mount", type=MountParamType(), nargs=-1) @@ -133,6 +147,90 @@ def mounts_remove(context: click.Context, mounts: list[str]) -> None: ) +@click.command(name="populate", help="TODO document command") +@click.argument("mounts", metavar="mount", type=str, nargs=-1) +@click.pass_obj +def mounts_populate(context, mounts: str) -> None: + """ + TODO document command + """ + container_name = "tutor_mounts_populate_temp" # TODO: improve name? + config = tutor_config.load(context.root) + paths_to_copy_by_image: dict[str, tuple[str, str]] = defaultdict(list) + + if not mounts: + mounts = bindmount.get_mounts(config) + + for mount in mounts: + mount_items: list[tuple[str, str, str]] = bindmount.parse_mount(mount) + if not mount_items: + raise exceptions.TutorError(f"No mount for {mount}") + _service, mount_host_path, _container_path = mount_items[ + 0 + ] # [0] is arbitrary, as all host_paths should be equal + mount_expanded = os.path.abspath(os.path.expanduser(mount)) + mount_name = os.path.basename(mount_expanded) + for ( + image, + path_on_image, + path_in_host_mount, + ) in hooks.Filters.COMPOSE_MOUNT_POPULATORS.iterate(mount_name): + paths_to_copy_by_image[image].append( + (path_on_image, f"{mount_expanded}/{path_in_host_mount}") + ) + for image_name, paths_to_copy in paths_to_copy_by_image.items(): + image_tag = _get_image_tag(config, image_name) + execute_shell("docker", "rm", "-f", container_name) + execute_shell("docker", "create", "--name", container_name, image_tag) + for path_on_image, path_on_host in paths_to_copy: + fmt.echo_info(f"Populating {path_on_host} from {image_name}") + execute_shell("rm", "-rf", path_on_host) + execute_shell( + "docker", "cp", f"{container_name}:{path_on_image}", path_on_host + ) + execute_shell("docker", "rm", "-f", container_name) + + +def _get_image_tag(config: Config, image_name: str) -> str: + """ + Translate from a Tutor/plugin-defined image name to a specific Docker image tag. + + Searches for image_name in IMAGES_PULL then IMAGES_BUILD. + Raises ImageNotFoundError if no match. + """ + try: + return next( + find_remote_image_tags(config, hooks.Filters.IMAGES_PULL, image_name) + ) + except ImageNotFoundError: + _name, _path, tag, _args = next(find_images_to_build(config, image_name)) + return tag + + +@hooks.Filters.COMPOSE_MOUNT_POPULATORS.add() +def _populate_edx_platform_generated_dirs( + populators: list[tuple[str, str, str]], mount_name: str +) -> list[str]: + """ + TODO write docstring + """ + if mount_name == "edx-platform": + populators += [ + ("openedx-dev", f"/openedx/edx-platform/{generated_dir}", generated_dir) + for generated_dir in [ + "Open_edX.egg-info", + "node_modules", + "common/static/node_copies", + "lms/static/css", + "lms/static/certificates/css", + "cms/static/css", + "common/static/bundles", + ] + ] + return populators + + mounts_command.add_command(mounts_list) mounts_command.add_command(mounts_add) mounts_command.add_command(mounts_remove) +mounts_command.add_command(mounts_populate) diff --git a/tutor/hooks/catalog.py b/tutor/hooks/catalog.py index 3676affdb6e..8443faaba3e 100644 --- a/tutor/hooks/catalog.py +++ b/tutor/hooks/catalog.py @@ -222,6 +222,17 @@ def your_filter_callback(some_data): #: conditionally add mounts. COMPOSE_MOUNTS: Filter[list[tuple[str, str]], [str]] = Filter() + #: TODO describe + #: + #: TODO show example + #: + #: :parameter list[tuple[str, str, str]] populators: each item is a + #: ``(image_name, path_on_image, path_in_host_mount)`` tuple. TODO finish describing. + #: :parameter str name: basename of the host-mounted folder. In the example above, + #: this is "edx-platform". When implementing this filter you should check this name to + #: conditionally add populators for this folder. + COMPOSE_MOUNT_POPULATORS: Filter[list[tuple[str, str, str], [str]]] = Filter() + #: Declare new default configuration settings that don't necessarily have to be saved in the user #: ``config.yml`` file. Default settings may be overridden with ``tutor config save --set=...``, in which #: case they will automatically be added to ``config.yml``. diff --git a/tutor/templates/jobs/init/mounted-edx-platform.sh b/tutor/templates/jobs/init/mounted-edx-platform.sh deleted file mode 100644 index 9516654ff58..00000000000 --- a/tutor/templates/jobs/init/mounted-edx-platform.sh +++ /dev/null @@ -1,26 +0,0 @@ -# When a new local copy of edx-platform is bind-mounted, certain build -# artifacts from the openedx image's edx-platform directory are lost. -# We regenerate them here. - -if [ -f /openedx/edx-platform/bindmount-canary ] ; then - # If this file exists, then edx-platform has not been bind-mounted, - # so no build artifacts need to be regenerated. - echo "Using edx-platform from image (not bind-mount)." - echo "No extra setup is required." - exit -fi - -echo "Performing additional setup for bind-mounted edx-platform." -set -x # Echo out executed lines - -# Regenerate Open_edX.egg-info -pip install -e . - -# Regenerate node_modules -npm clean-install - -# Regenerate static assets. -openedx-assets build --env=dev - -set -x -echo "Done setting up bind-mounted edx-platform."