Create default templates for MT endpoints

For new MT configurations, ensure there's a default user_endpoint configuration
template.  This provides a scaffolding for admins to modify, rather than
requiring a whole-cloth creation, with some low-hanging fruit defaults
(e.g., `max_blocks: 1`, `idle_heartbeats_soft`, etc.)

compute_endpoint/globus_compute_endpoint/endpoint/config/user_config_template.yaml

@@ -0,0 +1,44 @@
+# This is the default user-template provided with newly-configured Multi-Tenant
+# endpoints.  User endpoints generate a user-endpoint-specific configuration by
+# processing this YAML file as a Jinja template against user-provided
+# variables -- please modify this template to suit your site's requirements.
+#
+# For more information, please see the `user_endpoint_config` in Globus Compute
+# SDK's Executor.
+#
+# Some common options site-administrators may want to set:
+#  - address
+#  - provider (e.g., SlurmProvider, TorqueProvider, CobaltProvider, etc.)
+#  - account
+#  - scheduler_options
+#  - walltime
+#  - worker_init
+#
+# There are a number of example configurations available in the documentation:
+#    https://globus-compute.readthedocs.io/en/stable/endpoints.html#example-configurations
+
+engine:
+  type: HighThroughputEngine
+  max_workers_per_node: 1
+
+  provider:
+    type: LocalProvider
+
+    min_blocks: 0
+    max_blocks: 1
+    init_blocks: 1
+
+    endpoint_setup: {{ endpoint_setup|default() }}
+    endpoint_init: {{ endpoint_init|default() }}
+    worker_init: {{ worker_init|default() }}
+
+# Endpoints will be restarted when a user submits new tasks to the
+# web-services, so eagerly shut down if endpoint is idle.  At 30s/hb (default
+# value), 10 heartbeats is 300s.
+idle_heartbeats_soft: 10
+
+# If endpoint is *apparently* idle (e.g., outstanding tasks, but no movement)
+# for this many heartbeats, then shutdown anyway.  At 30s/hb (default value),
+# 5,760 heartbeats == "48 hours".  (Note that this value will be ignored if
+# idle_heartbeats_soft is 0 or not set.)
+idle_heartbeats_hard: 5760

compute_endpoint/globus_compute_endpoint/endpoint/config/user_environment.yaml

@@ -0,0 +1,19 @@
+# Use this YAML file to specify environment variables to inject into the user
+# endpoint processes.  Following standard process environment variables, any
+# variables specified here will be strings to the processes, and there is no
+# nesting -- this is a key-value description only.
+#
+# A couple of notes:
+#
+# - Values specified here will override any defaults (e.g., PATH)
+#
+# - Note that PATH is set to a sensible default, so typically won't need to
+#   be manually specified
+#
+# - Three variables cannot be set here: HOME, USER, and PWD.  These are set
+#   based on the user's GETENT(1) entry.
+#
+# Example:
+#
+#     PATH: /opt/bin:/other/dir/bin:/usr/bin
+#     SITE_SPECIFIC_VAR: some site specific value

compute_endpoint/globus_compute_endpoint/endpoint/config/utils.py

@@ -183,12 +183,12 @@ def _sanitize_user_opts(data):
 
 
 def render_config_user_template(endpoint_dir: pathlib.Path, user_opts: dict) -> str:
-    # Only load package when called by EP manager
-    import jinja2
+    import jinja2  # Only load package when called by EP manager
+    from globus_compute_endpoint.endpoint.endpoint import Endpoint
     from jinja2.sandbox import SandboxedEnvironment
 
     user_opts = _sanitize_user_opts(user_opts)
-    user_config_path = endpoint_dir / "config_user.yaml"
+    user_config_path = Endpoint.user_config_template_path(endpoint_dir)
 
     template_str = _read_config_yaml(user_config_path)
     environment = SandboxedEnvironment(undefined=jinja2.StrictUndefined)

compute_endpoint/globus_compute_endpoint/endpoint/endpoint.py

@@ -53,6 +53,14 @@ def __init__(self, debug=False):
     def _config_file_path(endpoint_dir: pathlib.Path) -> pathlib.Path:
         return endpoint_dir / "config.yaml"
 
+    @staticmethod
+    def user_config_template_path(endpoint_dir: pathlib.Path) -> pathlib.Path:
+        return endpoint_dir / "user_config_template.yaml"
+
+    @staticmethod
+    def _user_environment_path(endpoint_dir: pathlib.Path) -> pathlib.Path:
+        return endpoint_dir / "user_environment.yaml"
+
     @staticmethod
     def update_config_file(
         original_path: pathlib.Path,
@@ -81,11 +89,11 @@ def init_endpoint_dir(
     ):
         """Initialize a clean endpoint dir
 
-        :param endpoint_dir pathlib.Path Path to the endpoint configuration dir
-        :param endpoint_config str Path to a config file to be used instead
-         of the Globus Compute default config file
-        :param multi_tenant bool Whether the endpoint is a multi-user endpoint
-        :param display_name str A display name to use, if desired
+        :param endpoint_dir: Path to the endpoint configuration dir
+        :param endpoint_config: Path to a config file to be used instead of
+            the Globus Compute default config file
+        :param multi_tenant: Whether the endpoint is a multi-user endpoint
+        :param display_name: A display name to use, if desired
         """
         log.debug(f"Creating endpoint dir {endpoint_dir}")
         user_umask = os.umask(0o0077)
@@ -98,9 +106,9 @@ def init_endpoint_dir(
             endpoint_dir.mkdir(parents=True, exist_ok=True)
 
             config_target_path = Endpoint._config_file_path(endpoint_dir)
+            package_dir = pathlib.Path(__file__).resolve().parent
 
             if endpoint_config is None:
-                package_dir = pathlib.Path(__file__).resolve().parent
                 endpoint_config = package_dir / "config/default_config.yaml"
 
             Endpoint.update_config_file(
@@ -109,6 +117,25 @@ def init_endpoint_dir(
                 multi_tenant,
                 display_name,
             )
+
+            if multi_tenant:
+                # template must be readable by user-endpoint processes (see
+                # endpoint_manager.py)
+                world_readable = 0o0644 & ((0o0777 - user_umask) | 0o0444)
+                world_executable = 0o0711 & ((0o0777 - user_umask) | 0o0111)
+                endpoint_dir.chmod(world_executable)
+
+                src_user_tmpl_path = package_dir / "config/user_config_template.yaml"
+                src_user_env_path = package_dir / "config/user_environment.yaml"
+                dst_user_tmpl_path = Endpoint.user_config_template_path(endpoint_dir)
+                dst_user_env_path = Endpoint._user_environment_path(endpoint_dir)
+
+                shutil.copy(src_user_tmpl_path, dst_user_tmpl_path)
+                shutil.copy(src_user_env_path, dst_user_env_path)
+
+                dst_user_tmpl_path.chmod(world_readable)
+                dst_user_env_path.chmod(world_readable)
+
         finally:
             os.umask(user_umask)
 
@@ -131,14 +158,25 @@ def configure_endpoint(
         )
         config_path = Endpoint._config_file_path(conf_dir)
         if multi_tenant:
+            user_conf_tmpl_path = Endpoint.user_config_template_path(conf_dir)
+            user_env_path = Endpoint._user_environment_path(conf_dir)
+
             print(f"Created multi-tenant profile for endpoint named <{ep_name}>")
+            print(
+                f"\n\tConfiguration file: {config_path}\n"
+                f"\n\tUser endpoint configuration template: {user_conf_tmpl_path}"
+                f"\n\tUser endpoint environment variables: {user_env_path}"
+                "\n\nUse the `start` subcommand to run it:\n"
+                f"\n\t$ globus-compute-endpoint start {ep_name}"
+            )
+
         else:
             print(f"Created profile for endpoint named <{ep_name}>")
-        print(
-            f"\n\tConfiguration file: {config_path}\n"
-            "\nUse the `start` subcommand to run it:\n"
-            f"\n\t$ globus-compute-endpoint start {ep_name}"
-        )
+            print(
+                f"\n\tConfiguration file: {config_path}\n"
+                "\nUse the `start` subcommand to run it:\n"
+                f"\n\t$ globus-compute-endpoint start {ep_name}"
+            )
 
     @staticmethod
     def validate_endpoint_name(path_name: str) -> None:

compute_endpoint/tests/integration/endpoint/endpoint/test_endpoint_manager.py

@@ -16,6 +16,10 @@
 
 logger = logging.getLogger("mock_funcx")
 
+DEF_CONFIG_DIR = (
+    pathlib.Path(globus_compute_endpoint.endpoint.config.__file__).resolve().parent
+)
+
 
 def _fake_http_response(*, status: int = 200, method: str = "GET") -> requests.Response:
     req = requests.Request(method, "https://funcx.example.org/")
@@ -33,13 +37,9 @@ def test_setup_teardown(self, fs):
         config_dir = pathlib.Path(funcx_dir) / "mock_endpoint"
         assert not config_dir.exists()
         # pyfakefs will take care of newly created files, not existing config
-        def_config_path = (
-            pathlib.Path(globus_compute_endpoint.endpoint.config.__file__)
-            .resolve()
-            .parent
-            / "default_config.yaml"
-        )
-        fs.add_real_file(def_config_path)
+        fs.add_real_file(DEF_CONFIG_DIR / "default_config.yaml")
+        fs.add_real_file(DEF_CONFIG_DIR / "user_config_template.yaml")
+        fs.add_real_file(DEF_CONFIG_DIR / "user_environment.yaml")
 
         yield
 
@@ -62,7 +62,7 @@ def test_double_configure(self):
     def test_configure_multi_tenant_existing_config(self, mt):
         manager = Endpoint()
         config_dir = pathlib.Path("/some/path/mock_endpoint")
-        config_file = config_dir / "config.yaml"
+        config_file = Endpoint._config_file_path(config_dir)
         config_copy = str(config_dir.parent / "config2.yaml")
 
         # First, make an entry with multi_tenant

compute_endpoint/tests/unit/test_cli_behavior.py

@@ -16,6 +16,7 @@
 from globus_compute_endpoint.cli import app, init_config_dir
 from globus_compute_endpoint.endpoint.config import Config
 from globus_compute_endpoint.endpoint.config.utils import load_config_yaml
+from globus_compute_endpoint.endpoint.endpoint import Endpoint
 from pyfakefs import fake_filesystem as fakefs
 from pytest_mock import MockFixture
 
@@ -47,8 +48,8 @@ def make_endpoint_dir(mock_command_ensure):
     def func(name):
         ep_dir = mock_command_ensure.endpoint_config_dir / name
         ep_dir.mkdir(parents=True, exist_ok=True)
-        ep_config = ep_dir / "config.yaml"
-        ep_template = ep_dir / "config_user.yaml"
+        ep_config = Endpoint._config_file_path(ep_dir)
+        ep_template = Endpoint.user_config_template_path(ep_dir)
         ep_config.write_text(
             """
 display_name: null

compute_endpoint/tests/unit/test_endpoint_unit.py

@@ -19,7 +19,10 @@
 from globus_compute_endpoint.endpoint.config.default_config import (
     config as default_config,
 )
-from globus_compute_endpoint.endpoint.config.utils import serialize_config
+from globus_compute_endpoint.endpoint.config.utils import (
+    render_config_user_template,
+    serialize_config,
+)
 from globus_compute_endpoint.endpoint.endpoint import Endpoint
 
 _mock_base = "globus_compute_endpoint.endpoint.endpoint."
@@ -468,6 +471,28 @@ def test_endpoint_config_handles_umask_gracefully(tmp_path, umask):
     ep_dir.chmod(0o700)  # necessary for test to cleanup after itself
 
 
+def test_mt_endpoint_user_ep_yamls_world_readable(tmp_path):
+    ep_dir = tmp_path / "new_endpoint_dir"
+    Endpoint.init_endpoint_dir(ep_dir, multi_tenant=True)
+
+    user_tmpl_path = Endpoint.user_config_template_path(ep_dir)
+    user_env_path = Endpoint._user_environment_path(ep_dir)
+
+    assert user_env_path != user_tmpl_path, "Dev typo while developing"
+    for p in (user_tmpl_path, user_env_path):
+        assert p.exists()
+        assert p.stat().st_mode & 0o444 == 0o444, "Minimum world readable"
+    assert ep_dir.stat().st_mode & 0o111 == 0o111, "Minimum world executable"
+
+
+def test_mt_endpoint_user_ep_sensible_default(tmp_path):
+    ep_dir = tmp_path / "new_endpoint_dir"
+    Endpoint.init_endpoint_dir(ep_dir, multi_tenant=True)
+
+    # Doesn't crash; loads yaml, jinja template has defaults
+    render_config_user_template(ep_dir, {})
+
+
 def test_always_prints_endpoint_id_to_terminal(mocker, mock_ep_data):
     ep, ep_dir, log_to_console, no_color, ep_conf = mock_ep_data
     ep_id = str(uuid.uuid4())

compute_endpoint/tests/unit/test_endpointmanager_unit.py

@@ -20,6 +20,7 @@
 import yaml
 from globus_compute_endpoint.endpoint.config import Config
 from globus_compute_endpoint.endpoint.config.utils import render_config_user_template
+from globus_compute_endpoint.endpoint.endpoint import Endpoint
 from globus_compute_endpoint.endpoint.endpoint_manager import EndpointManager
 from globus_compute_endpoint.endpoint.utils import _redact_url_creds
 from globus_sdk import GlobusAPIError, NetworkError
@@ -42,7 +43,7 @@ def mock_conf():
 
 @pytest.fixture
 def user_conf_template(conf_dir):
-    template = conf_dir / "config_user.yaml"
+    template = Endpoint.user_config_template_path(conf_dir)
     template.write_text(
         """
 heartbeat_period: {{ heartbeat }}
@@ -949,7 +950,7 @@ def test_render_config_user_template(fs, data):
 
     ep_dir = pathlib.Path("my-ep")
     ep_dir.mkdir(parents=True, exist_ok=True)
-    template = ep_dir / "config_user.yaml"
+    template = Endpoint.user_config_template_path(ep_dir)
     template.write_text("heartbeat_period: {{ heartbeat }}")
 
     if is_valid:
@@ -965,7 +966,7 @@ def test_render_config_user_template(fs, data):
 def test_render_config_user_template_escape_strings(fs):
     ep_dir = pathlib.Path("my-ep")
     ep_dir.mkdir(parents=True, exist_ok=True)
-    template = ep_dir / "config_user.yaml"
+    template = Endpoint.user_config_template_path(ep_dir)
     template.write_text(
         """
 endpoint_setup: {{ setup }}
@@ -1017,7 +1018,7 @@ def test_render_config_user_template_option_types(fs, data):
 
     ep_dir = pathlib.Path("my-ep")
     ep_dir.mkdir(parents=True, exist_ok=True)
-    template = ep_dir / "config_user.yaml"
+    template = Endpoint.user_config_template_path(ep_dir)
     template.write_text("foo: {{ foo }}")
 
     user_opts = {"foo": val}
@@ -1043,7 +1044,7 @@ def test_render_config_user_template_sandbox(mocker: MockFixture, fs, data):
 
     ep_dir = pathlib.Path("my-ep")
     ep_dir.mkdir(parents=True, exist_ok=True)
-    template = ep_dir / "config_user.yaml"
+    template = Endpoint.user_config_template_path(ep_dir)
     template.write_text(f"foo: {jinja_op}")
 
     user_opts = {"foo": val}