From c3bee35f1fdc884dd84a8126956793ea83ce4bad Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Sat, 13 May 2023 22:44:34 +0200 Subject: [PATCH 1/5] Add pre-commit config and pyproject.toml for configuring tools --- .pre-commit-config.yaml | 64 +++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 33 +++++++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 .pre-commit-config.yaml create mode 100644 pyproject.toml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..4c8ca80 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,64 @@ +# pre-commit is a tool to perform a predefined set of tasks manually and/or +# automatically before git commits are made. +# +# Config reference: https://pre-commit.com/#pre-commit-configyaml---top-level +# +# Common tasks +# +# - Run on all files: pre-commit run --all-files +# - Register git hooks: pre-commit install --install-hooks +# +repos: + # Autoformat: Python code, syntax patterns are modernized + - repo: https://github.com/asottile/pyupgrade + rev: v3.4.0 + hooks: + - id: pyupgrade + args: + - --py37-plus + + # Autoformat: Python code + - repo: https://github.com/PyCQA/autoflake + rev: v2.1.1 + hooks: + - id: autoflake + # args ref: https://github.com/PyCQA/autoflake#advanced-usage + args: + - --in-place + + # Autoformat: Python code + - repo: https://github.com/pycqa/isort + rev: 5.12.0 + hooks: + - id: isort + + # Autoformat: Python code + - repo: https://github.com/psf/black + rev: 23.3.0 + hooks: + - id: black + + # Autoformat: markdown, yaml + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v3.0.0-alpha.9-for-vscode + hooks: + - id: prettier + + # Misc... + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + # ref: https://github.com/pre-commit/pre-commit-hooks#hooks-available + hooks: + - id: end-of-file-fixer + - id: check-case-conflict + - id: check-executables-have-shebangs + + # Lint: Python code + - repo: https://github.com/pycqa/flake8 + rev: "6.0.0" + hooks: + - id: flake8 + +# pre-commit.ci config reference: https://pre-commit.ci/#configuration +ci: + autoupdate_schedule: monthly diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..849d5bd --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,33 @@ +# autoflake is used for autoformatting Python code +# +# ref: https://github.com/PyCQA/autoflake#readme +# +[tool.autoflake] +ignore-init-module-imports = true +remove-all-unused-imports = true +remove-duplicate-keys = true +remove-unused-variables = true + + +# isort is used for autoformatting Python code +# +# ref: https://pycqa.github.io/isort/ +# +[tool.isort] +profile = "black" + + +# black is used for autoformatting Python code +# +# ref: https://black.readthedocs.io/en/stable/ +# +[tool.black] +# target-version should be all supported versions, see +# https://github.com/psf/black/issues/751#issuecomment-473066811 +target_version = [ + "py37", + "py38", + "py39", + "py310", + "py311", +] \ No newline at end of file From acaf8d80d1488054b5a05dcdfd9a08bdc98a6a38 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Sat, 13 May 2023 22:46:06 +0200 Subject: [PATCH 2/5] Add flake8 config --- .flake8 | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..08e2c86 --- /dev/null +++ b/.flake8 @@ -0,0 +1,7 @@ +[flake8] +# Ignore style and complexity +# E: style errors +# W: style warnings +# C: complexity +# D: docstring warnings (unused pydocstyle extension) +ignore = E, C, W, D From a4bdbdd3d9546ebe2c7ef3f6a97b5a1e360a3af1 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Sat, 13 May 2023 22:48:04 +0200 Subject: [PATCH 3/5] Remove unused import --- systemdspawner/systemdspawner.py | 1 - 1 file changed, 1 deletion(-) diff --git a/systemdspawner/systemdspawner.py b/systemdspawner/systemdspawner.py index dbb4453..8251af9 100644 --- a/systemdspawner/systemdspawner.py +++ b/systemdspawner/systemdspawner.py @@ -1,6 +1,5 @@ import os import pwd -import subprocess from traitlets import Bool, Unicode, List, Dict import asyncio From ea5c4a5bf194e95233297c12d65ec6fbfd7ef76e Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Sat, 13 May 2023 23:09:41 +0200 Subject: [PATCH 4/5] Remove unused string format call --- tests/test_systemd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_systemd.py b/tests/test_systemd.py index 69dd427..599ee30 100644 --- a/tests/test_systemd.py +++ b/tests/test_systemd.py @@ -166,7 +166,7 @@ async def test_properties_string(): await systemd.start_transient_service( unit_name, ['/bin/bash'], - ['-c', 'pwd > pwd'.format(d)], + ['-c', 'pwd > pwd'], working_dir='/bind-test', properties={ 'BindPaths': '{}:/bind-test'.format(d) From 0ea3c2f67ee939fd195a90909b13b40b18c5d1d4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 13 May 2023 21:21:34 +0000 Subject: [PATCH 5/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- README.md | 76 +++++++++---------- jupyterhub.service | 2 +- pyproject.toml | 2 +- setup.py | 10 +-- systemdspawner/systemd.py | 54 ++++++-------- systemdspawner/systemdspawner.py | 121 +++++++++++++++++-------------- tests/test_systemd.py | 93 +++++++++++------------- 7 files changed, 181 insertions(+), 177 deletions(-) diff --git a/README.md b/README.md index e1af45b..23f5c8b 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,12 @@ **[License](#license)** | **[Resources](#resources)** -# systemdspawner # +# systemdspawner The **systemdspawner** enables JupyterHub to spawn single-user notebook servers using [systemd](https://www.freedesktop.org/wiki/Software/systemd/). -## Features ## +## Features If you want to use Linux Containers (Docker, rkt, etc) for isolation and security benefits, but don't want the headache and complexity of @@ -70,9 +70,9 @@ The following features are currently available: 10. Dynamically allocate users with Systemd's [dynamic users](http://0pointer.net/blog/dynamic-users-with-systemd.html) facility. Very useful in conjunction with [tmpauthenticator](https://github.com/jupyterhub/tmpauthenticator). -## Requirements ## +## Requirements -### Systemd ### +### Systemd Systemd Spawner requires you to use a Linux Distro that ships with at least systemd v211. The security related features require systemd v228 or v227. We recommend running @@ -83,21 +83,21 @@ $ systemctl --version | head -1 systemd 231 ``` -### Kernel Configuration ### +### Kernel Configuration Certain kernel options need to be enabled for the CPU / Memory limiting features to work. If these are not enabled, CPU / Memory limiting will just fail silently. You can check if your kernel supports these features by running the [`check-kernel.bash`](check-kernel.bash) script. -### Root access ### +### Root access Currently, JupyterHub must be run as root to use Systemd Spawner. `systemd-run` needs to be run as root to be able to set memory & cpu limits. Simple sudo rules do not help, since unrestricted access to `systemd-run` is equivalent to root. We will explore hardening approaches soon. -### Local Users ### +### Local Users If running with `c.SystemdSpawner.dynamic_users = False` (the default), each user's server is spawned to run as a local unix user account. Hence this spawner @@ -109,28 +109,27 @@ are required. Systemd will automatically create dynamic users as required. See [this blog post](http://0pointer.net/blog/dynamic-users-with-systemd.html) for details. -### Linux Distro compatibility ## +### Linux Distro compatibility -#### Ubuntu 16.04 LTS ### +#### Ubuntu 16.04 LTS We recommend running this with systemd spawner. The default kernel has all the features we need, and a recent enough version of systemd to give us all the features. -#### Debian Jessie #### +#### Debian Jessie The systemd version that ships by default with Jessie doesn't provide all the features we need, and the default kernel doesn't ship with the features we need. However, if you [enable jessie-backports](https://backports.debian.org/Instructions/) you can install a new enough version of systemd and linux kernel to get it to work fine. -#### Centos 7 #### +#### Centos 7 The kernel has all the features we need, but the version of systemd (219) is too old for the security related features of systemdspawner. However, basic spawning, memory & cpu limiting will work. - -## Installation ## +## Installation You can install it from PyPI with: @@ -149,8 +148,7 @@ Note that to confirm systemdspawner has been installed in the correct jupyterhub environment, a newly generated config file should list `systemdspawner` as one of the available spawner classes in the comments above the configuration line. - -## Configuration ## +## Configuration Lots of configuration options for you to choose! You should put all of these in your `jupyterhub_config.py` file: @@ -170,11 +168,11 @@ in your `jupyterhub_config.py` file: - **[`readwrite_paths`](#readwrite_paths)** - **[`dynamic_users`](#dynamic_users)** -### `mem_limit` ### +### `mem_limit` Specifies the maximum memory that can be used by each individual user. It can be specified as an absolute byte value. You can use the suffixes `K`, `M`, `G` or `T` to -mean Kilobyte, Megabyte, Gigabyte or Terabyte respectively. Setting it to `None` disables +mean Kilobyte, Megabyte, Gigabyte or Terabyte respectively. Setting it to `None` disables memory limits. Even if you want individual users to use as much memory as possible, it is still good @@ -190,7 +188,7 @@ Defaults to `None`, which provides no memory limits. This info is exposed to the single-user server as the environment variable `MEM_LIMIT` as integer bytes. -### `cpu_limit` ### +### `cpu_limit` A float representing the total CPU-cores each user can use. `1` represents one full CPU, `4` represents 4 full CPUs, `0.5` represents half of one CPU, etc. @@ -211,7 +209,7 @@ Note: there is [a bug](https://github.com/systemd/systemd/issues/3851) in systemd v231 which prevents the CPU limit from being set to a value greater than 100%. -#### CPU fairness #### +#### CPU fairness Completely unrelated to `cpu_limit` is the concept of CPU fairness - that each user should have equal access to all the CPUs in the absense of limits. This @@ -228,7 +226,7 @@ This works out perfect for most cases, since this allows users to burst up and use all CPU when nobody else is using CPU & forces them to automatically yield when other users want to use the CPU. -### `user_workingdir` ### +### `user_workingdir` The directory to spawn each user's notebook server in. This directory is what users see when they open their notebooks servers. Usually this is the user's home directory. @@ -242,7 +240,7 @@ c.SystemdSpawner.user_workingdir = '/home/{USERNAME}' Defaults to the home directory of the user. Not respected if `dynamic_users` is true. -### `username_template` ### +### `username_template` Template for unix username each user should be spawned as. @@ -257,7 +255,7 @@ c.SystemdSpawner.username_template = 'jupyter-{USERNAME}' Not respected if `dynamic_users` is set to True -### `default_shell` ### +### `default_shell` The default shell to use for the terminal in the notebook. Sets the `SHELL` environment variable to this. @@ -265,10 +263,11 @@ variable to this. ```python c.SystemdSpawner.default_shell = '/bin/bash' ``` + Defaults to whatever the value of the `SHELL` environment variable is in the JupyterHub process, or `/bin/bash` if `SHELL` isn't set. -### `extra_paths` ### +### `extra_paths` List of paths that should be prepended to the `PATH` environment variable for the spawned notebook server. This is easier than setting the `env` property, since you want to @@ -284,7 +283,7 @@ appropriate values for the user being spawned. Defaults to `[]` which doesn't add any extra paths to `PATH` -### `unit_name_template` ### +### `unit_name_template` Template to form the Systemd Service unit name for each user notebook server. This allows differentiating between multiple jupyterhubs with Systemd Spawner on the same @@ -299,11 +298,14 @@ appropriate values for the user being spawned. Defaults to `jupyter-{USERNAME}-singleuser` -### `unit_extra_properties` ### +### `unit_extra_properties` + Dict of key-value pairs used to add arbitrary properties to the spawned Jupyerhub units. + ```python c.SystemdSpawner.unit_extra_properties = {'LimitNOFILE': '16384'} ``` + Read `man systemd-run` for details on per-unit properties available in transient units. `{USERNAME}` and `{USERID}` in each parameter value will be expanded to the @@ -311,7 +313,7 @@ appropriate values for the user being spawned. Defaults to `{}` which doesn't add any extra properties to the transient scope. -### `isolate_tmp` ### +### `isolate_tmp` Setting this to true provides a separate, private `/tmp` for each user. This is very useful to protect against accidental leakage of otherwise private information - it is @@ -327,7 +329,7 @@ Defaults to false. This requires systemd version > 227. If you enable this in earlier versions, spawning will fail. -### `isolate_devices` ### +### `isolate_devices` Setting this to true provides a separate, private `/dev` for each user. This prevents the user from directly accessing hardware devices, which could be a potential source of @@ -343,7 +345,7 @@ Defaults to false. This requires systemd version > 227. If you enable this in earlier versions, spawning will fail. -### `disable_user_sudo` ### +### `disable_user_sudo` Setting this to true prevents users from being able to use `sudo` (or any other means) to become other users (including root). This helps contain damage from a compromise of a user's @@ -359,7 +361,7 @@ Defaults to false. This requires systemd version > 228. If you enable this in earlier versions, spawning will fail. -### `readonly_paths` ### +### `readonly_paths` List of filesystem paths that should be mounted readonly for the users' notebook server. This will override any filesystem permissions that might exist. Subpaths of paths that are mounted @@ -379,11 +381,11 @@ Defaults to `None` which disables this feature. This requires systemd version > 228. If you enable this in earlier versions, spawning will fail. It can also contain only directories (not files) until systemd version 231. -### `readwrite_paths` ### +### `readwrite_paths` List of filesystem paths that should be mounted readwrite for the users' notebook server. This only makes sense if `readonly_paths` is used to make some paths readonly - this can then be -used to make specific paths readwrite. This does *not* override filesystem permissions - the +used to make specific paths readwrite. This does _not_ override filesystem permissions - the user needs to have appropriate rights to write to these paths. ```python @@ -398,7 +400,7 @@ Defaults to `None` which disables this feature. This requires systemd version > 228. If you enable this in earlier versions, spawning will fail. It can also contain only directories (not files) until systemd version 231. -### `dynamic_users` ### +### `dynamic_users` Allocate system users dynamically for each user. @@ -412,10 +414,10 @@ information. Requires systemd 235. -### `slice` ### +### `slice` -Run the spawned notebook in a given systemd slice. This allows aggregate configuration that -will apply to all the units that are launched. This can be used (for example) to control +Run the spawned notebook in a given systemd slice. This allows aggregate configuration that +will apply to all the units that are launched. This can be used (for example) to control the total amount of memory that all of the notebook users can use. See https://samthursfield.wordpress.com/2015/05/07/running-firefox-in-a-cgroup-using-systemd/ for @@ -423,12 +425,12 @@ an example of how this could look. For detailed configuration see the [manpage](http://man7.org/linux/man-pages/man5/systemd.slice.5.html) -## Getting help ## +## Getting help We encourage you to ask questions on the [mailing list](https://groups.google.com/forum/#!forum/jupyter). You can also participate in development discussions or get live help on [Gitter](https://gitter.im/jupyterhub/jupyterhub). -## License ## +## License We use a shared copyright model that enables all contributors to maintain the copyright on their contributions. diff --git a/jupyterhub.service b/jupyterhub.service index 9bb1640..d1125b7 100644 --- a/jupyterhub.service +++ b/jupyterhub.service @@ -5,4 +5,4 @@ ReadWriteDirectories=/var/lib/jupyterhub /var/log/ /proc/self/ WorkingDirectory=/var/local/lib/jupyterhub CapabilityBoundingSet=CAP_NET_BIND_SERVICE CAP_AUDIT_WRITE CAP_SETGID CAP_SETUID PrivateDevices=yes -PrivateTmp=yes \ No newline at end of file +PrivateTmp=yes diff --git a/pyproject.toml b/pyproject.toml index 849d5bd..d3f0321 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,4 +30,4 @@ target_version = [ "py39", "py310", "py311", -] \ No newline at end of file +] diff --git a/setup.py b/setup.py index 69eb851..8a658e7 100644 --- a/setup.py +++ b/setup.py @@ -9,11 +9,11 @@ description="JupyterHub Spawner using systemd for resource isolation", long_description=long_description, long_description_content_type="text/markdown", - url='https://github.com/jupyterhub/systemdspawner', - author='Yuvi Panda', - author_email='yuvipanda@gmail.com', - license='3 Clause BSD', - packages=['systemdspawner'], + url="https://github.com/jupyterhub/systemdspawner", + author="Yuvi Panda", + author_email="yuvipanda@gmail.com", + license="3 Clause BSD", + packages=["systemdspawner"], entry_points={ "jupyterhub.spawners": [ "systemd = systemdspawner:SystemdSpawner", diff --git a/systemdspawner/systemd.py b/systemdspawner/systemd.py index 8505630..4c7ed88 100644 --- a/systemdspawner/systemd.py +++ b/systemdspawner/systemd.py @@ -16,6 +16,7 @@ RUN_ROOT = "/run" + def ensure_environment_directory(environment_file_directory): """Ensure directory for environment files exists and is private""" # ensure directory exists @@ -78,8 +79,9 @@ async def start_transient_service( """ run_cmd = [ - 'systemd-run', - '--unit', unit_name, + "systemd-run", + "--unit", + unit_name, ] if properties is None: @@ -89,8 +91,8 @@ async def start_transient_service( # Set default policy so OOM only terminate the offending kernel and let the user session survive. # Can be overridden in unit_extra_properties. - properties.setdefault('OOMPolicy', 'continue') - + properties.setdefault("OOMPolicy", "continue") + # ensure there is a runtime directory where we can put our env file # If already set, can be space-separated list of paths runtime_directories = properties.setdefault("RuntimeDirectory", unit_name).split() @@ -107,10 +109,10 @@ async def start_transient_service( if properties: for key, value in properties.items(): if isinstance(value, list): - run_cmd += ['--property={}={}'.format(key, v) for v in value] + run_cmd += [f"--property={key}={v}" for v in value] else: # A string! - run_cmd.append('--property={}={}'.format(key, value)) + run_cmd.append(f"--property={key}={value}") if environment_variables: environment_file = make_environment_file( @@ -120,25 +122,25 @@ async def start_transient_service( # Explicitly check if uid / gid are not None, since 0 is valid value for both if uid is not None: - run_cmd += ['--uid', str(uid)] + run_cmd += ["--uid", str(uid)] if gid is not None: - run_cmd += ['--gid', str(gid)] + run_cmd += ["--gid", str(gid)] if slice is not None: - run_cmd += ['--slice={}'.format(slice)] + run_cmd += [f"--slice={slice}"] # We unfortunately have to resort to doing cd with bash, since WorkingDirectory property # of systemd units can't be set for transient units via systemd-run until systemd v227. # Centos 7 has systemd 219, and will probably never upgrade - so we need to support them. run_cmd += [ - '/bin/bash', - '-c', + "/bin/bash", + "-c", "cd {wd} && exec {cmd} {args}".format( wd=shlex.quote(working_dir), - cmd=' '.join([shlex.quote(c) for c in cmd]), - args=' '.join([shlex.quote(a) for a in args]) - ) + cmd=" ".join([shlex.quote(c) for c in cmd]), + args=" ".join([shlex.quote(a) for a in args]), + ), ] proc = await asyncio.create_subprocess_exec(*run_cmd) @@ -151,11 +153,11 @@ async def service_running(unit_name): Return true if service with given name is running (active). """ proc = await asyncio.create_subprocess_exec( - 'systemctl', - 'is-active', + "systemctl", + "is-active", unit_name, # hide stdout, but don't capture stderr at all - stdout=asyncio.subprocess.DEVNULL + stdout=asyncio.subprocess.DEVNULL, ) ret = await proc.wait() @@ -167,11 +169,11 @@ async def service_failed(unit_name): Return true if service with given name is in a failed state. """ proc = await asyncio.create_subprocess_exec( - 'systemctl', - 'is-failed', + "systemctl", + "is-failed", unit_name, # hide stdout, but don't capture stderr at all - stdout=asyncio.subprocess.DEVNULL + stdout=asyncio.subprocess.DEVNULL, ) ret = await proc.wait() @@ -184,11 +186,7 @@ async def stop_service(unit_name): Throws CalledProcessError if stopping fails """ - proc = await asyncio.create_subprocess_exec( - 'systemctl', - 'stop', - unit_name - ) + proc = await asyncio.create_subprocess_exec("systemctl", "stop", unit_name) await proc.wait() @@ -198,9 +196,5 @@ async def reset_service(unit_name): Throws CalledProcessError if resetting fails """ - proc = await asyncio.create_subprocess_exec( - 'systemctl', - 'reset-failed', - unit_name - ) + proc = await asyncio.create_subprocess_exec("systemctl", "reset-failed", unit_name) await proc.wait() diff --git a/systemdspawner/systemdspawner.py b/systemdspawner/systemdspawner.py index 8251af9..33b00e7 100644 --- a/systemdspawner/systemdspawner.py +++ b/systemdspawner/systemdspawner.py @@ -1,12 +1,12 @@ +import asyncio import os import pwd -from traitlets import Bool, Unicode, List, Dict -import asyncio - -from systemdspawner import systemd from jupyterhub.spawner import Spawner from jupyterhub.utils import random_port +from traitlets import Bool, Dict, List, Unicode + +from systemdspawner import systemd class SystemdSpawner(Spawner): @@ -21,11 +21,11 @@ class SystemdSpawner(Spawner): Defaults to the home directory of the user. Not respected if dynamic_users is set to True. - """ + """, ).tag(config=True) username_template = Unicode( - '{USERNAME}', + "{USERNAME}", help=""" Template for unix username each user should be spawned as. @@ -34,12 +34,12 @@ class SystemdSpawner(Spawner): This user should already exist in the system. Not respected if dynamic_users is set to True - """ + """, ).tag(config=True) default_shell = Unicode( - os.environ.get('SHELL', '/bin/bash'), - help='Default shell for users on the notebook terminal' + os.environ.get("SHELL", "/bin/bash"), + help="Default shell for users on the notebook terminal", ).tag(config=True) extra_paths = List( @@ -52,12 +52,12 @@ class SystemdSpawner(Spawner): ).tag(config=True) unit_name_template = Unicode( - 'jupyter-{USERNAME}-singleuser', + "jupyter-{USERNAME}-singleuser", help=""" Template to use to make the systemd service names. {USERNAME} and {USERID} are expanded} - """ + """, ).tag(config=True) # FIXME: Do not allow enabling this for systemd versions < 227, @@ -66,14 +66,14 @@ class SystemdSpawner(Spawner): False, help=""" Give each notebook user their own /tmp, isolated from the system & each other - """ + """, ).tag(config=True) isolate_devices = Bool( False, help=""" Give each notebook user their own /dev, with a very limited set of devices mounted - """ + """, ).tag(config=True) disable_user_sudo = Bool( @@ -116,7 +116,7 @@ class SystemdSpawner(Spawner): Used to add arbitrary properties for spawned Jupyter units. Read `man systemd-run` for details on per-unit properties available in transient units. - """ + """, ).tag(config=True) dynamic_users = Bool( @@ -133,7 +133,7 @@ class SystemdSpawner(Spawner): information. Requires systemd 235. - """ + """, ).tag(config=True) slice = Unicode( @@ -143,7 +143,7 @@ class SystemdSpawner(Spawner): Ensure that all users that are created are run within a given slice. This allow global configuration of the maximum resources that all users collectively can use by creating a a slice beforehand. - """ + """, ).tag(config=True) def __init__(self, *args, **kwargs): @@ -151,7 +151,9 @@ def __init__(self, *args, **kwargs): # All traitlets configurables are configured by now self.unit_name = self._expand_user_vars(self.unit_name_template) - self.log.debug('user:%s Initialized spawner with unit %s', self.user.name, self.unit_name) + self.log.debug( + "user:%s Initialized spawner with unit %s", self.user.name, self.unit_name + ) def _expand_user_vars(self, string): """ @@ -161,10 +163,7 @@ def _expand_user_vars(self, string): {USERNAME} -> Name of the user {USERID} -> UserID """ - return string.format( - USERNAME=self.user.name, - USERID=self.user.id - ) + return string.format(USERNAME=self.user.name, USERID=self.user.id) def get_state(self): """ @@ -178,7 +177,7 @@ def get_state(self): saved no state, so this helps with that too! """ state = super().get_state() - state['unit_name'] = self.unit_name + state["unit_name"] = self.unit_name return state def load_state(self, state): @@ -192,28 +191,46 @@ def load_state(self, state): JupyterHub before 0.7 also assumed your notebook was dead if it saved no state, so this helps with that too! """ - if 'unit_name' in state: - self.unit_name = state['unit_name'] + if "unit_name" in state: + self.unit_name = state["unit_name"] async def start(self): self.port = random_port() - self.log.debug('user:%s Using port %s to start spawning user server', self.user.name, self.port) + self.log.debug( + "user:%s Using port %s to start spawning user server", + self.user.name, + self.port, + ) # If there's a unit with this name running already. This means a bug in # JupyterHub, a remnant from a previous install or a failed service start # from earlier. Regardless, we kill it and start ours in its place. # FIXME: Carefully look at this when doing a security sweep. if await systemd.service_running(self.unit_name): - self.log.info('user:%s Unit %s already exists but not known to JupyterHub. Killing', self.user.name, self.unit_name) + self.log.info( + "user:%s Unit %s already exists but not known to JupyterHub. Killing", + self.user.name, + self.unit_name, + ) await systemd.stop_service(self.unit_name) if await systemd.service_running(self.unit_name): - self.log.error('user:%s Could not stop already existing unit %s', self.user.name, self.unit_name) - raise Exception('Could not stop already existing unit {}'.format(self.unit_name)) + self.log.error( + "user:%s Could not stop already existing unit %s", + self.user.name, + self.unit_name, + ) + raise Exception( + f"Could not stop already existing unit {self.unit_name}" + ) # If there's a unit with this name already but sitting in a failed state. # Does a reset of the state before trying to start it up again. if await systemd.service_failed(self.unit_name): - self.log.info('user:%s Unit %s in a failed state. Resetting state.', self.user.name, self.unit_name) + self.log.info( + "user:%s Unit %s in a failed state. Resetting state.", + self.user.name, + self.unit_name, + ) await systemd.reset_service(self.unit_name) env = self.get_env() @@ -221,13 +238,13 @@ async def start(self): properties = {} if self.dynamic_users: - properties['DynamicUser'] = 'yes' - properties['StateDirectory'] = self._expand_user_vars('{USERNAME}') + properties["DynamicUser"] = "yes" + properties["StateDirectory"] = self._expand_user_vars("{USERNAME}") # HOME is not set by default otherwise - env['HOME'] = self._expand_user_vars('/var/lib/{USERNAME}') + env["HOME"] = self._expand_user_vars("/var/lib/{USERNAME}") # Set working directory to $HOME too - working_dir = env['HOME'] + working_dir = env["HOME"] # Set uid, gid = None so we don't set them uid = gid = None else: @@ -235,7 +252,7 @@ async def start(self): unix_username = self._expand_user_vars(self.username_template) pwnam = pwd.getpwnam(unix_username) except KeyError: - self.log.exception('No user named {} found in the system'.format(unix_username)) + self.log.exception(f"No user named {unix_username} found in the system") raise uid = pwnam.pw_uid gid = pwnam.pw_gid @@ -245,46 +262,44 @@ async def start(self): working_dir = self._expand_user_vars(self.user_workingdir) if self.isolate_tmp: - properties['PrivateTmp'] = 'yes' + properties["PrivateTmp"] = "yes" if self.isolate_devices: - properties['PrivateDevices'] = 'yes' + properties["PrivateDevices"] = "yes" if self.extra_paths: - env['PATH'] = '{extrapath}:{curpath}'.format( - curpath=env['PATH'], - extrapath=':'.join( + env["PATH"] = "{extrapath}:{curpath}".format( + curpath=env["PATH"], + extrapath=":".join( [self._expand_user_vars(p) for p in self.extra_paths] - ) + ), ) - env['SHELL'] = self.default_shell + env["SHELL"] = self.default_shell if self.mem_limit is not None: # FIXME: Detect & use proper properties for v1 vs v2 cgroups - properties['MemoryAccounting'] = 'yes' - properties['MemoryLimit'] = self.mem_limit + properties["MemoryAccounting"] = "yes" + properties["MemoryLimit"] = self.mem_limit if self.cpu_limit is not None: # FIXME: Detect & use proper properties for v1 vs v2 cgroups # FIXME: Make sure that the kernel supports CONFIG_CFS_BANDWIDTH # otherwise this doesn't have any effect. - properties['CPUAccounting'] = 'yes' - properties['CPUQuota'] = '{}%'.format(int(self.cpu_limit * 100)) + properties["CPUAccounting"] = "yes" + properties["CPUQuota"] = f"{int(self.cpu_limit * 100)}%" if self.disable_user_sudo: - properties['NoNewPrivileges'] = 'yes' + properties["NoNewPrivileges"] = "yes" if self.readonly_paths is not None: - properties['ReadOnlyDirectories'] = [ - self._expand_user_vars(path) - for path in self.readonly_paths + properties["ReadOnlyDirectories"] = [ + self._expand_user_vars(path) for path in self.readonly_paths ] if self.readwrite_paths is not None: - properties['ReadWriteDirectories'] = [ - self._expand_user_vars(path) - for path in self.readwrite_paths + properties["ReadWriteDirectories"] = [ + self._expand_user_vars(path) for path in self.readwrite_paths ] for property, value in self.unit_extra_properties.items(): @@ -307,7 +322,7 @@ async def start(self): for i in range(self.start_timeout): is_up = await self.poll() if is_up is None: - return (self.ip or '127.0.0.1', self.port) + return (self.ip or "127.0.0.1", self.port) await asyncio.sleep(1) return None diff --git a/tests/test_systemd.py b/tests/test_systemd.py index 599ee30..f012bd5 100644 --- a/tests/test_systemd.py +++ b/tests/test_systemd.py @@ -3,22 +3,21 @@ Must run as root. """ -import tempfile -from systemdspawner import systemd -import pytest import asyncio import os +import tempfile import time +import pytest + +from systemdspawner import systemd + @pytest.mark.asyncio async def test_simple_start(): - unit_name = 'systemdspawner-unittest-' + str(time.time()) + unit_name = "systemdspawner-unittest-" + str(time.time()) await systemd.start_transient_service( - unit_name, - ['sleep'], - ['2000'], - working_dir='/' + unit_name, ["sleep"], ["2000"], working_dir="/" ) assert await systemd.service_running(unit_name) @@ -33,13 +32,13 @@ async def test_service_failed_reset(): """ Test service_failed and reset_service """ - unit_name = 'systemdspawner-unittest-' + str(time.time()) + unit_name = "systemdspawner-unittest-" + str(time.time()) # Running a service with an invalid UID makes it enter a failed state await systemd.start_transient_service( unit_name, - ['sleep'], - ['2000'], - working_dir='/systemdspawner-unittest-does-not-exist' + ["sleep"], + ["2000"], + working_dir="/systemdspawner-unittest-does-not-exist", ) await asyncio.sleep(0.1) @@ -56,20 +55,20 @@ async def test_service_running_fail(): """ Test service_running failing when there's no service. """ - unit_name = 'systemdspawner-unittest-' + str(time.time()) + unit_name = "systemdspawner-unittest-" + str(time.time()) assert not await systemd.service_running(unit_name) @pytest.mark.asyncio async def test_env_setting(): - unit_name = 'systemdspawner-unittest-' + str(time.time()) + unit_name = "systemdspawner-unittest-" + str(time.time()) with tempfile.TemporaryDirectory() as d: os.chmod(d, 0o777) await systemd.start_transient_service( unit_name, ["/bin/bash"], - ["-c", "pwd; ls -la {0}; env > ./env; sleep 3".format(d)], + ["-c", f"pwd; ls -la {d}; env > ./env; sleep 3"], working_dir=d, environment_variables={ "TESTING_SYSTEMD_ENV_1": "TEST 1", @@ -92,7 +91,7 @@ async def test_env_setting(): assert os.path.exists(env_file) assert (os.stat(env_file).st_mode & 0o777) == 0o400 # verify that the env had the desired effect - with open(os.path.join(d, 'env')) as f: + with open(os.path.join(d, "env")) as f: text = f.read() assert "TESTING_SYSTEMD_ENV_1=TEST 1" in text assert "TESTING_SYSTEMD_ENV_2=TEST 2" in text @@ -105,47 +104,48 @@ async def test_env_setting(): @pytest.mark.asyncio async def test_workdir(): - unit_name = 'systemdspawner-unittest-' + str(time.time()) + unit_name = "systemdspawner-unittest-" + str(time.time()) _, env_filename = tempfile.mkstemp() with tempfile.TemporaryDirectory() as d: await systemd.start_transient_service( unit_name, - ['/bin/bash'], - ['-c', 'pwd > {}/pwd'.format(d)], + ["/bin/bash"], + ["-c", f"pwd > {d}/pwd"], working_dir=d, ) # Wait a tiny bit for the systemd unit to complete running await asyncio.sleep(0.1) - - with open(os.path.join(d, 'pwd')) as f: + + with open(os.path.join(d, "pwd")) as f: text = f.read().strip() assert text == d @pytest.mark.asyncio async def test_slice(): - unit_name = 'systemdspawner-unittest-' + str(time.time()) + unit_name = "systemdspawner-unittest-" + str(time.time()) _, env_filename = tempfile.mkstemp() with tempfile.TemporaryDirectory() as d: await systemd.start_transient_service( unit_name, - ['/bin/bash'], - ['-c', 'pwd > {}/pwd; sleep 10;'.format(d)], + ["/bin/bash"], + ["-c", f"pwd > {d}/pwd; sleep 10;"], working_dir=d, - slice='user.slice', + slice="user.slice", ) # Wait a tiny bit for the systemd unit to complete running await asyncio.sleep(0.1) proc = await asyncio.create_subprocess_exec( - *['systemctl', 'status', unit_name], + *["systemctl", "status", unit_name], stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE) + stderr=asyncio.subprocess.PIPE, + ) stdout, stderr = await proc.communicate() - assert b'user.slice' in stdout + assert b"user.slice" in stdout @pytest.mark.asyncio @@ -160,24 +160,22 @@ async def test_properties_string(): This validates the Bind Mount is working, and hence properties are working. """ - unit_name = 'systemdspawner-unittest-' + str(time.time()) + unit_name = "systemdspawner-unittest-" + str(time.time()) _, env_filename = tempfile.mkstemp() with tempfile.TemporaryDirectory() as d: await systemd.start_transient_service( unit_name, - ['/bin/bash'], - ['-c', 'pwd > pwd'], - working_dir='/bind-test', - properties={ - 'BindPaths': '{}:/bind-test'.format(d) - } + ["/bin/bash"], + ["-c", "pwd > pwd"], + working_dir="/bind-test", + properties={"BindPaths": f"{d}:/bind-test"}, ) # Wait a tiny bit for the systemd unit to complete running await asyncio.sleep(0.1) - with open(os.path.join(d, 'pwd')) as f: + with open(os.path.join(d, "pwd")) as f: text = f.read().strip() - assert text == '/bind-test' + assert text == "/bind-test" @pytest.mark.asyncio @@ -195,13 +193,13 @@ async def test_properties_list(): This validates multiple ordered ExcePreStart calls are working, and hence properties with lists as values are working. """ - unit_name = 'systemdspawner-unittest-' + str(time.time()) + unit_name = "systemdspawner-unittest-" + str(time.time()) _, env_filename = tempfile.mkstemp() with tempfile.TemporaryDirectory() as d: await systemd.start_transient_service( unit_name, - ['/bin/bash'], - ['-c', 'pwd > test-1/test-2/pwd'], + ["/bin/bash"], + ["-c", "pwd > test-1/test-2/pwd"], working_dir=d, properties={ "ExecStartPre": [ @@ -212,7 +210,7 @@ async def test_properties_list(): # Wait a tiny bit for the systemd unit to complete running await asyncio.sleep(0.1) - with open(os.path.join(d, 'test-1', 'test-2', 'pwd')) as f: + with open(os.path.join(d, "test-1", "test-2", "pwd")) as f: text = f.read().strip() assert text == d @@ -228,21 +226,16 @@ async def test_uid_gid(): This validates that setting uid sets uid, gid sets the gid """ - unit_name = 'systemdspawner-unittest-' + str(time.time()) + unit_name = "systemdspawner-unittest-" + str(time.time()) _, env_filename = tempfile.mkstemp() with tempfile.TemporaryDirectory() as d: os.chmod(d, 0o777) await systemd.start_transient_service( - unit_name, - ['/bin/bash'], - ['-c', 'id > id'], - working_dir=d, - uid=65534, - gid=0 + unit_name, ["/bin/bash"], ["-c", "id > id"], working_dir=d, uid=65534, gid=0 ) # Wait a tiny bit for the systemd unit to complete running await asyncio.sleep(0.2) - with open(os.path.join(d, 'id')) as f: + with open(os.path.join(d, "id")) as f: text = f.read().strip() - assert text == 'uid=65534(nobody) gid=0(root) groups=0(root)' + assert text == "uid=65534(nobody) gid=0(root) groups=0(root)"