Skip to content

Commit

Permalink
Improve volume handling in the buildarr compose command (#188)
Browse files Browse the repository at this point in the history 
* Allow plugins to define volumes as a list of [short-syntax strings](https://docs.docker.com/compose/compose-file/05-services/#short-syntax-5), or a list of [long-syntax dictionaries](https://docs.docker.com/compose/compose-file/05-services/#long-syntax-5), in addition to the currently used simple mapping format.
* Convert all supported service volume formats into the "list of long-syntax dictionaries" format in the output Docker Compose file.
* Make troubleshooting easier for the user when two services share the same hostname, by outputting the specific instance defined under the specific plugin that was clashed with (instead of the service name in the output Docker Compose file, which the user won't have).
* Remove extra newline from the end of the `buildarr compose` command output.
* Add new options to the `dummy` plugin for testing the new feaures.
  • Loading branch information
@Callum027
Callum027 committed Mar 8, 2024
1 parent a7376fd commit c8a7e69
Show file tree
Hide file tree
Showing 3 changed files with 146 additions and 29 deletions.
101 changes: 75 additions & 26 deletions buildarr/cli/compose.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
)

if TYPE_CHECKING:
from typing import Any, Dict, Set
from typing import Any, Dict, List, Set


logger = getLogger(__name__)
Expand Down Expand Up @@ -177,7 +177,7 @@ def compose(
logger.debug(" %i. %s.instances[%s]", i, plugin_name, repr(instance_name))

compose_obj: Dict[str, Any] = {"version": compose_version, "services": {}}
hostnames: Dict[str, str] = {}
used_hostnames: Dict[str, Dict[str, str]] = {}
volumes: Set[str] = set()

for plugin_name, instance_name in state._execution_order:
Expand All @@ -204,30 +204,80 @@ def compose(
f"Invalid hostname '{hostname}' for {plugin_name} instance '{instance_name}': "
"Hostname must not be localhost for Docker Compose services",
)
if hostname in hostnames:
if hostname in used_hostnames:
used_by = used_hostnames[hostname]
raise ComposeInvalidHostnameError(
f"Invalid hostname '{hostname}' for {plugin_name} instance '{instance_name}': "
f"Hostname already used by service '{hostnames[hostname]}'",
(
f"Invalid hostname '{hostname}' for "
f"{plugin_name} instance '{instance_name}': "
"Hostname already used by "
f"{used_by['plugin_name']} instance '{used_by['instance_name']}'"
),
)
hostnames[hostname] = service_name
used_hostnames[hostname] = {"plugin_name": plugin_name, "instance_name": instance_name}
logger.debug("Finished validating service hostname")
logger.debug("Generating service-specific configuration")
try:
logger.debug("Generating service-specific configuration")
service: Dict[str, Any] = {
**manager.to_compose_service(
instance_config=instance_config,
compose_version=compose_version,
service_name=service_name,
),
"hostname": hostname,
"restart": compose_restart,
}
logger.debug("Finished generating service-specific configuration")
service_config = manager.to_compose_service(
instance_config=instance_config,
compose_version=compose_version,
service_name=service_name,
)
except NotImplementedError:
raise ComposeNotSupportedError(
f"Plugin '{plugin_name}' does not support Docker Compose "
"service generation from instance configurations",
) from None
if "volumes" in service_config:
# Convert the old-style dictionary volume definitions
# to the new-style list structure with long-form syntax.
if isinstance(service_config["volumes"], dict):
service_config["volumes"] = [
{
"type": (
"volume" if "/" not in source and "\\" not in source else "bind"
),
"source": source,
"target": target,
}
for source, target in service_config["volumes"].items()
]
# Otherwise, assume the definition is a list structure.
# Convert old-style string volume definitions to long-form syntax.
else:
new_volumes: List[Dict[str, Any]] = []
for volume in service_config["volumes"]:
if isinstance(volume, str):
try:
source, target, options_str = volume.split(":", maxsplit=3)
except ValueError:
source, target = volume.split(":", maxsplit=2)
options_str = None
options: Set[str] = (
set(o.strip().lower() for o in options_str.split(","))
if options_str
else set()
)
new_volume = {
"type": (
"volume" if "/" not in source and "\\" not in source else "bind"
),
"source": source,
"target": target,
"read_only": "ro" in options and "rw" not in options,
}
if new_volume["type"] == "bind":
new_volume["bind"] = {"create_host_path": True}
new_volumes.append(new_volume)
else:
new_volumes.append(volume)
service_config["volumes"] = new_volumes
service: Dict[str, Any] = {
**service_config,
"hostname": hostname,
"restart": compose_restart,
}
logger.debug("Finished generating service-specific configuration")
if (plugin_name, instance_name) in state._instance_dependencies:
depends_on: Set[str] = set()
logger.debug("Generating service dependencies")
Expand All @@ -239,22 +289,21 @@ def compose(
depends_on.add(target_service)
service["depends_on"] = list(depends_on)
logger.debug("Finished generating service dependencies")
# TODO: Handle more types of volume definitions
# (e.g. volume lists, modern-style mount definitions).
for volume_name in service.get("volumes", {}).keys():
if "/" not in volume_name and "\\" not in volume_name:
for volume in service.get("volumes", []):
if volume["type"] == "volume":
logger.debug(
"Adding named volume '%s' to the list of internal volumes",
volume_name,
volume["source"],
)
volumes.add(volume_name)
volumes.add(volume["source"])
else:
logger.debug(
(
"Volume '%s' determined to likely be a bind mount, "
"Volume '%s:%s' is a bind mount, "
"not adding to the list of internal volumes"
),
volume_name,
volume["source"],
volume["target"],
)
compose_obj["services"][service_name] = service
logger.debug("Finished generating Docker Compose service configuration")
Expand Down Expand Up @@ -282,4 +331,4 @@ def compose(
if volumes:
compose_obj["volumes"] = list(volumes)

click.echo(yaml.safe_dump(compose_obj, explicit_start=True, sort_keys=False))
click.echo(yaml.safe_dump(compose_obj, explicit_start=True, sort_keys=False), nl=False)
65 changes: 64 additions & 1 deletion buildarr/plugins/dummy/config/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,13 @@

from __future__ import annotations

from typing import TYPE_CHECKING, Dict, Optional
from typing import TYPE_CHECKING, Any, Dict, Literal, Optional

from typing_extensions import Self

from buildarr import __version__
from buildarr.config import ConfigPlugin
from buildarr.state import state
from buildarr.types import NonEmptyStr, Port

from ..api import api_get
Expand Down Expand Up @@ -135,6 +137,20 @@ class DummyInstanceConfig(_DummyInstanceConfig):
Configuration options for Dummy itself are set within this structure.
"""

use_service_volumes: bool = False
"""
Whether or not to configure volumes when generating the Docker Compose service definition.
Used in functional tests.
"""

service_volumes_type: Literal["dict", "list-str", "list-dict"] = "list-dict"
"""
The type to use for the service volumes when generating the Docker Compose service definition.
Used in functional tests.
"""

def uses_trash_metadata(self) -> bool:
"""
Return whether or not this instance configuration uses TRaSH-Guides metadata.
Expand Down Expand Up @@ -183,6 +199,53 @@ def from_remote(cls, secrets: DummySecrets) -> Self:
settings=DummySettingsConfig.from_remote(secrets),
)

def to_compose_service(self, compose_version: str, service_name: str) -> Dict[str, Any]:
"""
Generate a Docker Compose service definition corresponding to this instance configuration.
Plugins should implement this function to allow Docker Compose files to be generated from
Buildarr configuration using the `buildarr compose` command.
Args:
compose_version (str): Version of the Docker Compose file.
service_name (str): The unique name for the generated Docker Compose service.
Returns:
Docker Compose service definition dictionary
"""
service: Dict[str, Any] = {
"image": f"{state.config.buildarr.docker_image_uri}:{__version__}",
"entrypoint": ["flask"],
"command": ["--app", "buildarr.plugins.dummy.server:app", "run", "--debug"],
}
if self.use_service_volumes:
if self.service_volumes_type == "list-dict":
service["volumes"] = [
{
"type": "bind",
"source": str(state.config_files[0].parent),
"target": "/config",
"read_only": True,
},
{
"type": "volume",
"source": service_name,
"target": "/data",
"read_only": False,
},
]
elif self.service_volumes_type == "list-str":
service["volumes"] = [
f"{state.config_files[0].parent}:/config:ro",
f"{service_name}:/data",
]
else:
service["volumes"] = {
str(state.config_files[0].parent): "/config",
service_name: "/data",
}
return service


class DummyConfig(DummyInstanceConfig):
"""
Expand Down
9 changes: 7 additions & 2 deletions docs/usage.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -285,13 +285,17 @@ services:
sonarr_sonarr-hd:
image: lscr.io/linuxserver/sonarr:latest
volumes:
sonarr_sonarr-hd: /config
- type: volume
source: sonarr_sonarr-hd
target: /config
hostname: sonarr-hd
restart: always
sonarr_sonarr-4k:
image: lscr.io/linuxserver/sonarr:latest
volumes:
sonarr_sonarr-4k: /config
- type: volume
source: sonarr_sonarr-4k
target: /config
hostname: sonarr-4k
restart: always
depends_on:
Expand All @@ -305,6 +309,7 @@ services:
- type: bind
source: /opt/buildarr
target: /config
read_only: true
restart: always
depends_on:
- sonarr_sonarr-hd
Expand Down

0 comments on commit c8a7e69

Please sign in to comment.