From 55db827649ee8ef3fc37263f76611f690fdb2d8f Mon Sep 17 00:00:00 2001 From: Paulo Miguel Almeida Date: Thu, 7 Oct 2021 15:58:47 +1300 Subject: [PATCH] Implement singularity-compose.yml override configuration feature (#48) * implement deep_merge functionally * implement some changes suggested during code review * add tests for the config override feature * good old black formatting that I keep forgetting :) * add docs * adding support for singularity-compose check this will add jsonschema as an optional dependency to check the validity of a singularity-compose.yml file. We will eventually extend this to check (and preview) combined files. Signed-off-by: vsoch * implement deep_merge functionally * implement some changes suggested during code review * add tests for the config override feature * tweak check group so it takes into account multiple files instead * implement preview option * update changelog and add docs * Apply suggestions from code review Co-authored-by: Vanessasaurus <814322+vsoch@users.noreply.github.com> * move deep_merge stuff to a brand new module * black formatting * remove extra bullet point * implement code review changes * move config functions into proper module Signed-off-by: vsoch and moving validate.py into its own file so we do not always import * use bot.exit over print and sys.exit (which combines the two) Signed-off-by: vsoch Co-authored-by: vsoch Co-authored-by: Vanessasaurus <814322+vsoch@users.noreply.github.com> --- .github/workflows/main.yml | 5 +- CHANGELOG.md | 4 + docs/commands.md | 166 +++++++++++++++++- docs/spec/spec-2.0.md | 3 +- scompose/client/__init__.py | 28 ++- scompose/client/check.py | 37 ++++ scompose/config/__init__.py | 68 +++++++ scompose/config/schema.py | 123 +++++++++++++ scompose/project/__init__.py | 1 + scompose/project/project.py | 154 ++++++++++------ .../config_merge/singularity-compose-1.yml | 8 + .../config_merge/singularity-compose-2.yml | 12 ++ scompose/tests/test_config.py | 118 +++++++++++++ scompose/tests/test_utils.py | 2 + scompose/version.py | 4 +- setup.py | 6 +- 16 files changed, 665 insertions(+), 74 deletions(-) create mode 100644 scompose/client/check.py create mode 100644 scompose/config/__init__.py create mode 100644 scompose/config/schema.py create mode 100644 scompose/tests/configs/config_merge/singularity-compose-1.yml create mode 100644 scompose/tests/configs/config_merge/singularity-compose-2.yml create mode 100644 scompose/tests/test_config.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a6bdd39..3a4e80b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,12 +14,13 @@ jobs: - uses: actions/checkout@v2 - name: Setup black environment - run: conda create --quiet --name black black pyflakes + run: conda create --quiet --name black pyflakes - name: Check formatting with black run: | export PATH="/usr/share/miniconda/bin:$PATH" source activate black + pip install black==21.6b0 black --check scompose - name: Check imports with pyflakes @@ -28,4 +29,4 @@ jobs: source activate black pyflakes scompose/utils # Will have some issues - pyflakes scompose/client scompose/project || true + pyflakes scompose/client scompose/project scompose/config || true diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f045c9..39195a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,10 @@ and **Merged pull requests**. Critical items to know are: The versions coincide with releases on pypi. ## [0.0.x](https://github.com/singularityhub/singularity-compose/tree/master) (0.0.x) + + - adding jsonschema validation and check command (0.0.12) + - implement configuration override feature + - implement `--preview` argument for the `check` command - add network->enable option on composer file (0.1.11) - add network->allocate_ip option on composer file (0.1.10) - version 2.0 of the spec with added fakeroot network, start, exec, and run options (0.1.0) diff --git a/docs/commands.md b/docs/commands.md index 3e6b64c..f1e3694 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -3,7 +3,48 @@ The following commands are currently supported. Remember, you must be in the present working directory of the compose file to reference the correct instances. -## Build +## check + +To do a sanity check of your singularity-compose.yml, you can use `singularity-compose check` + +```bash +$ singularity-compose check +singularity-compose.yml is valid. + +$ singularity-compose -f singularity-compose.yml \ + -f singularity-compose.override.yml check +singularity-compose.yml is valid. +singularity-compose.override.yml is valid. +``` + +To view the combined compose files you can use `--preview`. + +```bash +$ singularity-compose -f singularity-compose.yml \ + -f singularity-compose.override.yml check --preview + +version: '2.0' +instances: + cvatdb: + start: + options: + - containall + network: + enable: false + volumes: + - ./recipes/postgres/env.sh:/.singularity.d/env/env.sh + - ./volumes/postgres/conf:/opt/bitnami/postgresql/conf + - ./volumes/postgres/tmp:/opt/bitnami/postgresql/tmp + - /home/vagrant/postgres_data:/bitnami/postgresql + build: + context: . + recipe: ./recipes/postgres/main.def + options: + - fakeroot + +``` + +## build Build will either build a container recipe, or pull a container to the instance folder. In both cases, it's named after the instance so we can @@ -21,7 +62,7 @@ If the build requires sudo (if you've defined sections in the config that warran setting up networking with sudo) the build will instead give you an instruction to run with sudo. -## Up +## up If you want to both build and bring them up, you can use "up." Note that for builds that require sudo, this will still stop and ask you to build with sudo. @@ -52,7 +93,7 @@ $ singularity-compose up --no-resolv Creating app ``` -## Create +## create Given that you have built your containers with `singularity-compose build`, you can create your instances as follows: @@ -93,7 +134,7 @@ INSTANCES NAME PID IMAGE 3 nginx 6543 nginx.sif ``` -## Shell +## shell It's sometimes helpful to peek inside a running instance, either to look at permissions, inspect binds, or manually test running something. @@ -104,7 +145,7 @@ $ singularity-compose shell app Singularity app.sif:~/Documents/Dropbox/Code/singularity/singularity-compose-example> ``` -## Exec +## exec You can easily execute a command to a running instance: @@ -134,7 +175,7 @@ usr var ``` -## Run +## run If a container has a `%runscript` section (or a Docker entrypoint/cmd that was converted to one), you can run that script easily: @@ -147,7 +188,7 @@ If your container didn't have any kind of runscript, the startscript will be used instead. -## Down +## down You can bring one or more instances down (meaning, stopping them) by doing: @@ -172,7 +213,7 @@ in order to kill instances after the specified number of seconds: singularity-compose down -t 100 ``` -## Logs +## logs You can of course view logs for all instances, or just specific named ones: @@ -190,7 +231,7 @@ nginx: [emerg] host not found in upstream "uwsgi" in /etc/nginx/conf.d/default.c nginx: [emerg] host not found in upstream "uwsgi" in /etc/nginx/conf.d/default.conf:22 ``` -## Config +## config You can load and validate the configuration file (singularity-compose.yml) and print it to the screen as follows: @@ -245,4 +286,111 @@ $ singularity-compose config } ``` +# Global arguments + +The following arguments are supported for all commands available. + +## debug + +Set logging verbosity to debug. + +```bash +$ singularity-compose --debug version +``` + +This is equivalent to passing `--log-level=DEBUG` to the CLI. + +```bash +$ singularity-compose --log-level='DEBUG' version +``` + +## log_level + +Change logging verbosity. Accepted values are: `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL` + +```bash +$ singularity-compose --log-level='INFO' version +``` + +## file + +Specify the location of a Compose configuration file + +Default value: `singularity-compose.yml` + +Aliases `--file`, `-f`. + +You can supply multiple `-f` configuration files. When you supply multiple files, `singularity-compose` + combines them into a single configuration. It builds the configuration in the order you supply the +files. Subsequent files override and add to their predecessors. + +For example consider this command: + +```bash +$ singularity-compose -f singularity-compose.yml -f singularity-compose.dev.yml up +``` + +The `singularity-compose.yml` file might specify a `webapp` instance: + +```yaml +instances: + webapp: + image: webapp.sif + start: + args: "start-daemon" + port: + - "80:80" + volumes: + - /mnt/shared_drive/folder:/webapp/data +``` + +if the `singularity-compose.dev.yml` also specifies this same service, any matching fields override +the previous files. + +```yaml +instances: + webapp: + start: + args: "start-daemon -debug" + volumes: + - /home/user/folder:/webapp/data +``` + +The result of the examples above would be translated in runtime to: + +```yaml +instances: + webapp: + image: webapp.sif + start: + args: "start-daemon -debug" + port: + - "80:80" + volumes: + - /home/user/folder:/webapp/data +``` + +## project-name + +Specify project name. + +Default value: `$PWD` + +Aliases `--project-name`, `-p`. + +```bash +$ singularity-compose --project-name 'my_cool_project' up +``` + +## project-directory + +Specify project working directory + +Default value: compose file location + + +```bash +$ singularity-compose --project-directory /home/user/myfolder up +``` + [home](/README.md#singularity-compose) diff --git a/docs/spec/spec-2.0.md b/docs/spec/spec-2.0.md index 114cc40..72a7d41 100644 --- a/docs/spec/spec-2.0.md +++ b/docs/spec/spec-2.0.md @@ -293,8 +293,7 @@ The fields for instances are discussed below: |Name| Description | |----|-------------| -|name|The name of the instance will be "nginx" unless the user provides a "name" -field (not defined above).| +|name|The name of the instance will be "nginx" unless the user provides a "name" field (not defined above).| |build| a section to define how and where to build the base container from.| |build.context| the folder with the Singularity file (and other relevant files). Must exist. |build.recipe| the Singularity recipe in the build context folder. It defaults to `Singularity`| diff --git a/scompose/client/__init__.py b/scompose/client/__init__.py index 9393c08..7c7bf5e 100644 --- a/scompose/client/__init__.py +++ b/scompose/client/__init__.py @@ -53,8 +53,9 @@ def get_parser(): "--file", "-f", dest="file", - help="specify compose file (default singularity-compose.yml)", - default="singularity-compose.yml", + help="specify compose file (default singularity-compose.yml). It can be used multiple times", + action="append", + default=[], ) parser.add_argument( @@ -96,6 +97,20 @@ def get_parser(): build = subparsers.add_parser("build", help="Build or rebuild containers") + # Check + + check = subparsers.add_parser( + "check", help="check or validate singularity-compose.yml" + ) + + check.add_argument( + "--preview", + dest="preview", + help="print compose file(s) interpolated content", + default=False, + action="store_true", + ) + # Config config = subparsers.add_parser("config", help="Validate and view the compose file") @@ -150,7 +165,7 @@ def get_parser(): execute = subparsers.add_parser("exec", help="execute a command to an instance") - run = subparsers.add_parser("run", help="run an instance runscript.") + run = subparsers.add_parser("run", help="run an instance runscript") shell = subparsers.add_parser("shell", help="shell into an instance") @@ -225,9 +240,16 @@ def show_help(return_code=0): print(scompose.__version__) sys.exit(0) + # argparse inherits a funny behaviour that appends default values to the dest value whether you've specified a value + # or not. The bug/behaviour is documented here: https://bugs.python.org/issue16399 + if len(args.file) == 0: + args.file = ["singularity-compose.yml"] + # Does the user want a shell? if args.command == "build": from scompose.client.build import main + elif args.command == "check": + from scompose.client.check import main elif args.command == "create": from scompose.client.create import main elif args.command == "config": diff --git a/scompose/client/check.py b/scompose/client/check.py new file mode 100644 index 0000000..f1408de --- /dev/null +++ b/scompose/client/check.py @@ -0,0 +1,37 @@ +""" + +Copyright (C) 2021 Vanessa Sochat. + +This Source Code Form is subject to the terms of the +Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed +with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" + +from scompose.logger import bot +from scompose.config import merge_config +from scompose.config.validate import validate_config +import yaml + + +def main(args, parser, extra): + """ + Validate compose files for correctness. + + CLI Arguments + ========== + --preview flag to show combined configs. + """ + + # validate compose files + for f in args.file: + result = validate_config(f) + if not result and not args.preview: + bot.info("%s is valid." % f) + elif result: + bot.exit("%s is not valid." % f) + + if args.preview: + # preview + config = merge_config(args.file) + print(yaml.dump(config, sort_keys=False)) diff --git a/scompose/config/__init__.py b/scompose/config/__init__.py new file mode 100644 index 0000000..0937216 --- /dev/null +++ b/scompose/config/__init__.py @@ -0,0 +1,68 @@ +""" + +Copyright (C) 2019-2021 Vanessa Sochat. + +This Source Code Form is subject to the terms of the +Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed +with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" + +import os + +from scompose.logger import bot +from scompose.utils import read_yaml + + +def merge_config(file_list): + """ + Given one or more config files, merge into one + """ + yaml_files = [] + for f in file_list: + try: + # ensure file exists + if not os.path.exists(f): + bot.exit("%s does not exist." % f) + + # read yaml file + yaml_files.append(read_yaml(f, quiet=True)) + except: # ParserError + bot.exit("Cannot parse %s, invalid yaml." % f) + + # merge/override yaml properties where applicable + return _deep_merge(yaml_files) + + +def _deep_merge(yaml_files): + """ + Merge yaml files into a single dict + """ + base_yaml = None + for idx, item in enumerate(yaml_files): + if idx == 0: + base_yaml = item + else: + base_yaml = _merge(base_yaml, item) + + return base_yaml + + +def _merge(a, b): + """ + Merge dict b into a + """ + for key in b: + if key in a: + # merge dicts recursively + if isinstance(a[key], dict) and isinstance(b[key], dict): + a[key] = _merge(a[key], b[key]) + # if types are equal, b takes precedence + elif isinstance(a[key], type(b[key])): + a[key] = b[key] + # if nothing matches then this means a conflict of types which shouldn't exist in the first place + else: + bot.exit("key '%s': type mismatch in different files." % key) + else: + a[key] = b[key] + return a diff --git a/scompose/config/schema.py b/scompose/config/schema.py new file mode 100644 index 0000000..d6c8ded --- /dev/null +++ b/scompose/config/schema.py @@ -0,0 +1,123 @@ +""" + +Copyright (C) 2021 Vanessa Sochat. + +This Source Code Form is subject to the terms of the +Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed +with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" + +import os +import sys + +from scompose.utils import read_yaml + +# We don't require jsonschema, so catch import error and alert user +try: + from jsonschema import validate +except ImportError as e: + msg = "pip install jsonschema" + sys.exit("jsonschema is required for checking and validation: %s\n %s" % (e, msg)) + + +def validate_config(filepath): + """ + Validate a singularity-compose.yaml file. + """ + cfg = read_yaml(filepath, quiet=True) + return validate(cfg, compose_schema) + + +## Singularity Compose Schema + +schema_url = "https://json-schema.org/draft-07/schema/#" + +# Common patterns of types +string_list = {"type": "array", "items": {"type": "string"}} + +# Instance groups +instance_build = { + "type": "object", + "properties": { + "recipe": {"type": "string"}, + "context": {"type": "string"}, + "options": string_list, + }, +} + +instance_network = { + "type": "object", + "properties": { + "allocate_ip": {"type": "boolean"}, + "enable": {"type": "boolean"}, + }, +} + + +instance_start = { + "type": "object", + "properties": { + "args": {"type": ["string", "array"]}, + "options": string_list, + }, +} + +instance_run = { + "type": "object", + "properties": { + "args": {"type": ["string", "array"]}, + "options": string_list, + }, +} + +instance_post = { + "type": "object", + "properties": { + "commands": string_list, + }, +} + +instance_exec = { + "type": "object", + "properties": {"options": string_list, "command": {"type": "string"}}, + "required": [ + "command", + ], +} + +# A single instance +instance = { + "type": "object", + "properties": { + "image": {"type": "string"}, + "build": instance_build, + "network": instance_network, + "ports": string_list, + "volumes": string_list, + "volumes_from": string_list, + "depends_on": string_list, + "start": instance_start, + "exec": instance_exec, + "run": {"oneOf": [instance_run, {"type": "array"}]}, + "post": instance_post, + }, +} + + +# instances define container services +instances = {"type": "object", "patternProperties": {"\\w[\\w-]*": instance}} + +properties = {"version": {"type": "string"}, "instances": instances} + +compose_schema = { + "$schema": schema_url, + "title": "Singularity Compose Schema", + "type": "object", + "required": [ + "version", + "instances", + ], + "properties": properties, + "additionalProperties": False, +} diff --git a/scompose/project/__init__.py b/scompose/project/__init__.py index f816466..6f1ff06 100644 --- a/scompose/project/__init__.py +++ b/scompose/project/__init__.py @@ -7,4 +7,5 @@ with this file, You can obtain one at http://mozilla.org/MPL/2.0/. """ + from .project import Project diff --git a/scompose/project/project.py b/scompose/project/project.py index ef80997..b07f0bf 100644 --- a/scompose/project/project.py +++ b/scompose/project/project.py @@ -10,7 +10,8 @@ from scompose.templates import get_template from scompose.logger import bot -from scompose.utils import read_yaml, read_file, write_file +from scompose.utils import read_file, write_file +from ..config import merge_config from spython.main import get_client from .instance import Instance from ipaddress import IPv4Network @@ -18,7 +19,6 @@ import os import re import subprocess -import sys class Project(object): @@ -46,8 +46,10 @@ def __repr__(self): return self.__str__() def get_instance_names(self): - """return a list of names, if a config file is loaded, and instances - are defined. + """ + Return a list of names. + + Do this if a config file is loaded, and instances are defined. """ names = [] if self.instances is not None: @@ -56,32 +58,42 @@ def get_instance_names(self): return names def set_filename(self, filename): - """set the filename to read the recipe from. If not provided, defaults - to singularity-compose.yml. The working directory is set to - be the directory name of the configuration file. + """Set the filename to read the recipe from. + + If not provided, defaults to singularity-compose.yml. The working directory + is set to be the directory name of the configuration file. Parameters ========== - filename: the singularity-compose.yml file to use + filename: the singularity-compose.yml file to use. This can be a str or a list of str """ - self.filename = filename or "singularity-compose.yml" - self.working_dir = os.path.dirname(os.path.abspath(self.filename)) + default_value = ["singularity-compose.yml"] + if filename is None: + self.filenames = default_value + elif isinstance(filename, list): + self.filenames = filename or default_value + else: + self.filenames = [filename] + + self.working_dir = os.getcwd() def set_name(self, name): - """set the filename to read the recipe from. If not provided, defaults - to singularity-compose.yml + """ + Set the filename to read the recipe from. + + If not provided, defaults to singularity-compose.yml Parameters ========== name: if a customize name is provided, use it """ - pwd = os.path.basename(os.path.dirname(os.path.abspath(self.filename))) - self.name = (name or pwd).lower() + self.name = (name or self.working_dir).lower() # Listing - def ps(self): - """ps will print a table of instances, including pids and names.""" + """ + Ps will print a table of instances, including pids and names. + """ instance_names = self.get_instance_names() table = [] for instance in self.client.instances(quiet=True, sudo=self.sudo): @@ -100,8 +112,10 @@ def ps(self): bot.table(table) def iter_instances(self, names): - """yield instances one at a time. If an invalid name is given, - exit with error. + """ + Yield instances one at a time. + + If an invalid name is given, exit with error. Parameters ========== @@ -116,9 +130,10 @@ def iter_instances(self, names): yield self.instances.get(name) def get_instance(self, name): - """get a specifically named instance. We first check that the - client has instances defined, and that the name we are looking - for is also included. If not found, we return None. + """Get a specifically named instance. + + We first check that the client has instances defined, and that the name + we are looking for is also included. If not found, we return None. Parameters ========== @@ -131,9 +146,11 @@ def get_instance(self, name): return instance # Loading Functions - def get_already_running(self): - """Since a user can bring select instances up and down, we need to + """ + Get already running instances. + + Since a user can bring select instances up and down, we need to derive a list of already running instances to include """ # Get list of existing instances to skip addresses @@ -147,18 +164,13 @@ def get_already_running(self): def load(self): """load a singularity-compose.yml recipe, and validate it.""" - - if not os.path.exists(self.filename): - bot.error("%s does not exist." % self.filename) - sys.exit(1) - - try: - self.config = read_yaml(self.filename, quiet=True) - except: # ParserError - bot.exit("Cannot parse %s, invalid yaml." % self.filename) + # merge/override yaml properties where applicable + self.config = merge_config(self.filenames) def parse(self): - """parse a loaded config""" + """ + Parse a loaded config + """ # If a port is defined, we need root. self.sudo = False @@ -197,7 +209,9 @@ def parse(self): instance.set_volumes_from(self.instances) def _sort_instances(self, instances): - """eventually reorder instances based on depends_on constraints""" + """ + Eventually reorder instances based on depends_on constraints + """ sorted_instances = [] for instance in self.instances: depends_on = self.instances[instance].params.get("depends_on", []) @@ -217,7 +231,10 @@ def _sort_instances(self, instances): # Networking def get_ip_lookup(self, names, bridge="10.22.0.0/16"): - """based on a bridge address that can serve other addresses (akin to + """ + Generate a pre-determined address for each container. + + Based on a bridge address that can serve other addresses (akin to a router, metaphorically, generate a pre-determined address for each container. @@ -252,9 +269,11 @@ def get_ip_lookup(self, names, bridge="10.22.0.0/16"): return lookup def get_bridge_address(self, name="sbr0"): - """get the (named) bridge address on the host. It should be automatically - created by Singularity over 3.0. This function currently is not used, - but is left in case it's needed. + """ + Get the (named) bridge address on the host. + + It should be automatically created by Singularity over 3.0. This function + currently is not used, but is left in case it's needed. Parameters ========== @@ -290,7 +309,9 @@ def create_hosts(self, lookup): return hosts_file def generate_resolv_conf(self): - """generate a resolv.conf file to bind to the containers. + """ + Generate a resolv.conf file to bind to the containers. + We use the template provided by scompose. """ resolv_conf = os.path.join(self.working_dir, "resolv.conf") @@ -302,7 +323,8 @@ def generate_resolv_conf(self): # Commands def shell(self, name): - """if an instance exists, shell into it. + """ + If an instance exists, shell into it. Parameters ========== @@ -316,7 +338,8 @@ def shell(self, name): self.client.shell(instance.instance.get_uri(), sudo=self.sudo) def run(self, name): - """if an instance exists, run it. + """ + If an instance exists, run it. Parameters ========== @@ -337,7 +360,8 @@ def run(self, name): print("".join([x for x in result["message"] if x])) def execute(self, name, commands): - """if an instance exists, execute a command to it. + """ + If an instance exists, execute a command to it. Parameters ========== @@ -363,7 +387,8 @@ def execute(self, name, commands): # Logs def clear_logs(self, names): - """clear_logs will remove all old error and output logs. + """ + Clear_logs will remove all old error and output logs. Parameters ========== @@ -374,7 +399,8 @@ def clear_logs(self, names): instance.clear_logs() def logs(self, names=None, tail=0): - """logs will print logs to the screen. + """ + Logs will print logs to the screen. Parameters ========== @@ -395,8 +421,9 @@ def view_config(self): # Down def down(self, names=None, timeout=None): - """stop one or more instances. If no names are provided, bring them - all down. + """ + Stop one or more instances. + If no names are provided, bring them all down. Parameters ========== @@ -417,18 +444,29 @@ def create( self, names=None, writable_tmpfs=True, bridge="10.22.0.0/16", no_resolv=False ): - """call the create function, which defaults to the command instance.create()""" + """ + Call the create function, which defaults to the command instance.create() + """ return self._create(names, writable_tmpfs=writable_tmpfs, no_resolv=no_resolv) def up( - self, names=None, writable_tmpfs=True, bridge="10.22.0.0/16", no_resolv=False, + self, + names=None, + writable_tmpfs=True, + bridge="10.22.0.0/16", + no_resolv=False, ): - """call the up function, instance.up(), which will build before if - a container binary does not exist. + """ + Call the up function, instance.up(). + + This will build before if a container binary does not exist. """ return self._create( - names, command="up", writable_tmpfs=writable_tmpfs, no_resolv=no_resolv, + names, + command="up", + writable_tmpfs=writable_tmpfs, + no_resolv=no_resolv, ) def _create( @@ -440,10 +478,12 @@ def _create( no_resolv=False, ): - """create one or more instances. "Command" determines the sub function - to call for the instance, which should be "create" or "up". - If the user provide a list of names, use them, otherwise default - to all instances. + """ + Create one or more instances. + + "Command" determines the sub function to call for the instance, + which should be "create" or "up". If the user provide a list of names, + use them, otherwise default to all instances. Parameters ========== @@ -503,7 +543,9 @@ def _create( # Build def build(self, names=None): - """given a loaded project, build associated containers (or pull).""" + """ + Given a loaded project, build associated containers (or pull). + """ names = names or self.get_instance_names() for instance in self.iter_instances(names): instance.build(working_dir=self.working_dir) diff --git a/scompose/tests/configs/config_merge/singularity-compose-1.yml b/scompose/tests/configs/config_merge/singularity-compose-1.yml new file mode 100644 index 0000000..f4b860c --- /dev/null +++ b/scompose/tests/configs/config_merge/singularity-compose-1.yml @@ -0,0 +1,8 @@ +version: "2.0" +instances: + echo: + build: + context: . + recipe: Singularity + start: + args: "arg0 arg1 arg2" diff --git a/scompose/tests/configs/config_merge/singularity-compose-2.yml b/scompose/tests/configs/config_merge/singularity-compose-2.yml new file mode 100644 index 0000000..139e7b7 --- /dev/null +++ b/scompose/tests/configs/config_merge/singularity-compose-2.yml @@ -0,0 +1,12 @@ +version: "2.0" +instances: + echo: + start: + options: + - fakeroot + args: "arg0 arg1" + + hello: + image: from_the_other_side.sif + start: + args: "how are you?" diff --git a/scompose/tests/test_config.py b/scompose/tests/test_config.py new file mode 100644 index 0000000..6351d80 --- /dev/null +++ b/scompose/tests/test_config.py @@ -0,0 +1,118 @@ +#!/usr/bin/python + +# Copyright (C) 2017-2021 Vanessa Sochat. + +# This Source Code Form is subject to the terms of the +# Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed +# with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +here = os.path.dirname(os.path.abspath(__file__)) + + +def test_merge(): + print("Testing utils._merge") + from scompose.config import _merge + + # No override + a = {"a": 123} + b = {"b": 456} + assert _merge(a, b) == {"a": 123, "b": 456} + + # Override + merge + a = {"a": 123} + b = {"b": 456, "a": 789} + assert _merge(a, b) == {"a": 789, "b": 456} + + # Override only + a = {"a": 123} + b = {"a": 789} + assert _merge(a, b) == {"a": 789} + + # Dict merge + a = {"a": 123, "b": {"c": "d"}} + b = {"b": {"e": "f"}} + assert _merge(a, b) == {"a": 123, "b": {"c": "d", "e": "f"}} + + # Dict merge + key override + a = {"a": 123, "b": {"c": "d"}} + b = {"b": {"c": "f"}} + assert _merge(a, b) == {"a": 123, "b": {"c": "f"}} + + +def test_deep_merge(): + print("Testing utils._deep_merge") + from scompose.utils import read_yaml + from scompose.config import _deep_merge + + config_override = os.path.join(here, "configs", "config_merge") + + # single file + yaml_files = [ + read_yaml( + os.path.join(config_override, "singularity-compose-1.yml"), quiet=True + ) + ] + ret = _deep_merge(yaml_files) + assert ret["instances"] == { + "echo": { + "build": {"context": ".", "recipe": "Singularity"}, + "start": {"args": "arg0 arg1 arg2"}, + } + } + + # multiple files + yaml_files = [ + read_yaml( + os.path.join(config_override, "singularity-compose-1.yml"), quiet=True + ), + read_yaml( + os.path.join(config_override, "singularity-compose-2.yml"), quiet=True + ), + ] + ret = _deep_merge(yaml_files) + assert ret["instances"] == { + "echo": { + "build": {"context": ".", "recipe": "Singularity"}, + "start": {"args": "arg0 arg1", "options": ["fakeroot"]}, + }, + "hello": { + "image": "from_the_other_side.sif", + "start": {"args": "how are you?"}, + }, + } + + +def test_merge_config(): + print("Testing utils.build_interpolated_config") + from scompose.config import merge_config + + config_override = os.path.join(here, "configs", "config_merge") + + # single file + file_list = [os.path.join(config_override, "singularity-compose-1.yml")] + ret = merge_config(file_list) + assert ret["instances"] == { + "echo": { + "build": {"context": ".", "recipe": "Singularity"}, + "start": {"args": "arg0 arg1 arg2"}, + } + } + + # multiple files + file_list = [ + os.path.join(config_override, "singularity-compose-1.yml"), + os.path.join(config_override, "singularity-compose-2.yml"), + ] + ret = merge_config(file_list) + assert ret["instances"] == { + "echo": { + "build": {"context": ".", "recipe": "Singularity"}, + "start": {"args": "arg0 arg1", "options": ["fakeroot"]}, + }, + "hello": { + "image": "from_the_other_side.sif", + "start": {"args": "how are you?"}, + }, + } diff --git a/scompose/tests/test_utils.py b/scompose/tests/test_utils.py index e2f45d4..6b2efd5 100644 --- a/scompose/tests/test_utils.py +++ b/scompose/tests/test_utils.py @@ -9,6 +9,8 @@ import os import pytest +here = os.path.dirname(os.path.abspath(__file__)) + def test_write_read_files(tmp_path): """test_write_read_files will test the functions write_file and read_file""" diff --git a/scompose/version.py b/scompose/version.py index c7cdb9f..e627bca 100644 --- a/scompose/version.py +++ b/scompose/version.py @@ -8,7 +8,7 @@ """ -__version__ = "0.1.11" +__version__ = "0.1.12" AUTHOR = "Vanessa Sochat" AUTHOR_EMAIL = "vsoch@users.noreply.github.com" NAME = "singularity-compose" @@ -26,6 +26,8 @@ ("pyaml", {"min_version": "5.1.1"}), ) +INSTALL_REQUIRES_CHECKS = INSTALL_REQUIRES + (("jsonschema", {"min_version": None}),) + TESTS_REQUIRES = (("pytest", {"min_version": "4.6.2"}),) INSTALL_REQUIRES_ALL = INSTALL_REQUIRES diff --git a/setup.py b/setup.py index 42cbcac..279ec56 100644 --- a/setup.py +++ b/setup.py @@ -68,6 +68,7 @@ def get_reqs(lookup=None, key="INSTALL_REQUIRES"): INSTALL_REQUIRES = get_reqs(lookup) INSTALL_REQUIRES_ALL = get_reqs(lookup, "INSTALL_REQUIRES_ALL") + INSTALL_REQUIRES_CHECKS = get_reqs(lookup, "INSTALL_REQUIRES_CHECKS") TESTS_REQUIRES = get_reqs(lookup, "TESTS_REQUIRES") setup( @@ -89,7 +90,10 @@ def get_reqs(lookup=None, key="INSTALL_REQUIRES"): install_requires=INSTALL_REQUIRES, setup_requires=["pytest-runner"], tests_require=TESTS_REQUIRES, - extras_require={"all": [INSTALL_REQUIRES_ALL]}, + extras_require={ + "all": [INSTALL_REQUIRES_ALL], + "checks": [INSTALL_REQUIRES_CHECKS], + }, classifiers=[ "Intended Audience :: Science/Research", "Intended Audience :: Developers",