Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: version specific profile config files (profile/config.v8+.yaml with profile/config.yaml as fallback that matches any version) #2498

Merged
merged 2 commits into from Oct 30, 2023
Merged

feat: version specific profile config files (profile/config.v8+.yaml with profile/config.yaml as fallback that matches any version) #2498

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0609149
docs
johanneskoester Oct 30, 2023
1288e77
fix config file path
johanneskoester Oct 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
19 changes: 3 additions & 16 deletions docs/executing/cli.rst
View file
Open in desktop
Expand Up @@ -133,7 +133,9 @@ The default profile to use when no ``--profile`` argument is specified can also
e.g. by specifying ``export SNAKEMAKE_PROFILE=myprofile`` in your ``~/.bashrc`` or the system wide shell defaults means that the ``--profile`` flag can be omitted.
In order unset the profile defined by this environment variable for individual runs without specifying and alternative profile you can provide the special value ``none``, i.e. ``--profile none``.

The profile folder is expected to contain a file ``config.yaml`` that defines default values for the Snakemake command line arguments.
The profile folder is expected to contain a configuration file that file that defines default values for the Snakemake command line arguments.
The file has to be named ``config.vX+.yaml`` with ``X`` denoting the minimum supported Snakemake major version (e.g. ``config.v8+.yaml``).
As fallback, it is also possible to provide a version agnostic ``config.yaml`` that matches any Snakemake version.
For example, the file

.. code-block:: yaml
Expand Down Expand Up @@ -231,18 +233,3 @@ All Options

All command line options can be printed by calling ``snakemake -h``.

.. _getting_started-bash_completion:

---------------
Bash Completion
---------------

Snakemake supports bash completion for filenames, rulenames and arguments.
To enable it globally, just append

.. code-block:: bash

`snakemake --bash-completion`

including the backticks to your ``.bashrc``.
This only works if the ``snakemake`` command is in your path.
125 changes: 51 additions & 74 deletions snakemake/cli.py
View file
Open in desktop
Expand Up @@ -246,24 +246,51 @@ def parse_jobs(jobs):
)


def get_profile_file(profile, file, return_default=False):
def get_profile_dir(profile: str) -> (Path, Path):
config_pattern = re.compile(r"config(.v(?P<min_major>\d+)\+)?.yaml")

def get_config_min_major(filename):
m = config_pattern.match(filename)
if m:
min_major = m.group("min_major")
if min_major is None:
return 0
min_major = int(min_major)

return min_major
return None

dirs = get_appdirs()
if os.path.exists(profile):
search_dirs = [os.path.dirname(profile)]
parent_dir = os.path.dirname(profile) or "."
search_dirs = [parent_dir]
profile = os.path.basename(profile)
else:
search_dirs = [os.getcwd(), dirs.user_config_dir, dirs.site_config_dir]
get_path = lambda d: os.path.join(d, profile, file)
for d in search_dirs:
p = get_path(d)
# "file" can actually be a full command. If so, `p` won't exist as the
# below would check if e.g. '/path/to/profile/script --arg1 val --arg2'
# exists. To fix this, we use shlex.split() to get the path to the
# script. We check for both, in case the path contains spaces or some
# other thing that would cause shlex.split() to mangle the path
# inaccurately.
if os.path.exists(p) or os.path.exists(shlex.split(p)[0]):
return p
d = Path(d)
files = os.listdir(d / profile)
curr_major = int(__version__.split(".")[0])
config_files = {
f: min_major
for f, min_major in zip(files, map(get_config_min_major, files))
if min_major is not None and curr_major >= min_major
}
if config_files:
config_file = max(config_files, key=config_files.get)
return d / profile, d / profile / config_file


def get_profile_file(profile_dir: Path, file, return_default=False):
p = profile_dir / file
# "file" can actually be a full command. If so, `p` won't exist as the
# below would check if e.g. '/path/to/profile/script --arg1 val --arg2'
# exists. To fix this, we use shlex.split() to get the path to the
# script. We check for both, in case the path contains spaces or some
# other thing that would cause shlex.split() to mangle the path
# inaccurately.
if p.exists() or os.path.exists(shlex.split(str(p))[0]):
return p

if return_default:
return file
Expand All @@ -276,14 +303,18 @@ def get_argument_parser(profiles=None):

dirs = get_appdirs()
config_files = []
profile_dir = None
if profiles:
for profile in profiles:
if profile == "":
print("Error: invalid profile name.", file=sys.stderr)
exit(1)

config_file = get_profile_file(profile, "config.yaml")
if config_file is None:
profile_entry = get_profile_dir(profile)
if profile_entry is not None:
profile_dir, config_file = profile_entry
config_files.append(config_file)
else:
print(
"Error: profile given but no config.yaml found. "
"Profile has to be given as either absolute path, relative "
Expand All @@ -294,7 +325,6 @@ def get_argument_parser(profiles=None):
file=sys.stderr,
)
exit(1)
config_files.append(config_file)

parser = snakemake.common.argparse.ArgumentParser(
description="Snakemake is a Python based language and execution "
Expand Down Expand Up @@ -1085,14 +1115,6 @@ def get_argument_parser(profiles=None):
"with --dry-run to list files without actually deleting anything. Note "
"that this will not recurse into subworkflows.",
)
group_utils.add_argument(
"--bash-completion",
action="store_true",
help="Output code to register bash completion for snakemake. Put the "
"following in your .bashrc (including the accents): "
"`snakemake --bash-completion` or issue it in an open terminal "
"session.",
)
group_utils.add_argument(
"--keep-incomplete",
action="store_true",
Expand Down Expand Up @@ -1567,7 +1589,7 @@ def get_argument_parser(profiles=None):
# Add namespaced arguments to parser for each plugin
_get_executor_plugin_registry().register_cli_args(parser)
StoragePluginRegistry().register_cli_args(parser)
return parser
return parser, profile_dir


def generate_parser_metadata(parser, args):
Expand All @@ -1583,7 +1605,7 @@ def generate_parser_metadata(parser, args):


def parse_args(argv):
parser = get_argument_parser()
parser, profile_dir = get_argument_parser()
args = parser.parse_args(argv)

snakefile = resolve_snakefile(args.snakefile, allow_missing=True)
Expand Down Expand Up @@ -1624,23 +1646,23 @@ def parse_args(argv):
file=sys.stderr,
)

parser = get_argument_parser(profiles=profiles)
parser, profile_dir = get_argument_parser(profiles=profiles)
args = parser.parse_args(argv)

def adjust_path(path_or_value):
if isinstance(path_or_value, str):
adjusted = get_profile_file(
args.profile, path_or_value, return_default=False
profile_dir, path_or_value, return_default=False
)
if adjusted is None:
if adjusted is None or os.path.exists(path_or_value):
return path_or_value
else:
return adjusted
else:
return path_or_value

# Update file paths to be relative to the profile if profile
# contains them.
# contains them and they don't exist without prepending it.
for key, _ in list(args._get_kwargs()):
setattr(args, key, adjust_path(getattr(args, key)))

Expand Down Expand Up @@ -1732,11 +1754,6 @@ def parse_rerun_triggers(values):
def args_to_api(args, parser):
"""Convert argparse args to API calls."""

if args.bash_completion:
cmd = b"complete -o bashdefault -C snakemake-bash-completion snakemake"
sys.stdout.buffer.write(cmd)
sys.exit(0)

# handle legacy executor names
if args.dryrun:
args.executor = "dryrun"
Expand Down Expand Up @@ -2011,43 +2028,3 @@ def main(argv=None):
print_exception(e)
sys.exit(1)
sys.exit(0 if success else 1)


def bash_completion(snakefile="Snakefile"):
"""Entry point for bash completion."""
if len(sys.argv) < 2:
print(
"Calculate bash completion for snakemake. This tool shall not be invoked by hand."
)
sys.exit(1)

def print_candidates(candidates):
if candidates:
candidates = sorted(set(candidates))
## Use bytes for avoiding '^M' under Windows.
sys.stdout.buffer.write(b"\n".join(s.encode() for s in candidates))

prefix = sys.argv[2]

if prefix.startswith("-"):
print_candidates(
action.option_strings[0]
for action in get_argument_parser()._actions
if action.option_strings and action.option_strings[0].startswith(prefix)
)
else:
candidates = []
files = glob.glob(f"{prefix}*")
if files:
candidates.extend(files)
if os.path.exists(snakefile):
workflow = Workflow(snakefile=snakefile)
workflow.include(snakefile)

candidates.extend(
[file for file in workflow.concrete_files if file.startswith(prefix)]
+ [rule.name for rule in workflow.rules if rule.name.startswith(prefix)]
)
if len(candidates) > 0:
print_candidates(candidates)
sys.exit(0)