From eed82ce7adf140e17963374b45fcabe908049faf Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Wed, 5 Aug 2026 14:39:08 -0400 Subject: [PATCH 1/2] chore: setup dependabot --- .github/dependabot.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..d960e62 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,32 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directories: + - "/" + schedule: + interval: "monthly" + groups: + github-actions: + patterns: + - "*" + cooldown: + default-days: 7 + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "monthly" + groups: + runtime: + dependency-type: "production" + exclude-patterns: + - "ccbr_tools" + build: + patterns: + - "ccbr_tools" + - "setuptools" + - "setuptools-scm" + - "wheel" + dev: + dependency-type: "development" + cooldown: + default-days: 7 From 16d1bdcce6e3164aa6c5f137877c31616b5560b9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 5 Aug 2026 18:43:37 +0000 Subject: [PATCH 2/2] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/extract_value_from_json.py | 2 +- scripts/extract_value_from_yaml.py | 3 +- scripts/filter_bam_by_readids.py | 3 +- scripts/filter_fastq_by_readids_highmem.py | 3 +- scripts/filter_fastq_by_readids_highmem_pe.py | 3 +- src/ccbr_tools/GSEA/deg2gs.py | 25 +++++++------- src/ccbr_tools/GSEA/multitext2excel.py | 25 ++++++-------- src/ccbr_tools/GSEA/ncbr_huse.py | 16 +++------ src/ccbr_tools/__main__.py | 8 ++--- src/ccbr_tools/gb2gtf.py | 3 +- src/ccbr_tools/github.py | 3 +- src/ccbr_tools/homologfinder/hf.py | 8 ++--- src/ccbr_tools/hooks/__main__.py | 4 +-- src/ccbr_tools/intersect.py | 1 - src/ccbr_tools/jobby.py | 12 +++---- src/ccbr_tools/jobinfo.py | 18 +++++----- src/ccbr_tools/paths.py | 8 ++--- src/ccbr_tools/peek.py | 12 +++---- src/ccbr_tools/pipeline/__init__.py | 2 +- src/ccbr_tools/pipeline/cache.py | 16 ++++----- src/ccbr_tools/pipeline/hpc.py | 2 +- src/ccbr_tools/pipeline/nextflow.py | 12 +++---- src/ccbr_tools/pipeline/util.py | 33 +++++++------------ src/ccbr_tools/pkg_util.py | 13 ++++---- src/ccbr_tools/send_email.py | 2 +- src/ccbr_tools/software.py | 10 +++--- src/ccbr_tools/spooker.py | 7 ++-- src/ccbr_tools/versions.py | 2 +- tests/test_cli.py | 5 +-- tests/test_gb2gtf.py | 2 +- tests/test_github.py | 3 +- tests/test_gsea.py | 6 ++-- tests/test_gsea_cli.py | 2 +- tests/test_homologfinder.py | 2 +- tests/test_hooks.py | 2 +- tests/test_intersect.py | 1 + tests/test_jobby.py | 15 +++++---- tests/test_jobinfo.py | 3 +- tests/test_module_list.py | 2 +- tests/test_nextflow.py | 7 ++-- tests/test_peek.py | 2 +- tests/test_pipeline_cache.py | 2 +- tests/test_pipeline_util.py | 6 ++-- tests/test_pkg_util.py | 3 +- tests/test_scripts.py | 2 +- tests/test_shell.py | 2 +- tests/test_software.py | 14 ++++---- tests/test_spooker.py | 10 ++---- tests/test_templates.py | 3 +- tests/test_versions.py | 10 +++--- 50 files changed, 168 insertions(+), 192 deletions(-) diff --git a/scripts/extract_value_from_json.py b/scripts/extract_value_from_json.py index 3acbd62..7f8cb5b 100755 --- a/scripts/extract_value_from_json.py +++ b/scripts/extract_value_from_json.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -import json import argparse +import json parser = argparse.ArgumentParser(description="extract value for key from JSON") parser.add_argument("-j", dest="json", required=True, help="input JSON file") diff --git a/scripts/extract_value_from_yaml.py b/scripts/extract_value_from_yaml.py index 56f7b92..31012a0 100755 --- a/scripts/extract_value_from_yaml.py +++ b/scripts/extract_value_from_yaml.py @@ -1,7 +1,8 @@ #!/usr/bin/env python -import yaml import argparse +import yaml + parser = argparse.ArgumentParser(description="extract value for key from YAML") parser.add_argument("-y", dest="yaml", required=True, help="input YAML file") parser.add_argument( diff --git a/scripts/filter_bam_by_readids.py b/scripts/filter_bam_by_readids.py index c2df125..19f5242 100755 --- a/scripts/filter_bam_by_readids.py +++ b/scripts/filter_bam_by_readids.py @@ -1,7 +1,8 @@ #!/usr/bin/env python -import pysam import argparse +import pysam + parser = argparse.ArgumentParser(description="Filter BAM by readids") parser.add_argument( "--inputBAM", dest="inputBAM", type=str, required=True, help="input BAM file" diff --git a/scripts/filter_fastq_by_readids_highmem.py b/scripts/filter_fastq_by_readids_highmem.py index 2767296..43ce62f 100755 --- a/scripts/filter_fastq_by_readids_highmem.py +++ b/scripts/filter_fastq_by_readids_highmem.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3 -import HTSeq import argparse import os +import HTSeq + def get_sname(s): """Return the sample name from the input path.""" diff --git a/scripts/filter_fastq_by_readids_highmem_pe.py b/scripts/filter_fastq_by_readids_highmem_pe.py index dff589c..1802a51 100755 --- a/scripts/filter_fastq_by_readids_highmem_pe.py +++ b/scripts/filter_fastq_by_readids_highmem_pe.py @@ -1,8 +1,9 @@ #!/usr/bin/env python -import HTSeq import argparse import os +import HTSeq + def get_sname(s): """Return the sample name from the input path.""" diff --git a/src/ccbr_tools/GSEA/deg2gs.py b/src/ccbr_tools/GSEA/deg2gs.py index ac4c91c..23dc26e 100755 --- a/src/ccbr_tools/GSEA/deg2gs.py +++ b/src/ccbr_tools/GSEA/deg2gs.py @@ -20,15 +20,16 @@ __version__ = "1.1" __copyright__ = "No copyright protection, can be used freely" -import sys +import argparse +import datetime import os import re -import datetime +import sys +from argparse import RawTextHelpFormatter + import pandas as pd -import argparse -from argparse import RawTextHelpFormatter -from .ncbr_huse import send_update, err_out +from .ncbr_huse import err_out, send_update #################################### @@ -190,9 +191,7 @@ def main(): log.write(" ".join(sys.argv) + "\n") log.write("deg2gs.py version " + __version__ + "\n") log.write( - "Exporting genes to {}, n={}, p={}, q={}, ".format( - outfile.name, nhits, pvalue, qvalue - ) + f"Exporting genes to {outfile.name}, n={nhits}, p={pvalue}, q={qvalue}, " ) log.flush() @@ -214,9 +213,7 @@ def main(): # if df.shape[1] != len(in_cols): errMsg = ( - '\nYour input file does not match the expected format "{}".\n'.format( - fformat - ) + f'\nYour input file does not match the expected format "{fformat}".\n' + "Please check the file or the selected format and try again\n." ) if keepLog: @@ -229,7 +226,7 @@ def main(): # split the ensemblID|Gene if necessary if fformat == "topTable": # and method == 'gsea': - df["gene"] = [re.sub("^.*\|", "", i) for i in df.index.values.tolist()] + df["gene"] = [re.sub(r"^.*\|", "", i) for i in df.index.values.tolist()] # Filter the df to the p-values, FDR, and number of hits specified df = filter_by_p(df, nhits, pvalue, qvalue) @@ -253,11 +250,11 @@ def main(): # Close out the log file # if keepLog: - send_update("deg2gs.py successfully completed. {} written.".format(fname), log) + send_update(f"deg2gs.py successfully completed. {fname} written.", log) send_update(str(datetime.datetime.now()) + "\n", log) log.close() else: - print("deg2gs.py successfully completed. {} written.".format(fname)) + print(f"deg2gs.py successfully completed. {fname} written.") if __name__ == "__main__": diff --git a/src/ccbr_tools/GSEA/multitext2excel.py b/src/ccbr_tools/GSEA/multitext2excel.py index 8975923..afba632 100755 --- a/src/ccbr_tools/GSEA/multitext2excel.py +++ b/src/ccbr_tools/GSEA/multitext2excel.py @@ -18,23 +18,22 @@ __copyright__ = "No copyright protection, can be used freely" # import csv -import sys -import os -import re -import datetime -import pandas as pd -import glob - # import scipy # import numpy - import argparse +import datetime +import glob +import os +import re +import sys from argparse import RawTextHelpFormatter + +import pandas as pd + from .ncbr_huse import ( send_update, ) - #################################### # # Functions @@ -150,11 +149,7 @@ def main(): sheet_name = re.sub(indir + "/", "", filename).split(splitter)[0] if firstsplitter != "": sheet_name = sheet_name.split(firstsplitter)[1] - print( - "Writing data from input file: {} to output tab: {}".format( - filename, sheet_name - ) - ) + print(f"Writing data from input file: {filename} to output tab: {sheet_name}") # Read in the data df = pd.read_csv(filename, sep=delimiter, header=0, encoding="unicode_escape") @@ -169,7 +164,7 @@ def main(): # Close out the log file # send_update( - "multitext2excel.py successfully completed. {} written.".format(outfile.name), + f"multitext2excel.py successfully completed. {outfile.name} written.", log, ) send_update(str(datetime.datetime.now()) + "\n", log) diff --git a/src/ccbr_tools/GSEA/ncbr_huse.py b/src/ccbr_tools/GSEA/ncbr_huse.py index cf793fd..509d2d8 100755 --- a/src/ccbr_tools/GSEA/ncbr_huse.py +++ b/src/ccbr_tools/GSEA/ncbr_huse.py @@ -10,10 +10,10 @@ __version__ = "1.0.0" __copyright__ = "none" -import sys import os import re import subprocess +import sys try: import MySQLdb @@ -139,27 +139,19 @@ def pause_for_input(txt, contkey="y", quitkey="q", log=None): err_out("User elected to quit. Exiting...\n", log) # if none, just return the input - if contkey is None: - result = answer - - # if there is a contkey, then be sure it is correctly typed - elif answer == contkey: + if contkey is None or answer == contkey: result = answer else: # give them additional help and increment the answer count - reminder = "Note: only {} to continue and {} to quit are valid options.\nPlease try again.\n".format( - contkey, quitkey - ) + reminder = f"Note: only {contkey} to continue and {quitkey} to quit are valid options.\nPlease try again.\n" if answer_cnt == 0: txt = "\n" + txt + "\n" + reminder # Otherwise 3 strikes and exit from the loop if answer_cnt == 2: err_out( - "User failed to continue ({}) or quit ({}) three times in a row. Exiting...".format( - contkey, quitkey - ), + f"User failed to continue ({contkey}) or quit ({quitkey}) three times in a row. Exiting...", log, ) diff --git a/src/ccbr_tools/__main__.py b/src/ccbr_tools/__main__.py index 95e931a..976f2a5 100755 --- a/src/ccbr_tools/__main__.py +++ b/src/ccbr_tools/__main__.py @@ -2,20 +2,19 @@ Entry point for CCBR Tools """ -import click import cffconvert.cli.cli +import click from .pkg_util import ( + CustomClickGroup, get_project_scripts, get_version, print_citation, repo_base, - CustomClickGroup, ) from .send_email import send_email_msg -from .templates import use_quarto_ext, get_quarto_extensions from .software import install as install_software - +from .templates import get_quarto_extensions, use_quarto_ext all_scripts = "All installed tools:\n" + "\n".join( [f" {cmd}" for cmd in get_project_scripts()] @@ -37,7 +36,6 @@ def cli(): https://ccbr.github.io/Tools/ """ - pass @click.command() diff --git a/src/ccbr_tools/gb2gtf.py b/src/ccbr_tools/gb2gtf.py index efc8834..117ac4d 100755 --- a/src/ccbr_tools/gb2gtf.py +++ b/src/ccbr_tools/gb2gtf.py @@ -9,8 +9,9 @@ # Usage:python gb2gtf.py sequence.gb > sequence.gtf import sys -from Bio import SeqIO + import Bio +from Bio import SeqIO def main(): diff --git a/src/ccbr_tools/github.py b/src/ccbr_tools/github.py index 095dfb5..9c801cf 100644 --- a/src/ccbr_tools/github.py +++ b/src/ccbr_tools/github.py @@ -10,6 +10,7 @@ """ import warnings + from .pkg_util import get_url_json @@ -64,7 +65,7 @@ def get_user_info(user_login): try: user_info = get_url_json(url) except ConnectionError as e: - warnings.warn(f"Could not retrieve user info for {user_login}. {str(e)}") + warnings.warn(f"Could not retrieve user info for {user_login}. {e!s}") return user_info diff --git a/src/ccbr_tools/homologfinder/hf.py b/src/ccbr_tools/homologfinder/hf.py index f70e464..7f2f3b5 100755 --- a/src/ccbr_tools/homologfinder/hf.py +++ b/src/ccbr_tools/homologfinder/hf.py @@ -25,13 +25,14 @@ import argparse import importlib.resources -import pandas as pd import sys +import pandas as pd + def exit_w_msg(message): """Gracefully exit with proper message""" - print("{} : EXITING!!".format(__file__)) + print(f"{__file__} : EXITING!!") print(message) sys.exit() @@ -42,7 +43,6 @@ def check_help(parser): print(__doc__) parser.print_help() parser.exit() - return def collect_args(): @@ -54,7 +54,7 @@ def collect_args(): # add version parser.add_argument( - "-v", "--version", action="version", version="%(prog)s {}".format(__version__) + "-v", "--version", action="version", version=f"%(prog)s {__version__}" ) # add joblist diff --git a/src/ccbr_tools/hooks/__main__.py b/src/ccbr_tools/hooks/__main__.py index 55ae98e..729c916 100644 --- a/src/ccbr_tools/hooks/__main__.py +++ b/src/ccbr_tools/hooks/__main__.py @@ -3,9 +3,10 @@ """ import click + from ..pkg_util import ( - get_version, CustomClickGroup, + get_version, ) from .detect_absolute_paths import detect_absolute_paths from .sync_nextflow_version import sync_nextflow_version @@ -25,7 +26,6 @@ def cli(): https://ccbr.github.io/Tools/hooks """ - pass cli.add_command(detect_absolute_paths) diff --git a/src/ccbr_tools/intersect.py b/src/ccbr_tools/intersect.py index 8f4d375..5f2a4b1 100755 --- a/src/ccbr_tools/intersect.py +++ b/src/ccbr_tools/intersect.py @@ -7,7 +7,6 @@ intersect file1 file2 """ -from __future__ import print_function import sys diff --git a/src/ccbr_tools/jobby.py b/src/ccbr_tools/jobby.py index 152eecc..07bfc90 100755 --- a/src/ccbr_tools/jobby.py +++ b/src/ccbr_tools/jobby.py @@ -44,9 +44,6 @@ ``` """ -from .pkg_util import get_version -from .paths import glob_files - import itertools import json import os @@ -55,6 +52,9 @@ import sys import warnings +from .paths import glob_files +from .pkg_util import get_version + # Graceful imports try: import pandas as pd @@ -123,7 +123,7 @@ def parse_time_to_seconds(t: str): s = parts[0] total_seconds = int( - round((int(days) * 86400 + int(h) * 3600 + int(m) * 60 + s)) + round(int(days) * 86400 + int(h) * 3600 + int(m) * 60 + s) ) except ValueError: warnings.warn(f"❌ Invalid time format: {t}. Time will be set to NaN.") @@ -171,7 +171,7 @@ def extract_jobids_from_file(filepath): job_ids.append(match_nextflow.group(1)) except FileNotFoundError: warnings.warn(f"❌ File not found: {filepath}") - return list(sorted(set(job_ids))) # deduplicate + return sorted(set(job_ids)) # deduplicate def list_records( @@ -242,7 +242,7 @@ def get_sacct_info( # If this is .batch, update resource usage fields if step_type.endswith(".batch"): for resource_field in ("MaxRSS", "AveRSS", "MaxVMSize"): - if resource_field in record_raw and record_raw[resource_field]: + if record_raw.get(resource_field): job_records[base_jobid][resource_field] = record_raw[ resource_field ] diff --git a/src/ccbr_tools/jobinfo.py b/src/ccbr_tools/jobinfo.py index 65da9eb..7c65261 100755 --- a/src/ccbr_tools/jobinfo.py +++ b/src/ccbr_tools/jobinfo.py @@ -24,12 +24,13 @@ __email__ = "vishal.koparde@nih.gov" import argparse -import subprocess +import datetime import json import os -import datetime -import time +import subprocess import sys +import time + import pandas as pd # SHORT_FIELDS used to display on screen @@ -45,7 +46,7 @@ def exit_w_msg(message): """Gracefully exit with proper message""" - print("{} : EXITING!!".format(__file__)) + print(f"{__file__} : EXITING!!") print(message) sys.exit() @@ -56,7 +57,6 @@ def check_help(parser): print(__doc__) parser.print_help() parser.exit() - return def check_host(): @@ -79,7 +79,7 @@ def collect_args(): # add version parser.add_argument( - "-v", "--version", action="version", version="%(prog)s {}".format(__version__) + "-v", "--version", action="version", version=f"%(prog)s {__version__}" ) # add joblist @@ -126,7 +126,7 @@ def collect_args(): if args.output: args.output = os.path.abspath(args.output) if not os.access(os.path.dirname(args.output), os.W_OK): - msg = "File is not writable: {}".format(args.output) + msg = f"File is not writable: {args.output}" exit_w_msg(msg) if args.joblist and args.snakemakelog: @@ -142,7 +142,7 @@ def collect_args(): cmd = ( 'grep "external jobid" ' + args.snakemakelog.name - + ' | awk \'{print $NF}\' | sed "s/\'//g" | sed "s/\.//g"' + + ' | awk \'{print $NF}\' | sed "s/\'//g" | sed "s/\\.//g"' ) p1 = subprocess.run(cmd, capture_output=True, text=True, shell=True) args.joblist = p1.stdout.strip().split("\n") @@ -292,7 +292,7 @@ def get_jobinfo(args): columns=LONG_FIELDS.split(","), ) except OSError: - msg = "File is not writable: {}".format(args.output) + msg = f"File is not writable: {args.output}" exit_w_msg(msg) return p1_table diff --git a/src/ccbr_tools/paths.py b/src/ccbr_tools/paths.py index 1c643f4..498ba87 100644 --- a/src/ccbr_tools/paths.py +++ b/src/ccbr_tools/paths.py @@ -56,11 +56,9 @@ def get_disk_usage(tree_dict, pipeline_outdir): """Get disk usage.""" try: report = next( - ( - item - for item in tree_dict - if isinstance(item, dict) and item.get("type") == "report" - ) + item + for item in tree_dict + if isinstance(item, dict) and item.get("type") == "report" ) dir_size = report.get("size", math.nan) except StopIteration: # occurs when there is no report in the tree dict diff --git a/src/ccbr_tools/peek.py b/src/ccbr_tools/peek.py index 75cae51..60eaf91 100755 --- a/src/ccbr_tools/peek.py +++ b/src/ccbr_tools/peek.py @@ -5,9 +5,8 @@ peek [buffer] """ -from __future__ import print_function -from pathlib import Path import sys +from pathlib import Path def usage(): @@ -29,7 +28,6 @@ def pargs(): sys.argv[1] except IndexError: usage() - return def max_string(data): @@ -43,7 +41,7 @@ def max_string(data): def print_header(filename, length): """Print filenames and divider""" - print("# {}".format(filename)) + print(f"# {filename}") print("{}".format("=" * length)) @@ -75,7 +73,7 @@ def pprint(headlist, data, linelength, fn): # Calculate spacing for justifying to the right insert_spaces = justify(len(column), len(value), linelength, rownumber) - print("{} {}{}{}".format(rownumber, column, insert_spaces, value)) + print(f"{rownumber} {column}{insert_spaces}{value}") def peek(filename, buffer, delim="\t"): @@ -85,9 +83,9 @@ def peek(filename, buffer, delim="\t"): """Peek at the input data.""" try: fh = open(filename, "r") - except IOError as e: + except OSError as e: # File does not exist - print("\n{}\nPlease check you filename!\n\n".format(e)) + print(f"\n{e}\nPlease check you filename!\n\n") usage() headerlist = fh.readline().split(delim) diff --git a/src/ccbr_tools/pipeline/__init__.py b/src/ccbr_tools/pipeline/__init__.py index 7af06ee..486f029 100755 --- a/src/ccbr_tools/pipeline/__init__.py +++ b/src/ccbr_tools/pipeline/__init__.py @@ -72,7 +72,7 @@ def count_samples(cls, tree_str): nsamples = len(sample_names) except Exception as err: warnings.warn( - f"Could not determine number of samples. See original error message below:\n{repr(err)}" + f"Could not determine number of samples. See original error message below:\n{err!r}" ) return nsamples, sample_names diff --git a/src/ccbr_tools/pipeline/cache.py b/src/ccbr_tools/pipeline/cache.py index bc69cb5..a45f8b6 100755 --- a/src/ccbr_tools/pipeline/cache.py +++ b/src/ccbr_tools/pipeline/cache.py @@ -81,9 +81,7 @@ def image_cache(sub_args, config): # If local sif does not exist on in cache, print warning # and default to pulling from URI in config/containers/images.json print( - 'Warning: Local image "{}" does not exist in singularity cache'.format( - sif - ), + f'Warning: Local image "{sif}" does not exist in singularity cache', file=sys.stderr, ) else: @@ -115,10 +113,10 @@ def check_cache(parser, cache, *args, **kwargs): elif os.path.isfile(cache): # Cache directory exists as file, raise error parser.error( - """\n\t\x1b[6;37;41mFatal: Failed to provided a valid singularity cache!\x1b[0m + f"""\n\t\x1b[6;37;41mFatal: Failed to provided a valid singularity cache!\x1b[0m The provided --singularity-cache already exists on the filesystem as a file. - Please run {} again with a different --singularity-cache location. - """.format(sys.argv[0]) + Please run {sys.argv[0]} again with a different --singularity-cache location. + """ ) elif os.path.isdir(cache): # Provide cache exists as directory @@ -130,11 +128,11 @@ def check_cache(parser, cache, *args, **kwargs): ): # User does NOT own the cache directory, raise error parser.error( - """\n\t\x1b[6;37;41mFatal: Failed to provided a valid singularity cache!\x1b[0m + f"""\n\t\x1b[6;37;41mFatal: Failed to provided a valid singularity cache!\x1b[0m The provided --singularity-cache already exists on the filesystem with a different owner. Singularity strictly enforces that the cache directory is not shared across users. - Please run {} again with a different --singularity-cache location. - """.format(sys.argv[0]) + Please run {sys.argv[0]} again with a different --singularity-cache location. + """ ) return cache diff --git a/src/ccbr_tools/pipeline/hpc.py b/src/ccbr_tools/pipeline/hpc.py index 3268e31..b6b833c 100755 --- a/src/ccbr_tools/pipeline/hpc.py +++ b/src/ccbr_tools/pipeline/hpc.py @@ -9,8 +9,8 @@ import re import shutil -from .cache import get_singularity_cachedir, get_sif_cache_dir from ..shell import shell_run +from .cache import get_sif_cache_dir, get_singularity_cachedir class Cluster: diff --git a/src/ccbr_tools/pipeline/nextflow.py b/src/ccbr_tools/pipeline/nextflow.py index c9daa39..5f817de 100755 --- a/src/ccbr_tools/pipeline/nextflow.py +++ b/src/ccbr_tools/pipeline/nextflow.py @@ -13,8 +13,8 @@ from ..pkg_util import msg_box from ..shell import shell_run from ..templates import use_template -from .util import copy_config from .hpc import get_hpc +from .util import copy_config def init(output, repo_base, pipeline_name="pipeline"): @@ -83,9 +83,7 @@ def run( prev_arg = arg # make sure profile matches biowulf or frce profiles = ( - set(args_dict["-profile"].split(",")) - if "-profile" in args_dict.keys() - else set() + set(args_dict["-profile"].split(",")) if "-profile" in args_dict else set() ) if mode == "slurm": profiles.add("slurm") @@ -95,16 +93,16 @@ def run( args_dict["-profile"] = ",".join(sorted(profiles)) # use -resume by default, or do not use resume if force_all is True - if force_all and "-resume" in args_dict.keys(): + if force_all and "-resume" in args_dict: args_dict.pop("-resume") - elif not force_all and "-resume" not in args_dict.keys(): + elif not force_all and "-resume" not in args_dict: args_dict["-resume"] = "" nextflow_command = " ".join( ["nextflow", "run", nextfile_path] + [f"{k} {v}" for k, v in args_dict.items()] ) # Print a preview before launching the actual run - if "-preview" not in args_dict.keys(): + if "-preview" not in args_dict: preview_command = ( (f'bash -c "module load {hpc_modules} && {nextflow_command} -preview"') if hpc and hpc_modules diff --git a/src/ccbr_tools/pipeline/util.py b/src/ccbr_tools/pipeline/util.py index 231cf73..3fc2c0f 100755 --- a/src/ccbr_tools/pipeline/util.py +++ b/src/ccbr_tools/pipeline/util.py @@ -4,20 +4,21 @@ import collections import datetime -import shutil -import sys +import glob import hashlib import json -import glob import os import pathlib import re +import shutil import stat import subprocess +import sys import warnings + import yaml -from ..pkg_util import repo_base, msg +from ..pkg_util import msg, repo_base from .hpc import get_hpcname @@ -139,13 +140,9 @@ def permissions(parser, path, *args, **kwargs): str: Returns absolute path if it exists and permissions are correct. """ if not exists(path): - parser.error( - "Path '{}' does not exists! Failed to provide valid input.".format(path) - ) + parser.error(f"Path '{path}' does not exists! Failed to provide valid input.") if not os.access(path, *args, **kwargs): - parser.error( - "Path '{}' exists, but cannot read path due to permissions!".format(path) - ) + parser.error(f"Path '{path}' exists, but cannot read path due to permissions!") return os.path.abspath(path) @@ -266,17 +263,13 @@ def require(cmds, suggestions, path=None): if not available: error = True err( - """\x1b[6;37;41m\n\tFatal: {} is not in $PATH and is required during runtime! - └── Solution: please 'module load {}' and run again!\x1b[0m""".format( - cmds[i], suggestions[i] - ) + f"""\x1b[6;37;41m\n\tFatal: {cmds[i]} is not in $PATH and is required during runtime! + └── Solution: please 'module load {suggestions[i]}' and run again!\x1b[0m""" ) if error: fatal() - return - def safe_copy(source, target, resources=[]): """ @@ -360,9 +353,7 @@ def check_python_version(MIN_PYTHON=(3, 11)): try: assert sys.version_info >= MIN_PYTHON print( - "Python version: {0}.{1}.{2}".format( - sys.version_info.major, sys.version_info.minor, sys.version_info.micro - ) + f"Python version: {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" ) except AssertionError: exit( @@ -485,7 +476,7 @@ def rename(filename): if not converted: raise NameError( - """\n\tFatal: Failed to rename provided input '{}'! + f"""\n\tFatal: Failed to rename provided input '{filename}'! Cannot determine the extension of the user provided input file. Please rename the file list above before trying again. Here is example of acceptable input file extensions: @@ -494,7 +485,7 @@ def rename(filename): sampleName_1.fastq.gz sampleName_2.fastq.gz Please also check that your input files are gzipped? If they are not, please gzip them before proceeding again. - """.format(filename) + """ ) return renamed_filename diff --git a/src/ccbr_tools/pkg_util.py b/src/ccbr_tools/pkg_util.py index 50f4fa5..d1dbca9 100755 --- a/src/ccbr_tools/pkg_util.py +++ b/src/ccbr_tools/pkg_util.py @@ -2,18 +2,19 @@ Miscellaneous utility functions for the package """ -import click import datetime -import importlib.resources import importlib.metadata +import importlib.resources import os import pathlib -import requests -from time import localtime, strftime -import tomllib import uuid +from time import localtime, strftime from urllib.parse import urlparse +import click +import requests +import tomllib + class CustomClickGroup(click.Group): def format_epilog(self, ctx, formatter): @@ -156,7 +157,7 @@ def msg_box(splash, errmsg=None): """ msg("-" * (len(splash) + 4)) msg(f"| {splash} |") - msg(("-" * (len(splash) + 4))) + msg("-" * (len(splash) + 4)) if errmsg: click.echo("\n" + errmsg, err=True) diff --git a/src/ccbr_tools/send_email.py b/src/ccbr_tools/send_email.py index 3d41143..f63fb8c 100644 --- a/src/ccbr_tools/send_email.py +++ b/src/ccbr_tools/send_email.py @@ -5,9 +5,9 @@ Intended to run from biowulf """ -from email.message import EmailMessage import os import smtplib +from email.message import EmailMessage def send_email_msg( diff --git a/src/ccbr_tools/software.py b/src/ccbr_tools/software.py index 4f4bdec..c8696f2 100644 --- a/src/ccbr_tools/software.py +++ b/src/ccbr_tools/software.py @@ -1,6 +1,6 @@ from .pipeline.hpc import Cluster from .shell import shell_run -from .versions import match_semver, get_major_minor_version +from .versions import get_major_minor_version, match_semver class Software: @@ -88,7 +88,7 @@ def bash(software: Software, hpc: Cluster, branch_tag=None): class PythonTool(Software): def __init__(self, name, version): - super(PythonTool, self).__init__(name, version) + super().__init__(name, version) self.repo_name = name.replace("ccbr_", "") def install(self, hpc: Cluster, branch_tag=None): @@ -103,7 +103,7 @@ def url(self): class BashTool(Software): def __init__(self, name, version): - super(BashTool, self).__init__(name, version) + super().__init__(name, version) def install(self, hpc: Cluster, branch_tag=None): """Return the install command.""" @@ -112,7 +112,7 @@ def install(self, hpc: Cluster, branch_tag=None): class Nextflow(Software): def __init__(self, name, version): - super(Nextflow, self).__init__(name.upper(), version) + super().__init__(name.upper(), version) def path(self, hpc: Cluster): """Return the installation path.""" @@ -130,7 +130,7 @@ def install(self, hpc: Cluster, branch_tag=None): class Snakemake(Software): def __init__(self, name, version): - super(Snakemake, self).__init__(name.upper(), version) + super().__init__(name.upper(), version) def path(self, hpc: Cluster): """Return the installation path.""" diff --git a/src/ccbr_tools/spooker.py b/src/ccbr_tools/spooker.py index 2891d0a..2f66886 100644 --- a/src/ccbr_tools/spooker.py +++ b/src/ccbr_tools/spooker.py @@ -9,17 +9,18 @@ See [](`~ccbr_tools.spooker.spooker`) for the main function """ -import click import gzip import json import os import pathlib -from .paths import get_tree, load_tree, get_disk_usage, glob_files +import click + +from .jobby import jobby +from .paths import get_disk_usage, get_tree, glob_files, load_tree from .pipeline import count_pipeline_samples from .pipeline.hpc import Cluster, list_modules, parse_modules from .pkg_util import get_random_string, get_timestamp -from .jobby import jobby from .shell import get_groups, shell_run diff --git a/src/ccbr_tools/versions.py b/src/ccbr_tools/versions.py index 9777570..325dce0 100644 --- a/src/ccbr_tools/versions.py +++ b/src/ccbr_tools/versions.py @@ -6,8 +6,8 @@ import re import warnings -from .shell import shell_run from .pkg_util import get_url_json +from .shell import shell_run def get_current_hash(): diff --git a/tests/test_cli.py b/tests/test_cli.py index fafa01b..f30371b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,9 +1,10 @@ -from ccbr_tools.shell import shell_run - import os import pathlib + import pytest +from ccbr_tools.shell import shell_run + is_ci = ( os.environ.get("CI", "false") == "true" ) # Set CI to false if not in a CI environment diff --git a/tests/test_gb2gtf.py b/tests/test_gb2gtf.py index e07af49..4a8a2b3 100644 --- a/tests/test_gb2gtf.py +++ b/tests/test_gb2gtf.py @@ -1,6 +1,6 @@ import pytest -import ccbr_tools.gb2gtf as gb2gtf +from ccbr_tools import gb2gtf from ccbr_tools.shell import exec_in_context diff --git a/tests/test_github.py b/tests/test_github.py index 74c3d5e..54c8eec 100644 --- a/tests/test_github.py +++ b/tests/test_github.py @@ -1,6 +1,7 @@ -from ccbr_tools.github import print_contributor_images, get_user_info import pytest +from ccbr_tools.github import get_user_info, print_contributor_images + def test_print_contributor_images(): """Test print contributor images.""" diff --git a/tests/test_gsea.py b/tests/test_gsea.py index 517bbc4..819f8fe 100644 --- a/tests/test_gsea.py +++ b/tests/test_gsea.py @@ -1,9 +1,9 @@ -from ccbr_tools.shell import shell_run +import sys import pytest -import sys -import ccbr_tools.GSEA.ncbr_huse as ncbr_huse +from ccbr_tools.GSEA import ncbr_huse +from ccbr_tools.shell import shell_run def test_help_deg(): diff --git a/tests/test_gsea_cli.py b/tests/test_gsea_cli.py index 04f85b6..009517e 100644 --- a/tests/test_gsea_cli.py +++ b/tests/test_gsea_cli.py @@ -3,8 +3,8 @@ import pandas as pd import pytest -import ccbr_tools.GSEA.deg2gs as deg2gs import ccbr_tools.GSEA.multitext2excel as mt2excel +from ccbr_tools.GSEA import deg2gs def test_deg2gs_gsea_pipeliner(tmp_path, mocker): diff --git a/tests/test_homologfinder.py b/tests/test_homologfinder.py index 2f60469..69ccaa8 100644 --- a/tests/test_homologfinder.py +++ b/tests/test_homologfinder.py @@ -1,6 +1,6 @@ import argparse -import ccbr_tools.homologfinder.hf as hf +from ccbr_tools.homologfinder import hf from ccbr_tools.shell import shell_run diff --git a/tests/test_hooks.py b/tests/test_hooks.py index 414a514..00a08e6 100644 --- a/tests/test_hooks.py +++ b/tests/test_hooks.py @@ -7,8 +7,8 @@ import pytest from click.testing import CliRunner -from ccbr_tools.hooks import detect_absolute_paths as hooks import ccbr_tools.hooks.__main__ as hooks_main +from ccbr_tools.hooks import detect_absolute_paths as hooks from ccbr_tools.hooks.sync_nextflow_version import ( sync_nextflow_version, update_manifest_version, diff --git a/tests/test_intersect.py b/tests/test_intersect.py index d582943..cbe1c28 100644 --- a/tests/test_intersect.py +++ b/tests/test_intersect.py @@ -1,4 +1,5 @@ import pytest + import ccbr_tools.intersect from ccbr_tools.shell import exec_in_context diff --git a/tests/test_jobby.py b/tests/test_jobby.py index 45d4741..e520045 100644 --- a/tests/test_jobby.py +++ b/tests/test_jobby.py @@ -1,20 +1,21 @@ import gzip import os -import numpy as np -import pandas as pd import pickle import pprint + +import numpy as np +import pandas as pd import pytest from ccbr_tools.jobby import ( - jobby, - parse_time_to_seconds, - parse_mem_to_gb, extract_jobids_from_file, - list_records, - records_to_df, format_df, get_job_logs, + jobby, + list_records, + parse_mem_to_gb, + parse_time_to_seconds, + records_to_df, ) from ccbr_tools.pipeline.hpc import get_hpcname from ccbr_tools.shell import shell_run diff --git a/tests/test_jobinfo.py b/tests/test_jobinfo.py index db757c7..19485fa 100644 --- a/tests/test_jobinfo.py +++ b/tests/test_jobinfo.py @@ -1,10 +1,11 @@ import argparse import json import sys + import pytest +from ccbr_tools.jobinfo import check_host, get_jobinfo, mem2gb, time2sec from ccbr_tools.shell import shell_run -from ccbr_tools.jobinfo import get_jobinfo, check_host, mem2gb, time2sec def test_jobinfo_cli(): diff --git a/tests/test_module_list.py b/tests/test_module_list.py index ceec912..571f5ee 100644 --- a/tests/test_module_list.py +++ b/tests/test_module_list.py @@ -1,5 +1,5 @@ +from ccbr_tools.pipeline.hpc import is_loaded, parse_modules from ccbr_tools.shell import shell_run -from ccbr_tools.pipeline.hpc import parse_modules, is_loaded def test_module_list_cli(): diff --git a/tests/test_nextflow.py b/tests/test_nextflow.py index 916dc4b..8e9cd19 100644 --- a/tests/test_nextflow.py +++ b/tests/test_nextflow.py @@ -1,10 +1,11 @@ import os import pathlib -import pytest import subprocess -from ccbr_tools.pipeline.nextflow import run, init -from ccbr_tools.pipeline.hpc import Biowulf, FRCE +import pytest + +from ccbr_tools.pipeline.hpc import FRCE, Biowulf +from ccbr_tools.pipeline.nextflow import init, run from ccbr_tools.shell import exec_in_context diff --git a/tests/test_peek.py b/tests/test_peek.py index fa50c2f..4223f82 100644 --- a/tests/test_peek.py +++ b/tests/test_peek.py @@ -1,4 +1,4 @@ -import ccbr_tools.peek as peek +from ccbr_tools import peek from ccbr_tools.shell import exec_in_context diff --git a/tests/test_pipeline_cache.py b/tests/test_pipeline_cache.py index 1a0f170..a65ddda 100644 --- a/tests/test_pipeline_cache.py +++ b/tests/test_pipeline_cache.py @@ -1,10 +1,10 @@ import argparse from ccbr_tools.pipeline.cache import ( + check_cache, get_sif_cache_dir, get_singularity_cachedir, image_cache, - check_cache, ) diff --git a/tests/test_pipeline_util.py b/tests/test_pipeline_util.py index 80c5054..d41a68a 100644 --- a/tests/test_pipeline_util.py +++ b/tests/test_pipeline_util.py @@ -3,13 +3,13 @@ import pathlib from ccbr_tools.pipeline.util import ( - copy_config, - get_tmp_dir, _get_file_mtime, + copy_config, + exists, get_genomes_dict, + get_tmp_dir, md5sum, permissions, - exists, which, ) from ccbr_tools.pkg_util import repo_base diff --git a/tests/test_pkg_util.py b/tests/test_pkg_util.py index 472e24c..333135b 100644 --- a/tests/test_pkg_util.py +++ b/tests/test_pkg_util.py @@ -1,6 +1,7 @@ -from ccbr_tools.pkg_util import repo_base, get_url_json import pytest +from ccbr_tools.pkg_util import get_url_json, repo_base + def test_repo_base(): """Test repo base.""" diff --git a/tests/test_scripts.py b/tests/test_scripts.py index eab6147..9551ea2 100644 --- a/tests/test_scripts.py +++ b/tests/test_scripts.py @@ -1,5 +1,5 @@ -from ccbr_tools.shell import shell_run from ccbr_tools.pipeline.hpc import get_hpcname +from ccbr_tools.shell import shell_run def test_scripts_help(): diff --git a/tests/test_shell.py b/tests/test_shell.py index efcab90..aee702e 100644 --- a/tests/test_shell.py +++ b/tests/test_shell.py @@ -1,4 +1,4 @@ -from ccbr_tools.shell import shell_run, exec_in_context, concat_newline +from ccbr_tools.shell import concat_newline, exec_in_context, shell_run def test_exec(): diff --git a/tests/test_software.py b/tests/test_software.py index 5a85185..01e5913 100644 --- a/tests/test_software.py +++ b/tests/test_software.py @@ -1,14 +1,14 @@ +import pytest + +from ccbr_tools.pipeline.hpc import Biowulf +from ccbr_tools.shell import exec_in_context from ccbr_tools.software import ( + FINAL_PERMISSIONS, + LATEST_SYMLINK, + SET_SYMLINK, Software, install, - SET_SYMLINK, - LATEST_SYMLINK, - FINAL_PERMISSIONS, ) -from ccbr_tools.pipeline.hpc import Biowulf -from ccbr_tools.shell import exec_in_context - -import pytest def test_python(): diff --git a/tests/test_spooker.py b/tests/test_spooker.py index 1687640..e44751a 100644 --- a/tests/test_spooker.py +++ b/tests/test_spooker.py @@ -1,8 +1,8 @@ -import pytest import gzip import json import subprocess +import pytest from ccbr_tools.spooker import spooker @@ -30,9 +30,7 @@ def test_spooker(data_dir_rel): "pipeline_version": "0.1.0", "sample_names": [], } - actual = { - k: v for k, v in spook_dat["pipeline_metadata"].items() if k in expected.keys() - } + actual = {k: v for k, v in spook_dat["pipeline_metadata"].items() if k in expected} assert actual == expected @@ -74,9 +72,7 @@ def test_spooker_cli(data_dir_rel): "sample_names": [], } actual_meta = { - k: v - for k, v in spook_dat["pipeline_metadata"].items() - if k in expected_meta.keys() + k: v for k, v in spook_dat["pipeline_metadata"].items() if k in expected_meta } assert expected_meta == actual_meta assert spook_dat.keys() == { diff --git a/tests/test_templates.py b/tests/test_templates.py index 78eead7..81bd640 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -1,9 +1,10 @@ import os import pathlib + import pytest -from ccbr_tools.templates import read_template, use_template, use_quarto_ext from ccbr_tools.pipeline.hpc import get_hpc +from ccbr_tools.templates import read_template, use_quarto_ext, use_template def test_read_template(): diff --git a/tests/test_versions.py b/tests/test_versions.py index 109ca2a..ce8de34 100644 --- a/tests/test_versions.py +++ b/tests/test_versions.py @@ -1,14 +1,14 @@ import pytest from ccbr_tools.versions import ( - get_releases, - get_latest_release_tag, - get_latest_release_hash, - get_tag_hash, - match_semver, check_version_increments_by_one, + get_latest_release_hash, + get_latest_release_tag, get_major_minor_version, + get_releases, + get_tag_hash, is_ancestor, + match_semver, )