Merge pull request #429 from pepkit/dev_126_selectors_on_flags

Dev 126 selectors on flags
pepkit · Dec 14, 2023 · 9af6ebe · 9af6ebe
2 parents 555f30c + ca97957
commit 9af6ebe
Show file tree

Hide file tree

Showing 7 changed files with 454 additions and 57 deletions.
diff --git a/docs/usage.md b/docs/usage.md
@@ -26,7 +26,7 @@ Each task is controlled by one of the following commands: `run`, `rerun`, `runp`
 Here you can see the command-line usage instructions for the main looper command and for each subcommand:
 ## `looper --help`
 ```console
-version: 1.5.2-dev
+version: 1.6.0
 usage: looper [-h] [--version] [--logfile LOGFILE] [--dbg] [--silent]
               [--verbosity V] [--logdev]
               {run,rerun,runp,table,report,destroy,check,clean,inspect,init,init-piface,link}
@@ -68,7 +68,7 @@ usage: looper run [-h] [-i] [-d] [-t S] [-x S] [-y S] [-f] [--divvy DIVCFG] [-p
                   [-c K [K ...]] [-u X] [-n N] [--looper-config LOOPER_CONFIG]
                   [-S YAML [YAML ...]] [-P YAML [YAML ...]] [-l N] [-k N]
                   [--sel-attr ATTR] [--sel-excl [E ...] | --sel-incl [I ...]]
-                  [-a A [A ...]]
+                  [--sel-flag [SELFLAG ...]] [--exc-flag [EXCFLAG ...]] [-a A [A ...]]
                   [config_file]
 
 Run or submit sample jobs.
@@ -111,14 +111,17 @@ sample selection arguments:
   --sel-attr ATTR                    Attribute for sample exclusion OR inclusion
   --sel-excl [E ...]                 Exclude samples with these values
   --sel-incl [I ...]                 Include only samples with these values
+  --sel-flag [SELFLAG ...]           Include samples with this flag status, e.g. completed
+  --exc-flag [EXCFLAG ...]           Exclude samples with this flag status, e.g. completed
 ```
 
 ## `looper runp --help`
 ```console
 usage: looper runp [-h] [-i] [-d] [-t S] [-x S] [-y S] [-f] [--divvy DIVCFG] [-p P] [-s S]
                    [-c K [K ...]] [--looper-config LOOPER_CONFIG] [-S YAML [YAML ...]]
                    [-P YAML [YAML ...]] [-l N] [-k N] [--sel-attr ATTR]
-                   [--sel-excl [E ...] | --sel-incl [I ...]] [-a A [A ...]]
+                   [--sel-excl [E ...] | --sel-incl [I ...]] [--sel-flag [SELFLAG ...]]
+                   [--exc-flag [EXCFLAG ...]] [-a A [A ...]]
                    [config_file]
 
 Run or submit project jobs.
@@ -159,6 +162,8 @@ sample selection arguments:
   --sel-attr ATTR                    Attribute for sample exclusion OR inclusion
   --sel-excl [E ...]                 Exclude samples with these values
   --sel-incl [I ...]                 Include only samples with these values
+  --sel-flag [SELFLAG ...]           Include samples with this flag status, e.g. completed
+  --exc-flag [EXCFLAG ...]           Exclude samples with this flag status, e.g. completed
 ```
 
 ## `looper rerun --help`
@@ -167,7 +172,7 @@ usage: looper rerun [-h] [-i] [-d] [-t S] [-x S] [-y S] [-f] [--divvy DIVCFG] [-
                     [-s S] [-c K [K ...]] [-u X] [-n N] [--looper-config LOOPER_CONFIG]
                     [-S YAML [YAML ...]] [-P YAML [YAML ...]] [-l N] [-k N]
                     [--sel-attr ATTR] [--sel-excl [E ...] | --sel-incl [I ...]]
-                    [-a A [A ...]]
+                    [--sel-flag [SELFLAG ...]] [--exc-flag [EXCFLAG ...]] [-a A [A ...]]
                     [config_file]
 
 Resubmit sample jobs with failed flags.
@@ -210,13 +215,16 @@ sample selection arguments:
   --sel-attr ATTR                    Attribute for sample exclusion OR inclusion
   --sel-excl [E ...]                 Exclude samples with these values
   --sel-incl [I ...]                 Include only samples with these values
+  --sel-flag [SELFLAG ...]           Include samples with this flag status, e.g. completed
+  --exc-flag [EXCFLAG ...]           Exclude samples with this flag status, e.g. completed
 ```
 
 ## `looper report --help`
 ```console
 usage: looper report [-h] [--looper-config LOOPER_CONFIG] [-S YAML [YAML ...]]
                      [-P YAML [YAML ...]] [-l N] [-k N] [--sel-attr ATTR]
-                     [--sel-excl [E ...] | --sel-incl [I ...]] [-a A [A ...]] [--project]
+                     [--sel-excl [E ...] | --sel-incl [I ...]] [--sel-flag [SELFLAG ...]]
+                     [--exc-flag [EXCFLAG ...]] [-a A [A ...]] [--project]
                      [config_file]
 
 Create browsable HTML report of project results.
@@ -243,13 +251,16 @@ sample selection arguments:
   --sel-attr ATTR                    Attribute for sample exclusion OR inclusion
   --sel-excl [E ...]                 Exclude samples with these values
   --sel-incl [I ...]                 Include only samples with these values
+  --sel-flag [SELFLAG ...]           Include samples with this flag status, e.g. completed
+  --exc-flag [EXCFLAG ...]           Exclude samples with this flag status, e.g. completed
 ```
 
 ## `looper table --help`
 ```console
 usage: looper table [-h] [--looper-config LOOPER_CONFIG] [-S YAML [YAML ...]]
                     [-P YAML [YAML ...]] [-l N] [-k N] [--sel-attr ATTR]
-                    [--sel-excl [E ...] | --sel-incl [I ...]] [-a A [A ...]] [--project]
+                    [--sel-excl [E ...] | --sel-incl [I ...]] [--sel-flag [SELFLAG ...]]
+                    [--exc-flag [EXCFLAG ...]] [-a A [A ...]] [--project]
                     [config_file]
 
 Write summary stats table for project samples.
@@ -276,13 +287,16 @@ sample selection arguments:
   --sel-attr ATTR                    Attribute for sample exclusion OR inclusion
   --sel-excl [E ...]                 Exclude samples with these values
   --sel-incl [I ...]                 Include only samples with these values
+  --sel-flag [SELFLAG ...]           Include samples with this flag status, e.g. completed
+  --exc-flag [EXCFLAG ...]           Exclude samples with this flag status, e.g. completed
 ```
 
 ## `looper inspect --help`
 ```console
 usage: looper inspect [-h] [--looper-config LOOPER_CONFIG] [-S YAML [YAML ...]]
                       [-P YAML [YAML ...]] [-l N] [-k N] [--sel-attr ATTR]
-                      [--sel-excl [E ...] | --sel-incl [I ...]] [-a A [A ...]]
+                      [--sel-excl [E ...] | --sel-incl [I ...]] [--sel-flag [SELFLAG ...]]
+                      [--exc-flag [EXCFLAG ...]] [-a A [A ...]]
                       [--sample-names [SAMPLE_NAMES ...]] [--attr-limit ATTR_LIMIT]
                       [config_file]
 
@@ -311,6 +325,8 @@ sample selection arguments:
   --sel-attr ATTR                    Attribute for sample exclusion OR inclusion
   --sel-excl [E ...]                 Exclude samples with these values
   --sel-incl [I ...]                 Include only samples with these values
+  --sel-flag [SELFLAG ...]           Include samples with this flag status, e.g. completed
+  --exc-flag [EXCFLAG ...]           Exclude samples with this flag status, e.g. completed
 ```
 
 ## `looper init --help`
@@ -339,7 +355,8 @@ options:
 usage: looper destroy [-h] [-d] [--force-yes] [--looper-config LOOPER_CONFIG]
                       [-S YAML [YAML ...]] [-P YAML [YAML ...]] [-l N] [-k N]
                       [--sel-attr ATTR] [--sel-excl [E ...] | --sel-incl [I ...]]
-                      [-a A [A ...]] [--project]
+                      [--sel-flag [SELFLAG ...]] [--exc-flag [EXCFLAG ...]] [-a A [A ...]]
+                      [--project]
                       [config_file]
 
 Remove output files of the project.
@@ -369,14 +386,17 @@ sample selection arguments:
   --sel-attr ATTR                    Attribute for sample exclusion OR inclusion
   --sel-excl [E ...]                 Exclude samples with these values
   --sel-incl [I ...]                 Include only samples with these values
+  --sel-flag [SELFLAG ...]           Include samples with this flag status, e.g. completed
+  --exc-flag [EXCFLAG ...]           Exclude samples with this flag status, e.g. completed
 ```
 
 ## `looper check --help`
 ```console
 usage: looper check [-h] [--describe-codes] [--itemized] [-f [F ...]]
                     [--looper-config LOOPER_CONFIG] [-S YAML [YAML ...]]
                     [-P YAML [YAML ...]] [-l N] [-k N] [--sel-attr ATTR]
-                    [--sel-excl [E ...] | --sel-incl [I ...]] [-a A [A ...]] [--project]
+                    [--sel-excl [E ...] | --sel-incl [I ...]] [--sel-flag [SELFLAG ...]]
+                    [--exc-flag [EXCFLAG ...]] [-a A [A ...]] [--project]
                     [config_file]
 
 Check flag status of current runs.
@@ -406,14 +426,16 @@ sample selection arguments:
   --sel-attr ATTR                    Attribute for sample exclusion OR inclusion
   --sel-excl [E ...]                 Exclude samples with these values
   --sel-incl [I ...]                 Include only samples with these values
+  --sel-flag [SELFLAG ...]           Include samples with this flag status, e.g. completed
+  --exc-flag [EXCFLAG ...]           Exclude samples with this flag status, e.g. completed
 ```
 
 ## `looper clean --help`
 ```console
 usage: looper clean [-h] [-d] [--force-yes] [--looper-config LOOPER_CONFIG]
                     [-S YAML [YAML ...]] [-P YAML [YAML ...]] [-l N] [-k N]
                     [--sel-attr ATTR] [--sel-excl [E ...] | --sel-incl [I ...]]
-                    [-a A [A ...]]
+                    [--sel-flag [SELFLAG ...]] [--exc-flag [EXCFLAG ...]] [-a A [A ...]]
                     [config_file]
 
 Run clean scripts of already processed jobs.
@@ -442,5 +464,7 @@ sample selection arguments:
   --sel-attr ATTR                    Attribute for sample exclusion OR inclusion
   --sel-excl [E ...]                 Exclude samples with these values
   --sel-incl [I ...]                 Include only samples with these values
+  --sel-flag [SELFLAG ...]           Include samples with this flag status, e.g. completed
+  --exc-flag [EXCFLAG ...]           Exclude samples with this flag status, e.g. completed
 ```
 
diff --git a/docs/usage.template b/docs/usage.template
@@ -18,7 +18,7 @@ Each task is controlled by one of the following commands: `run`, `rerun`, `runp`
 
 - `looper destroy`: Deletes all output results for this project.
 
-- `looper inspect`: Display the Prioject or Sample information
+- `looper inspect`: Display the Project or Sample information
 
 - `looper init`: Initialize a looper dotfile (`.looper.yaml`) in the current directory
 

diff --git a/looper/cli_looper.py b/looper/cli_looper.py
@@ -433,6 +433,7 @@ def add_subparser(cmd):
                 metavar="ATTR",
                 help="Attribute for sample exclusion OR inclusion",
             )
+
             protocols = fetch_samples_group.add_mutually_exclusive_group()
             protocols.add_argument(
                 f"--{SAMPLE_EXCLUSION_OPTNAME}",
@@ -446,6 +447,22 @@ def add_subparser(cmd):
                 metavar="I",
                 help="Include only samples with these values",
             )
+            fetch_samples_group.add_argument(
+                f"--{SAMPLE_SELECTION_FLAG_OPTNAME}",
+                default=None,
+                nargs="*",
+                metavar="SELFLAG",
+                help="Include samples with this flag status, e.g. completed",
+            )
+
+            fetch_samples_group.add_argument(
+                f"--{SAMPLE_EXCLUSION_FLAG_OPTNAME}",
+                default=None,
+                nargs="*",
+                metavar="EXCFLAG",
+                help="Exclude samples with this flag status, e.g. completed",
+            )
+
             subparser.add_argument(
                 "-a",
                 "--amend",
@@ -633,6 +650,10 @@ def main(test_args=None):
         select_divvy_config(filepath=args.divvy) if hasattr(args, "divvy") else None
     )
 
+    # Ignore flags if user is selecting or excluding on flags:
+    if args.sel_flag or args.exc_flag:
+        args.ignore_flags = True
+
     # Initialize project
     if is_registry_path(args.config_file):
         if vars(args)[SAMPLE_PL_ARG]:
@@ -678,6 +699,8 @@ def main(test_args=None):
         selector_attribute=args.sel_attr,
         selector_include=args.sel_incl,
         selector_exclude=args.sel_excl,
+        selector_flag=args.sel_flag,
+        exclusion_flag=args.exc_flag,
     ) as prj:
         if args.command in ["run", "rerun"]:
             run = Runner(prj)

diff --git a/looper/const.py b/looper/const.py
@@ -85,6 +85,8 @@
     "SAMPLE_SELECTION_ATTRIBUTE_OPTNAME",
     "SAMPLE_EXCLUSION_OPTNAME",
     "SAMPLE_INCLUSION_OPTNAME",
+    "SAMPLE_SELECTION_FLAG_OPTNAME",
+    "SAMPLE_EXCLUSION_FLAG_OPTNAME",
     "DEBUG_JOBS",
     "DEBUG_COMMANDS",
     "DEBUG_EIDO_VALIDATION",
@@ -249,6 +251,8 @@ def _get_apperance_dict(type, templ=APPEARANCE_BY_FLAG):
 SAMPLE_SELECTION_ATTRIBUTE_OPTNAME = "sel-attr"
 SAMPLE_EXCLUSION_OPTNAME = "sel-excl"
 SAMPLE_INCLUSION_OPTNAME = "sel-incl"
+SAMPLE_SELECTION_FLAG_OPTNAME = "sel-flag"
+SAMPLE_EXCLUSION_FLAG_OPTNAME = "exc-flag"
 
 MESSAGE_BY_SUBCOMMAND = {
     "run": "Run or submit sample jobs.",