fixed tests and updatd documentation

docs/looper-config.md

@@ -33,4 +33,4 @@ one of supported ways: `namespace/name`, `pephub::namespace/name`, `namespace/na
 - `pipeline interfaces` is a local path to project or sample pipelines.
 
 To run pipeline, go to the directory of .looper.config and execute command in your terminal:
-`looper run` or `looper runp`.
+`looper run --looper-config {looper_config_path}` or `looper runp --looper-config {looper_config_path}`.

looper/cli_looper.py

@@ -567,10 +567,6 @@ def main(test_args=None):
             not init_dotfile(
                 dotfile_path(),
                 args.config_file,
-                args.output_dir,
-                args.sample_pipeline_interfaces,
-                args.project_pipeline_interfaces,
-                args.force,
             )
         )
 

looper/const.py

@@ -88,7 +88,6 @@
     "DEBUG_JOBS",
     "DEBUG_COMMANDS",
     "DEBUG_EIDO_VALIDATION",
-
 ]
 
 FLAGS = ["completed", "running", "failed", "waiting", "partial"]
@@ -119,6 +118,7 @@ def _get_apperance_dict(type, templ=APPEARANCE_BY_FLAG):
             ret[flag][key] = ret[flag][key].format(type=type)
     return ret
 
+
 # Debug keys
 DEBUG_JOBS = "Jobs submitted"
 DEBUG_COMMANDS = "Commands submitted"

looper/looper.py

@@ -518,9 +518,7 @@ def __call__(self, args, rerun=False, **compute_kwargs):
             )
         )
         _LOGGER.info("Commands submitted: {} of {}".format(cmd_sub_total, max_cmds))
-        self.debug[DEBUG_COMMANDS] = "{} of {}".format(
-            cmd_sub_total, max_cmds
-        )
+        self.debug[DEBUG_COMMANDS] = "{} of {}".format(cmd_sub_total, max_cmds)
         if args.dry_run:
             job_sub_total_if_real = job_sub_total
             job_sub_total = 0

looper/utils.py

@@ -360,70 +360,96 @@ def init_generic_pipeline():
 
 def init_dotfile(
     path: str,
-    cfg_path: str = None,
+    looper_config_path: str,
+    force=False,
+):
+    """
+    Initialize looper dotfile
+
+    :param str path: absolute path to the dot file to initialize
+    :param str looper_config_path: path to the looper config file. Absolute or relative to 'path'
+    :param bool force: whether the existing file should be overwritten
+    :return bool: whether the file was initialized
+    """
+    if os.path.exists(path) and not force:
+        print(f"Can't initialize, file exists: {path}")
+        return False
+    dot_dict = {
+        "looper_config": os.path.relpath(looper_config_path, os.path.dirname(path)),
+    }
+
+    with open(path, "w") as dotfile:
+        yaml.dump(dot_dict, dotfile)
+    print(f"Initialized looper dotfile: {path}")
+    return True
+
+
+def initiate_looper_config(
+    looper_config_path: str,
+    pep_path: str = None,
     output_dir: str = None,
     sample_pipeline_interfaces: Union[List[str], str] = None,
     project_pipeline_interfaces: Union[List[str], str] = None,
     force=False,
 ):
     """
-    Initialize looper dotfile
+    Initialize looper config file
 
-    :param str path: absolute path to the file to initialize
-    :param str cfg_path: path to the config file. Absolute or relative to 'path'
+    :param str looper_config_path: absolute path to the file to initialize
+    :param str pep_path: path to the PEP to be used in pipeline
     :param str output_dir: path to the output directory
     :param str|list sample_pipeline_interfaces: path or list of paths to sample pipeline interfaces
     :param str|list project_pipeline_interfaces: path or list of paths to project pipeline interfaces
     :param bool force: whether the existing file should be overwritten
     :return bool: whether the file was initialized
     """
-    if os.path.exists(path) and not force:
-        print("Can't initialize, file exists: {}".format(path))
+    if os.path.exists(looper_config_path) and not force:
+        print(f"Can't initialize, file exists: {looper_config_path}")
         return False
-    if cfg_path:
-        if is_registry_path(cfg_path):
+
+    if pep_path:
+        if is_registry_path(pep_path):
             pass
         else:
-            cfg_path = expandpath(cfg_path)
-            if not os.path.isabs(cfg_path):
-                cfg_path = os.path.join(os.path.dirname(path), cfg_path)
-            assert os.path.exists(cfg_path), OSError(
+            pep_path = expandpath(pep_path)
+            if not os.path.isabs(pep_path):
+                pep_path = os.path.join(os.path.dirname(looper_config_path), pep_path)
+            assert os.path.exists(pep_path), OSError(
                 "Provided config path is invalid. You must provide path "
-                "that is either absolute or relative to: {}".format(
-                    os.path.dirname(path)
-                )
+                f"that is either absolute or relative to: {os.path.dirname(looper_config_path)}"
             )
     else:
-        cfg_path = "example/pep/path"
+        pep_path = "example/pep/path"
 
     if not output_dir:
         output_dir = "."
 
     looper_config_dict = {
-        "pep_config": os.path.relpath(cfg_path, os.path.dirname(path)),
+        "pep_config": os.path.relpath(pep_path),
         "output_dir": output_dir,
         "pipeline_interfaces": {
             "sample": sample_pipeline_interfaces,
             "project": project_pipeline_interfaces,
         },
     }
 
-    with open(path, "w") as dotfile:
+    with open(looper_config_path, "w") as dotfile:
         yaml.dump(looper_config_dict, dotfile)
-    print("Initialized looper dotfile: {}".format(path))
+    print(f"Initialized looper config file: {looper_config_path}")
     return True
 
 
 def read_looper_dotfile():
     """
     Read looper config file
 
-    :return str: path to the config file read from the dotfile
     :raise MisconfigurationException: if the dotfile does not consist of the
         required key pointing to the PEP
     """
     dot_file_path = dotfile_path(must_exist=True)
-    return read_looper_config_file(looper_config_path=dot_file_path)
+    with open(dot_file_path, "r") as file:
+        looper_config_path = yaml.safe_load(file)["looper_config"]
+    return read_looper_config_file(looper_config_path=looper_config_path)
 
 
 def read_looper_config_file(looper_config_path: str) -> dict:

tests/conftest.py

@@ -265,8 +265,14 @@ def prepare_pep_with_dot_file(prep_temp_pep):
         },
     }
 
+    looper_config_path = os.path.join(os.path.dirname(pep_config), "looper_config.yaml")
+
+    with open(looper_config_path, "w") as f:
+        config = dump(looper_config, f)
+
+    looper_dot_file_content = {"looper_config": looper_config_path}
     dot_file_path = ".looper.yaml"
     with open(dot_file_path, "w") as f:
-        config = dump(looper_config, f)
+        config = dump(looper_dot_file_content, f)
 
     return dot_file_path

tests/smoketests/test_run.py

@@ -465,8 +465,8 @@ def test_looper_respects_pkg_selection(self, prep_temp_pep, cmd):
         x = test_args_expansion(tp, cmd, ["--package", "local"])
         try:
             main(test_args=x)
-        except Exception:
-            raise pytest.fail("DID RAISE {0}".format(Exception))
+        except Exception as err:
+            raise pytest.fail(f"DID RAISE {err}")
         sd = os.path.join(get_outdir(tp), "submission")
         subs_list = [os.path.join(sd, f) for f in os.listdir(sd) if f.endswith(".sub")]
         assert_content_not_in_any_files(subs_list, "#SBATCH")
@@ -540,8 +540,8 @@ def test_cli_compute_overwrites_yaml_settings_spec(self, prep_temp_pep, cmd):
         )
         try:
             main(test_args=x)
-        except Exception:
-            raise pytest.fail("DID RAISE {0}".format(Exception))
+        except Exception as err:
+            raise pytest.fail(f"DID RAISE {err}")
 
         sd = os.path.join(get_outdir(tp), "submission")
         subs_list = [os.path.join(sd, f) for f in os.listdir(sd) if f.endswith(".sub")]
@@ -552,29 +552,26 @@ class TestLooperConfig:
     @pytest.mark.parametrize("cmd", ["run", "runp"])
     def test_init_config_file(self, prep_temp_pep, cmd, dotfile_path):
         tp = prep_temp_pep
-        # stdout, stderr, rc = subp_exec(tp, "init")
-        # print_standard_stream(stderr)
-        # print_standard_stream(stdout)
         x = test_args_expansion(tp, "init")
         try:
             result = main(test_args=x)
-        except Exception:
-            raise pytest.fail("DID RAISE {0}".format(Exception))
+        except Exception as err:
+            raise pytest.fail(f"DID RAISE: {err}")
         assert result == 0
         assert_content_in_all_files(dotfile_path, tp)
         x = test_args_expansion(tp, cmd)
         try:
             result = main(test_args=x)
-        except Exception:
-            raise pytest.fail("DID RAISE {0}".format(Exception))
+        except Exception as err:
+            raise pytest.fail(f"DID RAISE {err}")
 
     def test_correct_execution_of_config(self, prepare_pep_with_dot_file):
-        dot_file_path = prepare_pep_with_dot_file
+        dot_file_path = os.path.abspath(prepare_pep_with_dot_file)
         x = test_args_expansion("", "run")
         try:
             main(test_args=x)
-        except Exception:
-            raise pytest.fail("DID RAISE {0}".format(Exception))
+        except Exception as err:
+            raise pytest.fail(f"DID RAISE {err}")
         os.remove(dot_file_path)