Merge pull request #46 from vreuter/fix_master

No more null logger

looper/__init__.py

@@ -23,9 +23,6 @@
 DEFAULT_LOGGING_FMT = "%(asctime)s %(name)s %(module)s : %(lineno)d - [%(levelname)s] > %(message)s"
 
 
-LOOPER_LOGGER = None
-
-
 def setup_looper_logger(level, additional_locations=None,
                         fmt=None, datefmt=None):
     """
@@ -37,11 +34,11 @@ def setup_looper_logger(level, additional_locations=None,
         destination(s) to which to ship logs
     :param str fmt: message format string for log message, optional
     :param str datefmt: datetime format string for log message time, optional
+    :return logging.Logger: project-root logger
     """
 
     # Establish the logger.
-    global LOOPER_LOGGER
-    LOOPER_LOGGER = logging.getLogger(__name__.split(".")[0])
+    LOOPER_LOGGER = logging.getLogger("looper")
     LOOPER_LOGGER.handlers = []
     try:
         LOOPER_LOGGER.setLevel(level)
@@ -87,3 +84,5 @@ def setup_looper_logger(level, additional_locations=None,
         handler.setLevel(level)
         handler.setFormatter(formatter)
         LOOPER_LOGGER.addHandler(handler)
+
+    return LOOPER_LOGGER

looper/looper.py

@@ -28,7 +28,7 @@
 		Project, PipelineInterface, ProtocolMapper, LOOPERENV_VARNAME
 
 
-_LOGGER = logging.getLogger(__name__)
+_LOGGER = None
 
 
 def parse_arguments():
@@ -113,10 +113,11 @@ def parse_arguments():
 
 	# To enable the loop to pass args directly on to the pipelines...
 	args, remaining_args = parser.parse_known_args()
-	global _LOGGER
-	_LOGGER = setup_looper_logger(
+	setup_looper_logger(
 		args.logging_level, (args.logfile, ),
 		fmt=args.logging_fmt, datefmt=args.logging_datefmt)
+	global _LOGGER
+	_LOGGER = logging.getLogger(__name__)
 
 	if len(remaining_args) > 0:
 		logging.info("Remaining arguments passed to pipelines: {}".
@@ -225,7 +226,7 @@ def run(prj, args, remaining_args):
 
 			# add pipeline-specific attributes (read type and length, inputs, etc)
 			sample.set_pipeline_attributes(pipeline_interface, pl_id)
-			_LOGGER.info("({.2f} Gb)".format(sample.input_file_size))
+			_LOGGER.info("({:.2f} Gb)".format(sample.input_file_size))
 
 			# Check for any required inputs before submitting
 			try:
@@ -665,6 +666,7 @@ def check(prj):
 
 
 def main():
+
 	# Parse command-line arguments and establish logger.
 	args, remaining_args = parse_arguments()
 

looper/models.py

@@ -55,7 +55,6 @@
 from pkg_resources import resource_filename
 
 import pandas as _pd
-from pandas.io.parsers import EmptyDataError
 import yaml as _yaml
 
 from . import LOOPERENV_VARNAME
@@ -424,7 +423,7 @@ def set_compute(self, setting):
 
             self._logger.debug("%s: %s", str(setting),
                                self.looperenv.compute[setting])
-            self._logger.info("Compute: %s", str(self.looperenv.compute))
+            self._logger.debug("Compute: %s", str(self.looperenv.compute))
 
             if not _os.path.isabs(self.compute.submission_template):
                 # self.compute.submission_template = _os.path.join(self.metadata.pipelines_dir, self.compute.submission_template)
@@ -646,10 +645,6 @@ def check_sheet(self, dtype):
             self.df = _pd.read_csv(self.csv, dtype=dtype)
         except IOError("Given csv file couldn't be read.") as e:
             raise e
-        except EmptyDataError:
-            self._logger.error("Attempted to read {} as data type {}".
-                               format(self.csv, str(dtype)))
-            raise
 
         # Check mandatory items are there
         req = ["sample_name"]
@@ -852,7 +847,7 @@ def to_yaml(self, path=None):
         :param path: A file path to write yaml to.
         :type path: str
         """
-        def obj2dict(obj, to_skip=["samples", "sheet", "sheet_attributes"]):
+        def obj2dict(obj, to_skip=("samples", "sheet", "sheet_attributes")):
             """
             Build representation of object as a dict, recursively
             for all objects that might be attributes of self.
@@ -864,9 +859,9 @@ def obj2dict(obj, to_skip=["samples", "sheet", "sheet_attributes"]):
             if type(obj) is list:  # recursive serialization (lists)
                 return [obj2dict(i) for i in obj]
             elif type(obj) is dict:  # recursive serialization (dict)
-                return {k: obj2dict(v) for k, v in obj.items() if (k not in to_skip)}
+                return {k: obj2dict(v) for k, v in obj.items() if (k not in to_skip and not isinstance(v, logging.Logger))}
             elif any([isinstance(obj, t) for t in [AttributeDict, Project, Paths, Sample]]):  # recursive serialization (AttributeDict and children)
-                return {k: obj2dict(v) for k, v in obj.__dict__.items() if (k not in to_skip)}
+                return {k: obj2dict(v) for k, v in obj.__dict__.items() if (k not in to_skip and not isinstance(v, logging.Logger))}
             elif hasattr(obj, 'dtype'):  # numpy data types
                 return obj.item()
             elif _pd.isnull(obj):  # Missing values as evaluated by pd.isnull() <- this gets correctly written into yaml
@@ -1291,11 +1286,14 @@ class PipelineInterface(object):
 
     def __init__(self, yaml_config_file):
         import yaml
+        self._logger = logging.getLogger(
+            "{}.{}".format(__name__, self.__class__.__name__))
+        self._logger.info("Creating %s from file '%s'",
+                          self.__class__.__name__, yaml_config_file)
         self.looper_config_file = yaml_config_file
         with open(yaml_config_file, 'r') as f:
             self.looper_config = yaml.load(f)
-        self._logger = logging.getLogger(
-            "{}.{}".format(__name__, self.__class__.__name__))
+
 
     def select_pipeline(self, pipeline_name):
         """
@@ -1316,7 +1314,7 @@ def select_pipeline(self, pipeline_name):
     def uses_looper_args(self, pipeline_name):
         config = self.select_pipeline(pipeline_name)
 
-        if hasattr(config, "looper_args") and config.looper_args:
+        if "looper_args" in config and config["looper_args"]:
             return True
         else:
             return False

requirements.txt

@@ -1,2 +1,3 @@
-pandas
-pyyaml
+pandas>=0.19.2
+pyyaml>=3.12
+

tests/conftest.py

@@ -52,7 +52,7 @@
 
 PIPELINE_INTERFACE_CONFIG_LINES = """testpipeline.sh:
   name: test_pipeline  # Name used by pypiper so looper can find the logs
-  looper_args: True
+  looper_args: False
   arguments:
     "--input": file
   optional_arguments:
@@ -87,6 +87,12 @@
       partition: "longq"
 """.splitlines(True)
 
+# Determined by "looper_args" in pipeline interface lines.
+LOOPER_ARGS_BY_PIPELINE = {
+    "testpipeline.sh": False,
+    "testngs.sh": True
+}
+
 # These per-sample file lists pertain to the expected required inputs.
 # These map to required_input_files in the pipeline interface config files.
 _FILE_FILE2_BY_SAMPLE = [

tests/test_looper.py

@@ -13,8 +13,8 @@
 from looper.models import COL_KEY_SUFFIX
 
 from conftest import \
-    DERIVED_COLNAMES, EXPECTED_MERGED_SAMPLE_FILES, \
-    FILE_BY_SAMPLE, MERGED_SAMPLE_INDICES, NGS_SAMPLE_INDICES, \
+    DERIVED_COLNAMES, EXPECTED_MERGED_SAMPLE_FILES, FILE_BY_SAMPLE, \
+    LOOPER_ARGS_BY_PIPELINE, MERGED_SAMPLE_INDICES, NGS_SAMPLE_INDICES, \
     NUM_SAMPLES, PIPELINE_TO_REQD_INFILES_BY_SAMPLE
 
 
@@ -151,11 +151,17 @@ def test_ngs_pipe_ngs_sample(self, proj, pipe_iface, sample_index):
                observed_required_input_basename
 
 
-
     @pytest.mark.parametrize(argnames="sample_index",
                              argvalues=set(range(NUM_SAMPLES)) -
                                        NGS_SAMPLE_INDICES)
     def test_ngs_pipe_non_ngs_sample(self, proj, pipe_iface, sample_index):
             sample = proj.samples[sample_index]
             with pytest.raises(TypeError):
                 sample.set_pipeline_attributes(pipe_iface, "testngs.sh")
+
+
+    @pytest.mark.parametrize(argnames="pipeline,expected",
+                             argvalues=LOOPER_ARGS_BY_PIPELINE.items())
+    def test_looper_args_usage(self, pipe_iface, pipeline, expected):
+        observed = pipe_iface.uses_looper_args(pipeline)
+        assert (expected and observed) or not (observed or expected)