Merge pull request #1259 from nipreps/fix/outputs-multiple-runs

FIX: Make sure new logs and config file output are compatible with parallel processes

.circleci/circle_T1w.txt

@@ -5,8 +5,8 @@ dataset_description.json
 group_T1w.html
 group_T1w.tsv
 logs
-logs/mriqcconfig.toml
-logs/mriqc.log
+logs/config-18480913-163000_PhineasG-ageh-adhi-sacc-ident9b1ab0f.toml
+logs/mriqc-18480913-163000_PhineasG-ageh-adhi-sacc-ident9b1ab0f.log
 sub-50137
 sub-50137/anat
 sub-50137/anat/sub-50137_T1w.json

.circleci/circle_bold.txt

@@ -5,8 +5,8 @@ dataset_description.json
 group_bold.html
 group_bold.tsv
 logs
-logs/mriqcconfig.toml
-logs/mriqc.log
+logs/config-18480913-163000_PhineasG-ageh-adhi-sacc-ident9b1ab0f.toml
+logs/mriqc-18480913-163000_PhineasG-ageh-adhi-sacc-ident9b1ab0f.log
 sub-ds205s03
 sub-ds205s03/figures
 sub-ds205s03/figures/sub-ds205s03_task-functionallocalizer_run-01_desc-background_bold.svg

.circleci/config.yml

@@ -389,6 +389,7 @@ jobs:
             docker run -u $( id -u ) --rm -ti \
                        -v /tmp/data/${TEST_DATA_NAME}:/data:ro \
                        -v /tmp/t1w:/scratch -w /scratch \
+                       -e MRIQC_DEV=1 \
                        nipreps/mriqc:latest \
                        /data derivatives/ participant \
                        -vv --verbose-reports --profile -m T1w --dsname circletests \
@@ -400,7 +401,6 @@ jobs:
           name: Move temporary but relevant artifacts
           command: |
             mkdir /tmp/t1w/misc
-            mv /tmp/t1w/work/.mriqc.*.toml /tmp/t1w/misc
             mv /tmp/t1w/work/.resources.*.tsv /tmp/t1w/misc
             mv /tmp/t1w/work/.resources.*.png /tmp/t1w/misc
 
@@ -419,6 +419,7 @@ jobs:
             docker run -u $( id -u ) --rm -ti \
                        -v /tmp/data/${TEST_DATA_NAME}:/data:ro \
                        -v /tmp/t1w:/scratch -w /scratch \
+                       -e MRIQC_DEV=1 \
                        nipreps/mriqc:latest \
                        /data derivatives/ group \
                        -m T1w -vv
@@ -551,6 +552,7 @@ jobs:
             # Run MRIQC
             docker run -u $( id -u ) --rm -ti -v /tmp/data/${TEST_DATA_NAME}:/data:ro \
                        -v $PWD:/scratch -w /scratch \
+                       -e MRIQC_DEV=1 \
                        nipreps/mriqc:latest \
                        /data derivatives/ participant \
                        -vv --verbose-reports --profile -m bold --dsname circletests \
@@ -561,7 +563,6 @@ jobs:
           name: Move temporary but relevant artifacts
           command: |
             mkdir /tmp/bold/misc
-            mv /tmp/bold/work/.mriqc.*.toml /tmp/bold/misc
             mv /tmp/bold/work/.resources.*.tsv /tmp/bold/misc
             mv /tmp/bold/work/.resources.*.png /tmp/bold/misc
 
@@ -578,6 +579,7 @@ jobs:
           no_output_timeout: 2h
           command: |
             docker run -u $( id -u ) --rm -ti -v /tmp/data/${TEST_DATA_NAME}:/data:ro \
+                       -e MRIQC_DEV=1 \
                        -v $PWD:/scratch -w /scratch \
                        nipreps/mriqc:latest \
                        /data derivatives/ group \

mriqc/cli/parser.py

@@ -491,7 +491,11 @@ def parse_args(args=None, namespace=None):
 
     config.loggers.init()
 
-    _log_file = Path(opts.output_dir) / 'logs' / 'mriqc.log'
+    _log_file = (
+        Path(opts.output_dir)
+        / 'logs'
+        / f'mriqc-{config.execution.run_uuid}.log'
+    )
     _log_file.parent.mkdir(exist_ok=True, parents=True)
     _handler = FileHandler(_log_file)
     _handler.setFormatter(_LogFormatter(

mriqc/cli/run.py

@@ -264,7 +264,9 @@ def main():
             '%Hh %Mmin %Ss',
             time.gmtime(time.time() - config.settings.start_time))
     ))
-    config.to_filename(config.execution.log_dir / 'mriqcconfig.toml')
+    config.to_filename(
+        config.execution.log_dir / f'config-{config.execution.run_uuid}.toml'
+    )
     sys.exit(exitcode)
 
 

mriqc/config.py

@@ -91,7 +91,6 @@
 import sys
 from contextlib import suppress
 from pathlib import Path
-from tempfile import mkstemp
 from time import strftime
 from uuid import uuid4
 
@@ -117,14 +116,16 @@
 os.environ['NIPYPE_NO_ET'] = '1'
 os.environ['NO_ET'] = '1'
 
+_mriqc_dev = os.getenv('MRIQC_DEV', '0').lower() in ('1', 'on', 'true', 'y', 'yes')
+
 if not hasattr(sys, '_is_pytest_session'):
     sys._is_pytest_session = False  # Trick to avoid sklearn's FutureWarnings
 # Disable all warnings in main and children processes only on production versions
 if not any(
     (
         '+' in __version__,
         __version__.endswith('.dirty'),
-        os.getenv('MRIQC_DEV', '0').lower() in ('1', 'on', 'true', 'y', 'yes'),
+        _mriqc_dev,
     )
 ):
     os.environ['PYTHONWARNINGS'] = 'ignore'
@@ -212,6 +213,12 @@
 if _fs_home and not _default_model_path.exists():
     _default_model_path = None
 
+# Override the unique ID if MRIQC_DEV is set
+_run_uuid = (
+    '{}_{}'.format(strftime('%Y%m%d-%H%M%S'), uuid4()) if not _mriqc_dev
+    else '18480913-163000_PhineasG-ageh-adhi-sacc-ident9b1ab0f'
+)
+
 
 class _Config:
     """An abstract class forbidding instantiation."""
@@ -422,7 +429,7 @@ class execution(_Config):
     """Enable resource monitor."""
     run_id = None
     """Filter input dataset by run identifier."""
-    run_uuid = '{}_{}'.format(strftime('%Y%m%d-%H%M%S'), uuid4())
+    run_uuid = _run_uuid
     """Unique identifier of this particular run."""
     session_id = None
     """Filter input dataset by session identifier."""
@@ -703,12 +710,8 @@ def to_filename(filename=None):
     if filename:
         settings.file_path = Path(filename)
     elif settings.file_path is None:
-        settings.file_path = Path(
-            mkstemp(
-                dir=execution.work_dir,
-                prefix='.mriqc.',
-                suffix='.toml'
-            )[1],
+        settings.file_path = (
+            execution.work_dir / f'config-{execution.run_uuid}.toml'
         )
 
     settings.file_path.parent.mkdir(exist_ok=True, parents=True)