Skip to content
This repository has been archived by the owner on Nov 2, 2021. It is now read-only.

Commit

Permalink
Merge pull request #60 from bpoldrack/enh-conversion
Browse files Browse the repository at this point in the history 
ENH: conversion
  • Loading branch information
@bpoldrack
bpoldrack committed Aug 22, 2018
2 parents 9af862e + d9c303e commit f5da2a3
Show file tree
Hide file tree
Showing 6 changed files with 165 additions and 117 deletions.
39 changes: 32 additions & 7 deletions datalad_hirni/commands/dicom2spec.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,12 @@

lgr = logging.getLogger('datalad.hirni.dicom2spec')


############################## Build plugin mechanism for Rules finally!
# ############################# Build plugin mechanism for Rules finally!
#########################################


def add_to_spec(ds_metadata, spec_list, basepath,
subject=None, anon_subject=None, session=None):
subject=None, anon_subject=None, session=None, overrides=None):

from datalad_hirni.support.dicom2bids_rules import \
get_rules_from_metadata, series_is_valid # TODO: RF?
Expand All @@ -43,7 +42,7 @@ def add_to_spec(ds_metadata, spec_list, basepath,
# Note: The first 4 entries aren't a dict and have no
# "approved flag", since they are automatically managed
'type': 'dicomseries',
'status': None, # TODO: process state convention; flags
#'status': None, # TODO: process state convention; flags
'location': op.relpath(ds_metadata['path'], basepath),
'uid': series['SeriesInstanceUID'],
'dataset_id': ds_metadata['dsid'],
Expand All @@ -68,8 +67,11 @@ def add_to_spec(ds_metadata, spec_list, basepath,
base_list[idx][k] = {'value': values[k],
'approved': False}

# merge with existing spec:
# merge with existing spec plus overrides:
for series in base_list:

series.update(overrides)

existing = [i for s, i in
zip(spec_list, range(len(spec_list)))
if s['uid'] == series['uid']]
Expand Down Expand Up @@ -126,6 +128,12 @@ class Dicom2Spec(Interface):
doc="""session identifier. If not specified, an attempt will be made
to derive SESSION from DICOM headers""",
constraints=EnsureStr() | EnsureNone()),
properties=Parameter(
args=("--properties",),
metavar="PATH or JSON string",
doc="""""",
constraints=EnsureStr() | EnsureNone()),

recursive=recursion_flag,
# TODO: invalid, since datalad-metadata doesn't support it:
# recursion_limit=recursion_limit,
Expand All @@ -135,7 +143,8 @@ class Dicom2Spec(Interface):
@datasetmethod(name='hirni_dicom2spec')
@eval_results
def __call__(path=None, spec=None, dataset=None, subject=None,
anon_subject=None, session=None, recursive=False):
anon_subject=None, session=None, recursive=False,
properties=None):

dataset = require_dataset(dataset, check_installed=True,
purpose="spec from dicoms")
Expand Down Expand Up @@ -200,12 +209,28 @@ def __call__(path=None, spec=None, dataset=None, subject=None,
continue

found_some = True

overrides = dict()
if properties:
# load from file or json string
props = json_py.load(properties) \
if op.exists(properties) else json_py.loads(properties)
# turn into editable, pre-approved records
props = {k: dict(value=v, approved=True) for k, v in props.items()}
overrides.update(props)

spec_series_list = add_to_spec(meta,
spec_series_list,
op.dirname(spec),
subject=subject,
anon_subject=anon_subject,
session=session)
# session=session,
# TODO: parameter "session" was what
# we now call acquisition. This is
# NOT a good default for bids_session!
# Particularly wrt to anonymization
overrides=overrides
)

if not found_some:
yield dict(status='impossible',
Expand Down
46 changes: 24 additions & 22 deletions datalad_hirni/commands/import_dicoms.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ class ImportDicoms(Interface):
acqid=Parameter(
args=("acqid",),
metavar="ACQUISITION ID",
doc="""session identifier for the imported DICOM files. If not
doc="""acquisition identifier for the imported DICOM files. If not
specified, an attempt will be made to derive ACQUISITION_ID from DICOM
headers.""",
nargs="?",
Expand All @@ -178,11 +178,10 @@ class ImportDicoms(Interface):
metavar="ANON_SUBJECT",
doc="""TODO""",
constraints=EnsureStr() | EnsureNone()),
session=Parameter(
args=("--session",),
metavar="SESSION",
doc="""session identifier. If not specified, an attempt will be made
to derive SESSION from DICOM headers""",
properties=Parameter(
args=("--properties",),
metavar="PATH or JSON string",
doc="""""",
constraints=EnsureStr() | EnsureNone()),

)
Expand All @@ -191,41 +190,43 @@ class ImportDicoms(Interface):
@datasetmethod(name='hirni_import_dcm')
@eval_results
def __call__(path, acqid=None, dataset=None,
subject=None, anon_subject=None, session=None):
subject=None, anon_subject=None, properties=None):
ds = require_dataset(dataset, check_installed=True,
purpose="import DICOM session")
if acqid:
# session was specified => we know where to create subds
ses_dir = op.join(ds.path, acqid)
if not op.exists(ses_dir):
makedirs(ses_dir)
# acquisition was specified => we know where to create subds
acq_dir = op.join(ds.path, acqid)
if not op.exists(acq_dir):
makedirs(acq_dir)
# TODO: if exists: needs to be empty?

dicom_ds = _create_subds_from_tarball(path, ses_dir)
dicom_ds = _create_subds_from_tarball(path, acq_dir)

else:
# we don't know the session yet => create in tmp
# we don't know the acquisition id yet => create in tmp

ses_dir = op.join(ds.path, '.git', 'datalad', 'hirni_import')
assert not op.exists(ses_dir)
acq_dir = op.join(ds.path, '.git', 'datalad', 'hirni_import')
assert not op.exists(acq_dir)
# TODO: don't assert; check and adapt instead

try:
dicom_ds = _create_subds_from_tarball(path, ses_dir)
dicom_ds = _create_subds_from_tarball(path, acq_dir)
dicom_ds = _guess_acquisition_and_move(dicom_ds, ds)
except FileExistsError as e:
except OSError as e:
# TODO: Was FileExistsError. Find more accurate PY2/3 solution
# than just OSError
yield dict(status='impossible',
path=e.filename,
type='file',
action='import DICOM tarball',
logger=lgr,
message='%s already exists' % e.filename)
rmtree(ses_dir)
rmtree(acq_dir)

finally:
if op.exists(ses_dir):
lgr.debug("Killing temp dataset at %s ...", ses_dir)
rmtree(ses_dir)
if op.exists(acq_dir):
lgr.debug("Killing temp dataset at %s ...", acq_dir)
rmtree(acq_dir)

acqid = op.basename(op.dirname(dicom_ds.path))
ds.add(
Expand All @@ -240,7 +241,8 @@ def __call__(path, acqid=None, dataset=None,
dicom_ds.path, op.pardir, "studyspec.json")),
subject=subject,
anon_subject=anon_subject,
session=session
session=acqid,
properties=properties
)

# We have the tarball and can drop extracted stuff:
Expand Down
152 changes: 77 additions & 75 deletions datalad_hirni/commands/spec2bids.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,82 +144,84 @@ def __call__(specfile, dataset=None, anonymize=False):
dataset.config.reload()

if not ran_heudiconv and \
heuristic.has_specval(spec_snippet, 'converter') and \
heuristic.get_specval(spec_snippet, 'converter') == 'heudiconv':
# special treatment of DICOMs (using heudiconv)
# But it's one call to heudiconv for all DICOMs of an
# acquisition!
from mock import patch
from tempfile import mkdtemp

# relative path to not-needed-heudiconv output:
rel_trash_path = relpath(mkdtemp(prefix="hirni-tmp-",
dir=opj(dataset.path,
".git")),
dataset.path)
run_results = list()
with patch.dict('os.environ',
{'HIRNI_STUDY_SPEC': rel_spec_path,
'HIRNI_SPEC2BIDS_SUBJECT': replacements['bids_subject']}):

for r in dataset.containers_run(
['heudiconv',
# XXX absolute path will make rerun on other system
# impossible -- hard to avoid
'-f', heuristic.__file__,
# leaves identifying info in run record
'-s', replacements['bids_subject'],
'-c', 'dcm2niix',
# TODO decide on the fate of .heudiconv/
# but ATM we need to (re)move it:
# https://github.com/nipy/heudiconv/issues/196
'-o', rel_trash_path,
'-b',
'-a', '{dspath}',
'-l', '',
# avoid glory details provided by dcmstack,
# we have them in the aggregated DICOM
# metadata already
'--minmeta',
'--files', replacements['location']
],
sidecar=anonymize,
container_name=dataset.config.get(
"datalad.hirni.conversion-container",
"conversion"),
inputs=[replacements['location'], rel_spec_path],
outputs=[dataset.path],
message="Convert DICOM data for subject {}"
"".format(replacements['bids_subject']),
return_type='generator',
):
# if there was an issue with containers-run,
# yield original result, otherwise swallow:
if r['status'] not in ['ok', 'notneeded']:
yield r

run_results.append(r)

if not all(r['status'] in ['ok', 'notneeded']
for r in run_results):
yield {'action': 'heudiconv',
'path': spec_path,
'snippet': spec_snippet,
'status': 'error',
'message': "acquisition conversion failed. "
"See previous message(s)."}
heuristic.has_specval(spec_snippet, 'converter') and \
heuristic.get_specval(spec_snippet, 'converter') == 'heudiconv':
# TODO: location!

# special treatment of DICOMs (using heudiconv)
# But it's one call to heudiconv for all DICOMs of an
# acquisition!
from mock import patch
from tempfile import mkdtemp

# relative path to not-needed-heudiconv output:
rel_trash_path = relpath(mkdtemp(prefix="hirni-tmp-",
dir=opj(dataset.path,
".git")),
dataset.path)
run_results = list()
with patch.dict('os.environ',
{'HIRNI_STUDY_SPEC': rel_spec_path,
'HIRNI_SPEC2BIDS_SUBJECT': replacements['bids_subject']}):

for r in dataset.containers_run(
['heudiconv',
# XXX absolute path will make rerun on other system
# impossible -- hard to avoid
'-f', heuristic.__file__,
# leaves identifying info in run record
'-s', replacements['bids_subject'],
'-c', 'dcm2niix',
# TODO decide on the fate of .heudiconv/
# but ATM we need to (re)move it:
# https://github.com/nipy/heudiconv/issues/196
'-o', rel_trash_path,
'-b',
'-a', '{dspath}',
'-l', '',
# avoid glory details provided by dcmstack,
# we have them in the aggregated DICOM
# metadata already
'--minmeta',
'--files', replacements['location']
],
sidecar=anonymize,
container_name=dataset.config.get(
"datalad.hirni.conversion-container",
"conversion"),
inputs=[replacements['location'], rel_spec_path],
outputs=[dataset.path],
message="Convert DICOM data for subject {}"
"".format(replacements['bids_subject']),
return_type='generator',
):
# if there was an issue with containers-run,
# yield original result, otherwise swallow:
if r['status'] not in ['ok', 'notneeded']:
yield r

run_results.append(r)

else:
yield {'action': 'heudiconv',
'path': spec_path,
'snippet': spec_snippet,
'status': 'ok',
'message': "acquisition converted."}

# remove superfluous heudiconv output
rmtree(opj(dataset.path, rel_trash_path))
# run heudiconv only once
ran_heudiconv = True
if not all(r['status'] in ['ok', 'notneeded']
for r in run_results):
yield {'action': 'heudiconv',
'path': spec_path,
'snippet': spec_snippet,
'status': 'error',
'message': "acquisition conversion failed. "
"See previous message(s)."}

else:
yield {'action': 'heudiconv',
'path': spec_path,
'snippet': spec_snippet,
'status': 'ok',
'message': "acquisition converted."}

# remove superfluous heudiconv output
rmtree(opj(dataset.path, rel_trash_path))
# run heudiconv only once
ran_heudiconv = True

elif heuristic.has_specval(spec_snippet, 'converter') and \
heuristic.get_specval(spec_snippet, 'converter') != 'heudiconv':
Expand Down

0 comments on commit f5da2a3

Please sign in to comment.