Fixes globbing issues

bin/convert_predictions_to_segmentation.py

@@ -1,4 +1,5 @@
 from __future__ import print_function
+from ilastik.utility.data_url import Dataset
 import os
 import copy
 import h5py
@@ -40,7 +41,7 @@ def print_progress(progress_percent):
     opExport.progressSignal.subscribe(print_progress)
 
     for input_path in input_paths:
-        opReader.FilePath.setValue(input_path)
+        opReader.Dataset.setValue(Dataset.from_string(input_path, deglob=False))
 
         input_pathcomp = PathComponents(input_path)
         opExport.OutputFilenameFormat.setValue(str(input_pathcomp.externalPath))

bin/convert_predictions_to_uncertainties.py

@@ -1,4 +1,5 @@
 from __future__ import print_function
+from ilastik.utility.data_url import Dataset
 import os
 import copy
 import h5py
@@ -20,7 +21,7 @@ def convert_predictions_to_uncertainties(input_path, parsed_export_args):
     graph = Graph()
     opReader = OpInputDataReader(graph=graph)
     opReader.WorkingDirectory.setValue(os.getcwd())
-    opReader.FilePath.setValue(input_path)
+    opReader.Dataset.setValue(Dataset.from_string(input_path, deglob=False))
 
     opUncertainty = OpEnsembleMargin(graph=graph)
     opUncertainty.Input.connect(opReader.Output)

bin/train_headless.py

@@ -31,6 +31,7 @@
 """
 from __future__ import print_function
 from builtins import range
+from ilastik.utility.data_url import Dataset
 import os
 
 
@@ -190,7 +191,7 @@ def generate_trained_project_file(
         opReader = OpInputDataReader(graph=graph)
         try:
             opReader.WorkingDirectory.setValue(cwd)
-            opReader.FilePath.setValue(label_data_path)
+            opReader.Dataset.setValue(Dataset.from_string(label_data_path, deglob=False))
 
             print("Reading label volume: {}".format(label_data_path))
             label_volume = opReader.Output[:].wait()

ilastik/applets/batchProcessing/batchProcessingGui.py

@@ -315,7 +315,7 @@ def run_export(self):
             return
 
         # Run the export in a separate thread
-        lane_configs = self.parentApplet.dataSelectionApplet.create_lane_configs(role_inputs=role_inputs)
+        lane_configs = self.parentApplet.dataSelectionApplet.create_lane_configs(role_inputs=role_inputs, deglob=False)
 
         export_req = Request(partial(self.parentApplet.run_export, lane_configs=lane_configs))
         export_req.notify_failed(self.handle_batch_processing_failure)

ilastik/applets/dataSelection/dataSelectionApplet.py

@@ -21,6 +21,7 @@
 from __future__ import division
 from __future__ import absolute_import
 from builtins import range
+from ilastik.utility.data_url import Dataset, PrecomputedChunksUrl
 import os
 import sys
 import glob
@@ -243,15 +244,14 @@ def _role_name_to_arg_name(role_name):
         return role_name.lower().replace(" ", "_").replace("-", "_")
 
     def create_dataset_info(
-        self, url: Union[Path, str], axistags: Optional[vigra.AxisTags] = None, sequence_axis: str = "z"
+        self, url: Union[Path, str], deglob: bool, axistags: Optional[vigra.AxisTags] = None, sequence_axis: str = "z"
     ) -> DatasetInfo:
         url = str(url)
         if isUrl(url):
-            return UrlDatasetInfo(url=url, axistags=axistags)
+            return UrlDatasetInfo(url=PrecomputedChunksUrl.from_string(url), axistags=axistags)
         else:
-            return RelativeFilesystemDatasetInfo.create_or_fallback_to_absolute(
-                filePath=url, axistags=axistags, sequence_axis=sequence_axis
-            )
+            dataset = Dataset.split(url, deglob=deglob)
+            return FilesystemDatasetInfo(dataset=dataset, axistags=axistags, sequence_axis=sequence_axis)
 
     def convert_info_to_h5(self, info: DatasetInfo) -> DatasetInfo:
         tmp_path = tempfile.mktemp() + ".h5"
@@ -260,16 +260,18 @@ def convert_info_to_h5(self, info: DatasetInfo) -> DatasetInfo:
         with h5py.File(tmp_path, mode="w") as tmp_h5:
             logger.info(f"Converting info {info.effective_path} to h5 at {full_path}")
             info.dumpToHdf5(h5_file=tmp_h5, inner_path=inner_path)
-            return self.create_dataset_info(url=full_path)
+            return self.create_dataset_info(url=full_path, deglob=False)
 
     def create_lane_configs(
         self,
+        *,
         role_inputs: Dict[str, List[str]],
         input_axes: Sequence[Optional[vigra.AxisTags]] = (),
         preconvert_stacks: bool = False,
         ignore_training_axistags: bool = False,
         stack_along: str = "z",
-    ) -> List[Dict[str, DatasetInfo]]:
+        deglob: bool,
+    ) -> List[Dict[str, Optional[DatasetInfo]]]:
         if not input_axes or not any(input_axes):
             if ignore_training_axistags or self.num_lanes == 0:
                 input_axes = [None] * len(self.role_names)
@@ -283,7 +285,10 @@ def create_lane_configs(
         rolewise_infos: Dict[str, List[DatasetInfo]] = {}
         for role_name, axistags in zip(self.role_names, input_axes):
             role_urls = role_inputs.get(role_name, [])
-            infos = [self.create_dataset_info(url, axistags=axistags, sequence_axis=stack_along) for url in role_urls]
+            infos = [
+                self.create_dataset_info(url, axistags=axistags, sequence_axis=stack_along, deglob=deglob)
+                for url in role_urls
+            ]
             if preconvert_stacks:
                 infos = [self.convert_info_to_h5(info) if info.is_stack() else info for info in infos]
             rolewise_infos[role_name] = infos
@@ -309,6 +314,7 @@ def lane_configs_from_parsed_args(self, parsed_args: argparse.Namespace) -> List
             preconvert_stacks=parsed_args.preconvert_stacks,
             ignore_training_axistags=parsed_args.ignore_training_axistags,
             stack_along=parsed_args.stack_along,
+            deglob=True,
         )
 
     def pushLane(self, role_infos: Dict[str, DatasetInfo]):

ilastik/applets/dataSelection/dataSelectionGui.py

@@ -1,4 +1,6 @@
 from __future__ import absolute_import
+from ilastik.utility.data_url import PrecomputedChunksUrl
+from ilastik.utility.data_url import ArchiveDataPath, Dataset
 
 ###############################################################################
 #   ilastik: interactive learning and segmentation toolkit
@@ -22,7 +24,7 @@
 ###############################################################################
 # Python
 import os
-from pathlib import Path
+from pathlib import Path, PurePosixPath
 from typing import Dict, List, Set, Union, Optional
 from vigra import AxisTags
 import threading
@@ -46,9 +48,11 @@
 from ilastik.utility import bind, log_exception
 from ilastik.utility.gui import ThreadRouter, threadRouted
 from ilastik.applets.layerViewer.layerViewerGui import LayerViewerGui
+from ilastik.applets.dataSelection.dataSelectionApplet import DataSelectionApplet
 from ilastik.applets.base.applet import DatasetConstraintError
 
 from .opDataSelection import (
+    AxisTagsHint,
     DatasetInfo,
     RelativeFilesystemDatasetInfo,
     FilesystemDatasetInfo,
@@ -57,7 +61,13 @@
 )
 from .dataLaneSummaryTableModel import DataLaneSummaryTableModel
 from .datasetInfoEditorWidget import DatasetInfoEditorWidget
-from ilastik.widgets.stackFileSelectionWidget import StackFileSelectionWidget, SubvolumeSelectionDlg
+from ilastik.widgets.stackFileSelectionWidget import (
+    DatasetSelectionMode,
+    DatasetSelectionWidget,
+    SubvolumeSelectionDlg,
+    select_single_file_datasets,
+    create_dataset,
+)
 from .datasetDetailedInfoTableModel import DatasetDetailedInfoTableModel
 from .datasetDetailedInfoTableView import DatasetDetailedInfoTableView
 from .precomputedVolumeBrowser import PrecomputedVolumeBrowser
@@ -150,7 +160,7 @@ class UserCancelledError(Exception):
 
     def __init__(
         self,
-        parentApplet,
+        parentApplet: DataSelectionApplet,
         dataSelectionOperator,
         serializer,
         instructionText,
@@ -184,7 +194,7 @@ def __init__(
         self._initAppletDrawerUic(instructionText)
 
         self._viewerControlWidgetStack = QStackedWidget(self)
-        self._default_h5n5_volumes: Dict[int, Set[str]] = {}
+        self._default_h5n5_volumes: Dict[int, Set[PurePosixPath]] = {}
 
         self.__cleanup_fns.append(self.topLevelOperator.DatasetGroup.notifyRemove(bind(self._handleImageRemove)))
 
@@ -410,21 +420,40 @@ def addFiles(self, roleIndex, startingLaneNum=None):
           the GUI table and the top-level operator inputs.
         """
         # Launch the "Open File" dialog
-        paths = ImageFileDialog(self).getSelectedPaths()
-        self.addFileNames(paths, startingLaneNum, roleIndex)
+        datasets = select_single_file_datasets(
+            parent=self, internal_path_hints=self._get_previously_used_inner_paths(roleIndex)
+        )
+
+        self.addDatasets(datasets, startingLaneNum, roleIndex)
 
-    def addFileNames(self, paths: List[Path], startingLaneNum: int, roleIndex: int):
+    def addFileNames(self, paths: List[str], startingLaneNum: Optional[int], roleIndex: int):
         # If the user didn't cancel
-        for path in paths or []:
+        datasets: List[Dataset] = []
+        for path in paths:
+            dataset = create_dataset(
+                parent=self, raw_file_paths=[path], internal_path_hints=self._get_previously_used_inner_paths(roleIndex)
+            )
+            if not dataset:
+                return
+            datasets.append(dataset)
+        self.addDatasets(datasets, startingLaneNum=startingLaneNum, roleIndex=roleIndex)
+
+    def addDatasets(self, datasets: Optional[List[Dataset]], startingLaneNum: Optional[int], roleIndex: int):
+        # If the user didn't cancel
+        axistags_hint = self.guess_axistags_for(role=roleIndex)
+        for dataset in datasets or []:
             try:
-                full_path = self._get_dataset_full_path(path, roleIndex=roleIndex)
-                info = self.instantiate_dataset_info(url=str(full_path), role=roleIndex)
+                info = FilesystemDatasetInfo(dataset=dataset, axistags_hint=axistags_hint)
                 self.addLanes([info], roleIndex=roleIndex, startingLaneNum=startingLaneNum)
             except DataSelectionGui.UserCancelledError:
                 pass
             except Exception as ex:
                 log_exception(logger)
                 QMessageBox.critical(self, "Error loading file", str(ex))
+        for dataset in datasets or []:
+            internal_paths = dataset.internal_paths()
+            if internal_paths:
+                self._add_default_internal_path(roleIndex=roleIndex, internal_path=internal_paths[0])
 
     def _findFirstEmptyLane(self, roleIndex):
         opTop = self.topLevelOperator
@@ -528,56 +557,23 @@ def _determineLaneRange(self, infos: List[DatasetInfo], startingLaneNum=None):
 
         return (startingLaneNum, endingLane)
 
-    def _add_default_inner_path(self, roleIndex: int, inner_path: str):
+    def _add_default_internal_path(self, roleIndex: int, internal_path: PurePosixPath):
         paths = self._default_h5n5_volumes.get(roleIndex, set())
-        paths.add(inner_path)
+        paths.add(internal_path)
         self._default_h5n5_volumes[roleIndex] = paths
 
-    def _get_previously_used_inner_paths(self, roleIndex: int) -> Set[str]:
+    def _get_previously_used_inner_paths(self, roleIndex: int) -> Set[PurePosixPath]:
         previous_paths = self._default_h5n5_volumes.get(roleIndex, set())
         return previous_paths.copy()
 
-    def _get_dataset_full_path(self, filePath: Path, roleIndex: int) -> Path:
-        if not DatasetInfo.fileHasInternalPaths(filePath):
-            return filePath
-        datasetNames = DatasetInfo.getPossibleInternalPathsFor(filePath.absolute())
-        if len(datasetNames) == 0:
-            raise RuntimeError(f"File {filePath} has no image datasets")
-        if len(datasetNames) == 1:
-            selected_dataset = datasetNames.pop()
-        else:
-            auto_inner_paths = self._get_previously_used_inner_paths(roleIndex).intersection(set(datasetNames))
-            if len(auto_inner_paths) == 1:
-                selected_dataset = auto_inner_paths.pop()
-            else:
-                # Ask the user which dataset to choose
-                dlg = SubvolumeSelectionDlg(datasetNames, self)
-                if dlg.exec_() != QDialog.Accepted:
-                    raise DataSelectionGui.UserCancelledError()
-                selected_index = dlg.combo.currentIndex()
-                selected_dataset = str(datasetNames[selected_index])
-        self._add_default_inner_path(roleIndex=roleIndex, inner_path=selected_dataset)
-        return filePath / selected_dataset.lstrip("/")
-
-    def guess_axistags_for(self, role: Union[str, int], info: DatasetInfo) -> Optional[AxisTags]:
+    def guess_axistags_for(self, role: Union[str, int]) -> Optional[AxisTagsHint]:
         if self.parentApplet.num_lanes == 0:
-            return info.axistags
+            return None
         lane = self.parentApplet.get_lane(-1)
         previous_info = lane.get_dataset_info(role)
         if previous_info is None or previous_info.default_tags == previous_info.axistags:
-            return info.axistags
-        if len(previous_info.axistags) != len(info.axistags) or previous_info.shape5d.c != info.shape5d.c:
-            return info.axistags
-        return previous_info.axistags
-
-    def instantiate_dataset_info(self, url: str, role: Union[str, int], *info_args, **info_kwargs) -> DatasetInfo:
-        info = self.parentApplet.create_dataset_info(url=url, *info_args, **info_kwargs)
-        if info_kwargs.get("axistags") is not None:
-            return info
-        axistags = self.guess_axistags_for(role=role, info=info)
-        if axistags in (info.axistags, None):
-            return info
-        return self.parentApplet.create_dataset_info(url=url, *info_args, **info_kwargs, axistags=axistags)
+            return None
+        return AxisTagsHint(axistags=previous_info.axistags, num_channels=int(previous_info.shape5d.c))
 
     def _checkDataFormatWarnings(self, roleIndex, startingLaneNum, endingLane):
         warn_needed = False
@@ -596,19 +592,25 @@ def _checkDataFormatWarnings(self, roleIndex, startingLaneNum, endingLane):
                 "(For HDF5 files, be sure to enable chunking on your dataset.)",
             )
 
-    def addStack(self, roleIndex, laneIndex):
+    def addStack(self, roleIndex: int, laneIndex):
         """
         The user clicked the "Import Stack Files" button.
         """
-        stackDlg = StackFileSelectionWidget(self)
+        stackDlg = DatasetSelectionWidget(selection_mode=DatasetSelectionMode.STACK)
         stackDlg.exec_()
-        if stackDlg.result() != QDialog.Accepted or not stackDlg.selectedFiles:
+        if stackDlg.result() != QDialog.Accepted:
             return
+        stack = stackDlg.selected_datasets[0]
 
         # FIXME: ask first if stack should be internalized to project file
         # also, check prefer_2d, size/volume and presence of 'z' to determine this
-        url = os.path.pathsep.join(stackDlg.selectedFiles)
-        stack_info = self.instantiate_dataset_info(url=url, role=roleIndex, sequence_axis=stackDlg.sequence_axis)
+        axistags_hint = self.guess_axistags_for(role=roleIndex)
+        stack_info = RelativeFilesystemDatasetInfo.create_or_fallback_to_absolute(
+            project_file=self.get_project_file(),
+            dataset=stack,
+            sequence_axis=stackDlg.stacking_axis,
+            axistags_hint=axistags_hint,
+        )
 
         try:
             # FIXME: do this inside a Request
@@ -645,15 +647,19 @@ def editDatasetInfo(self, roleIndex, laneIndexes):
         if editorDlg.exec_() == QDialog.Accepted:
             self.applyDatasetInfos(editorDlg.edited_infos, selected_info_slots)
 
-    def addPrecomputedVolume(self, roleIndex, laneIndex):
+    def addPrecomputedVolume(self, roleIndex: int, laneIndex):
         # add history...
         history = []
         browser = PrecomputedVolumeBrowser(history=history, parent=self)
 
         if browser.exec_() == PrecomputedVolumeBrowser.Rejected:
             return
-
-        info = self.instantiate_dataset_info(url=browser.selected_url, role=roleIndex)
+        selected_url = browser.selected_url
+        if selected_url is None:
+            return
+        precomputed_url = PrecomputedChunksUrl.create(selected_url)
+        axistags_hint = self.guess_axistags_for(role=roleIndex)
+        info = UrlDatasetInfo(url=precomputed_url, axistags_hint=axistags_hint)
         self.addLanes([info], roleIndex=roleIndex, startingLaneNum=laneIndex)
 
     def addDvidVolume(self, roleIndex, laneIndex):