Merge pull request #115 from ilastik/export-import-refactoring

Import/export command refactoring

README.md

@@ -251,37 +251,15 @@ as `ImagePlus`, containing `ImageProcessors` to access the underlying data. We t
 containers everywhere which are roughly wrapped as `Dataset > ImgPlus > Img >
 RandomAccessibleInterval`.
 
-### Demo
+### Examples
 
-`src/test/java/org/ilastik/ilastik4ij/Demo.java` contains usage examples of various workflows.
+If you want to run workflows from your own Java code, have a look at
+`src/test/java/org/ilastik/ilastik4ij/Demo.java`.
+Example macros can be found in the `examples` directory.
 
 ### Manually test in a local Fiji installation
 
-If you have [Fiji][fiji] installed locally somewhere in your system, it is
-possible to automatically copy the plugin into the Fiji's plugins directory.
-You need to specify path to your Fiji installation in local `.m2/settings.xml`:
-
-```xml
-<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
-          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-          xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 https://maven.apache.org/xsd/settings-1.0.0.xsd">
-    <profiles>
-        <profile>
-            <id>local-install-to-fiji</id>
-            <properties>
-                <fiji.home>/Applications/Fiji.app</fiji.home>
-            </properties>
-        </profile>
-    </profiles>
-</settings>
-```
-
-POM of this project has a build profile that will be automatically activated
-when the property `fiji.home` is defined, so you can run  `mvn install`
-or `mvn install -DskipTests` to put a fresh build of the plugin into Fiji.
-
-Make sure that the released plugin version is not installed in Fiji, and that
-`local-install-to-fiji` profile is not disabled.
+Run `scripts/fiji-deploy.py` to build project and copy the newly built plugin to Fiji.
 
 ### Deployment
 

examples/export.ijm

@@ -0,0 +1,15 @@
+// Lines that start with "//" are comments.
+
+// Lines that start with "#@" declare input parameters that interactively ask the user for values.
+// If you want to run a macro in batch or headless mode, remove those lines and set input parameters explicitly.
+
+#@ File (label = "Input file", style = "file") input
+// input = "/absolute/path/to/some/directory/src/test/resources/2d_cells_apoptotic.tif";
+
+#@ File (label = "Output file", style = "save") exportPath
+// exportPath = "/path/to/some/directory/output.h5";
+
+datasetName = "/data";
+compressionLevel = 0;
+
+run("Export HDF5", "input=[" + input + "] datasetname=[" + datasetName + "] compressionlevel=[" + compressionLevel + "] exportpath=[" + exportPath + "]");

examples/import.ijm

@@ -0,0 +1,7 @@
+#@ File (label = "Input file", style = "file") input
+// input = "/absolute/path/to/some/directory/src/test/resources/2d_cells_apoptotic_1channel.h5";
+
+datasetname = "/data";
+axisorder = "tzyxc";
+
+run("Import HDF5", "select=[" + input + "] datasetname=[" + datasetname + "] axisorder=[" + axisorder + "]");

examples/object_classification.ijm

@@ -0,0 +1,13 @@
+#@ File (label = "Project file", style = "file") project
+// project = "/absolute/path/to/some/directory/obj_class_2d_cells_apoptotic.ilp";
+
+#@ File (label = "Input file", style = "file") input
+// input = "/absolute/path/to/some/directory/2d_cells_apoptotic.tif";
+
+#@ File (label = "Second input file", style = "file") secondInput
+// secondInput = "/absolute/path/to/some/directory/2d_cells_apoptotic_1channel-data_Probabilities.tif";
+
+type = "Probabilities";
+
+open(input);
+run("Run Object Classification Prediction", "projectfilename=[" + project + "] input=[" + input + "] inputproborsegimage=[" + secondInput + "] secondinputtype=[" + type + "]");

examples/pixel_classification.ijm

@@ -1,5 +1,10 @@
-project = "src/test/resources/pixel_class_2d_cells_apoptotic.ilp";
-input = "2d_cells_apoptotic.tif";
+#@ File (label = "Project file", style = "file") project
+// project = "/absolute/path/to/some/directory/pixel_class_2d_cells_apoptotic.ilp";
+
+#@ File (label = "Input file", style = "file") input
+// input = "/absolute/path/to/some/directory/2d_cells_apoptotic.tif";
+
+type = "Probabilities";
 
 open(input);
-run("Run Pixel Classification Prediction", "projectfilename=" + project + " input=" + input + " pixelclassificationtype=Probabilities");
+run("Run Pixel Classification Prediction", "projectfilename=[" + project + "] input=[" + input + "] pixelclassificationtype=[" + type + "]");

pom.xml

@@ -86,47 +86,6 @@
         </repository>
     </repositories>
 
-    <profiles>
-        <profile>
-            <id>local-install-to-fiji</id>
-            <activation>
-                <property>
-                    <name>fiji.home</name>
-                </property>
-            </activation>
-            <build>
-                <plugins>
-                    <plugin>
-                        <groupId>org.apache.maven.plugins</groupId>
-                        <artifactId>maven-dependency-plugin</artifactId>
-                        <version>3.6.0</version>
-                        <executions>
-                            <execution>
-                                <id>copy-installed</id>
-                                <phase>install</phase>
-                                <goals>
-                                    <goal>copy</goal>
-                                </goals>
-                                <configuration>
-                                    <artifactItems>
-                                        <artifactItem>
-                                            <groupId>${project.groupId}</groupId>
-                                            <artifactId>${project.artifactId}</artifactId>
-                                            <version>${project.version}</version>
-                                            <type>${project.packaging}</type>
-                                        </artifactItem>
-                                    </artifactItems>
-                                    <outputDirectory>${fiji.home}/plugins</outputDirectory>
-                                    <stripVersion>true</stripVersion>
-                                </configuration>
-                            </execution>
-                        </executions>
-                    </plugin>
-                </plugins>
-            </build>
-        </profile>
-    </profiles>
-
     <dependencies>
         <dependency>
             <groupId>org.json</groupId>

scripts/fiji-deploy.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+
+"""Build the project and copy plugin to Fiji."""
+
+import argparse
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("fiji", type=Path, help="Path to Fiji directory")
+    args = ap.parse_args()
+
+    if not shutil.which("mvn"):
+        sys.exit("error: maven not found in system path")
+
+    print("cleaning project")
+    subprocess.run(["mvn", "clean"], check=True)
+
+    print("building project")
+    subprocess.run(["mvn", "package", "-DskipTests=true"], check=True)
+
+    jars = list(Path("target").glob("ilastik4ij-*-SNAPSHOT.jar"))
+    if len(jars) != 1:
+        sys.exit(f"error: expected exactly one jar, found {jars}")
+    jar = jars[0]
+
+    for subdir in ("plugins", "jars"):
+        for old_jar in (args.fiji / subdir).glob("ilastik4ij*.jar"):
+            print(f"removing {old_jar}")
+            old_jar.unlink()
+
+    plugins_dir = args.fiji / "plugins"
+    print(f"copying {jar} to {plugins_dir}")
+    shutil.copy(jar, plugins_dir)
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except KeyboardInterrupt:
+        pass

scripts/generate-dataset.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 """Generate HDF5 dataset of the specified size filled with zeros."""
 
@@ -9,16 +9,8 @@
 import numpy
 
 
-def parse_shape(s: str) -> tuple[int, ...]:
-    return tuple(map(int, reversed(s.split(","))))
-
-
-def show_shape(shape: tuple[int, ...]) -> str:
-    return ",".join(map(str, reversed(shape)))
-
-
 def human_size(shape: tuple[int, ...], itemsize: int) -> str:
-    n = numpy.prod(shape) * itemsize
+    n = int(numpy.prod(shape)) * itemsize
     suffixes = "bytes", "KiB", "MiB", "GiB", "TiB", "PiB"
     suffix = suffixes[0]
     for suffix in suffixes:
@@ -43,35 +35,37 @@ def main():
     ap.add_argument(
         "-s",
         "--shape",
-        help="comma-separated column-major shape (default: 64,64,3,64,10)",
-        default="64,64,3,64,10",
+        help="comma-separated shape (default: 10,64,3,64,64)",
+        default="10,64,3,64,64",
     )
     ap.add_argument(
         "-c",
         "--chunk",
-        help="comma-separated column-major chunk shape (default: no chunking)",
+        help="comma-separated chunk shape (default: no chunking)",
     )
     args = ap.parse_args()
 
     path = args.path
     dataset = "/" + args.dataset.removeprefix("/")
     dtype = numpy.dtype(args.dtype)
-    shape = parse_shape(args.shape)
-    chunk = parse_shape(args.chunk) if args.chunk is not None else None
+    shape = tuple(map(int, args.shape))
+    chunk = tuple(map(int, args.chunk)) if args.chunk is not None else None
 
     report = {
         "path": path,
         "dataset": dataset,
         "dtype": str(dtype),
-        "shape": show_shape(shape),
+        "shape": ",".join(map(str, shape)),
         "shape_size": human_size(shape, dtype.itemsize),
     }
     if chunk is not None:
-        report["chunk"] = show_shape(chunk)
+        report["chunk"] = ",".join(map(str, chunk))
         report["chunk_size"] = human_size(chunk, dtype.itemsize)
     print(json.dumps(report, indent=2))
 
-    with h5py.File(path, "w") as f:
+    with h5py.File(path, "a") as f:
+        if dataset in f:
+            del f[dataset]
         ds = f.create_dataset(dataset, shape=shape, dtype=dtype, chunks=chunk)
         ds.write_direct(numpy.zeros(shape, dtype=dtype))
 

scripts/inspect-hdf5.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+
+"""Print all datasets in an HDF5 file."""
+
+import argparse
+import json
+
+import numpy
+import h5py
+
+
+def to_str(seq):
+    return "[" + ",".join(map(to_str, seq)) + "]" if isinstance(seq, list) else str(seq)
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("hdf5", type=str, help="Path to HDF5 file")
+    args = ap.parse_args()
+
+    datasets = {}
+
+    def visit(name, obj):
+        attrs = {}
+        for k, v in obj.attrs.items():
+            if isinstance(v, bytes):
+                v = v.decode("utf-8")
+            elif isinstance(v, numpy.ndarray):
+                v = to_str(v.tolist())
+            attrs[k] = v
+
+        if isinstance(obj, h5py.Dataset):
+            name = "/" + name.removeprefix("/")
+            datasets[name] = {
+                "dtype": obj.dtype.name,
+                "shape": ",".join(map(str, obj.shape)),
+            }
+            if attrs:
+                datasets[name]["attrs"] = {k: attrs[k] for k in sorted(attrs)}
+
+
+    with h5py.File(args.hdf5, "r") as f:
+        f.visititems(visit)
+
+    datasets = {k: datasets[k] for k in sorted(datasets)}
+    print(json.dumps(datasets, indent=2))
+
+
+if __name__ == "__main__":
+    main()

src/main/java/org/ilastik/ilastik4ij/hdf5/DatasetDescription.java

@@ -75,7 +75,7 @@ static Optional<DatasetDescription> ofHdf5(IHDF5Reader reader, String path) {
             axesGuessed = true;
         }
 
-        path = "/" + path.replaceFirst("/+", "");
+        path = "/" + path.replaceFirst("^/+", "");
         return Optional.of(new DatasetDescription(path, type.get(), dims, axes, axesGuessed));
     }