prepare can take a filelist instead of folder path (#55)

* added filelist argument

* generating targets from filelist when necessary

* putting initialization where it should be

* prepare with filelist seems to be working!

* prepare filelist working, tests passing

* using simpler pip install for tests

* removing unnecessary export

* raising an exception if a plate has no CommentAnnotation

.omero/cli-build

@@ -22,5 +22,4 @@ conda activate omero
 
 export OMERO_DIST=${OMERO_DIST:-/opt/omero/server/OMERO.server}
 omero $PLUGIN -h
-
 python setup.py test -t test -i ${OMERO_DIST}/etc/ice.config -vs

.omero/py-check

@@ -6,7 +6,6 @@ set -u
 set -x
 
 TARGET=${TARGET:-..}
-
 cd $TARGET
 if [ -f .pre-commit-config.yaml ]; then
     pre-commit run -a

.omero/py-setup

@@ -23,7 +23,6 @@ conda install -c bioconda bftools
 
 cd $TARGET
 cd $(setup_dir)
-python setup.py sdist
-pip install -U dist/*.tar.gz
+pip install .
 python setup.py clean
 rm -rf dist *egg-info src/*egg-info

README.md

@@ -13,20 +13,20 @@ in that object, plus an XML file detailing the links between entities, annotatio
 The CLI plugin add the subcommand `transfer`, which in its turn has two further subcommands `omero transfer pack` and `omero transfer unpack`. Both subcommands (pack and unpack) will use an existing OMERO session created via CLI or prompt the user for parameters to create one.
 
 # Installation
-tl;dr: if you have `python>=3.7`, a simple `pip install omero-cli-transfer` _might_ do. We recommend conda, though.
+tl;dr: if you have `python>=3.8`, a simple `pip install omero-cli-transfer` _might_ do. We recommend conda, though.
 
-`omero-cli-transfer` requires at least Python 3.7. This is due to `ome-types` requiring that as well;
+`omero-cli-transfer` requires at least Python 3.8. This is due to `ome-types` requiring that as well;
 this package relies heavily on it, and it is not feasible without it. 
 
 Of course, this CAN be an issue, especially given `omero-py` _officially_ only supports Python 3.6. However,
-it is possible to run `omero-py` in Python 3.7 or newer as well. Our recommended way to do so it using `conda`.
+it is possible to run `omero-py` in Python 3.8 or newer as well. Our recommended way to do so it using `conda`.
 With conda installed, you can do
 ```
-conda create -n myenv -c conda-force python=3.7 zeroc-ice==3.6.5
+conda create -n myenv -c conda-force python=3.8 zeroc-ice==3.6.5
 conda activate myenv
 pip install omero-cli-transfer
 ```
-It is possible to do the same thing without `conda` as long as your python/pip version is at least 3.7,
+It is possible to do the same thing without `conda` as long as your python/pip version is at least 3.8,
 but that will require locally building a wheel for `zeroc-ice` (which pip does automatically) - it is a
 process that can be anything from "completely seamless and without issues" to "I need to install every 
 dependency ever imagined". Try at your own risk.

src/generate_omero_objects.py

@@ -150,6 +150,7 @@ def create_plate_map(ome: OME, img_map: dict, conn: BlitzGateway
     map_ref_ids = []
     for plate in ome.plates:
         ann_ids = [i.id for i in plate.annotation_ref]
+        file_path = ""
         for ann in ome.structured_annotations:
             if (ann.id in ann_ids and
                     type(ann) == CommentAnnotation and
@@ -159,6 +160,9 @@ def create_plate_map(ome: OME, img_map: dict, conn: BlitzGateway
                 file_path = ann.value
         q = conn.getQueryService()
         params = Parameters()
+        if not file_path:
+            raise ValueError(f"Plate ID {plate.id} does not have a \
+                             CommentAnnotation with a file path!")
         path_query = str(file_path).strip('/')
         if path_query.endswith('mock_folder'):
             path_query = path_query.rstrip("mock_folder")

src/generate_xml.py

@@ -463,34 +463,43 @@ def create_provenance_metadata(conn: BlitzGateway, img_id: int,
     return kv, ref
 
 
-def create_objects(folder):
+def create_objects(folder, filelist):
     img_files = []
-    for path, subdirs, files in os.walk(folder):
-        for f in files:
-            img_files.append(os.path.abspath(os.path.join(path, f)))
-    targets = copy.deepcopy(img_files)
     cli = CLI()
     cli.loadplugins()
-    for img in img_files:
-        if img not in (targets):
-            continue
-        cmd = ["omero", 'import', '-f', img, "\n"]
-        res = cli.popen(cmd, stdout=PIPE, stderr=DEVNULL)
-        std = res.communicate()
-        files = parse_files_import(std[0].decode('UTF-8'))
-        if len(files) > 1:
+    if not filelist:
+        for path, subdirs, files in os.walk(folder):
             for f in files:
-                targets.remove(f)
-            targets.append(img)
-        if len(files) == 0:
-            targets.remove(img)
+                img_files.append(os.path.abspath(os.path.join(path, f)))
+        targets = copy.deepcopy(img_files)
+        for img in img_files:
+            if img not in (targets):
+                continue
+            cmd = ["omero", 'import', '-f', img, "\n"]
+            res = cli.popen(cmd, stdout=PIPE, stderr=DEVNULL)
+            std = res.communicate()
+            files = parse_files_import(std[0].decode('UTF-8'))
+            if len(files) > 1:
+                for f in files:
+                    targets.remove(f)
+                targets.append(img)
+            if len(files) == 0:
+                targets.remove(img)
+    else:
+        with open(folder, "r") as f:
+            targets_str = f.read().splitlines()
+        targets = []
+        par_folder = Path(folder).parent
+        for target in targets_str:
+            targets.append(str((par_folder / target).resolve()))
     images = []
     plates = []
     annotations = []
     counter_imgs = 1
     counter_pls = 1
     for target in targets:
-        print(f"Processing file {target}\n")
+        target = str(Path(target).absolute())
+        print(f"Processing file {target}")
         res = run_showinf(target, cli)
         imgs, pls, anns = parse_showinf(res,
                                         counter_imgs, counter_pls, target)
@@ -854,20 +863,23 @@ def populate_xml(datatype: str, id: int, filepath: str, conn: BlitzGateway,
     return ome, path_id_dict
 
 
-def populate_xml_folder(folder: str, conn: BlitzGateway, session: str
-                        ) -> Tuple[OME, dict]:
+def populate_xml_folder(folder: str, filelist: bool, conn: BlitzGateway,
+                        session: str) -> Tuple[OME, dict]:
     ome = OME()
-    images, plates, annotations = create_objects(folder)
+    images, plates, annotations = create_objects(folder, filelist)
     ome.images = images
     ome.plates = plates
     ome.structured_annotations = annotations
-    filepath = str(Path(folder) / "transfer.xml")
-    if Path(folder).exists():
-        with open(filepath, 'w') as fp:
-            print(to_xml(ome), file=fp)
-            fp.close()
+    if filelist:
+        filepath = str(Path(folder).parent.resolve() / "transfer.xml")
     else:
-        raise ValueError("Folder cannot be found!")
+        if Path(folder).exists():
+            filepath = str(Path(folder) / "transfer.xml")
+        else:
+            raise ValueError("Folder cannot be found!")
+    with open(filepath, 'w') as fp:
+        print(to_xml(ome), file=fp)
+        fp.close()
     path_id_dict = list_file_ids(ome)
     return ome, path_id_dict
 

src/omero_cli_transfer.py

@@ -119,8 +119,21 @@
 is intended as a first step on a bulk-import workflow, followed by using
 `omero transfer unpack` to complete an import.
 
+Note: images imported from an XML generated with this tool will have whichever
+names `showinf` reports them to have; that is, the names on their internal
+metadata, which might be different from filenames. For multi-image files,
+image names follow the pattern "filename [imagename]", where 'imagename' is
+the one reported by `showinf`.
+
+--filelist allows you to specify a text file containing a list of file paths
+(one per line). Relative paths should be relative to the location of the file
+list. The XML file will only take those files into consideration.
+The resulting `transfer.xml` file will be created on the same directory of
+your file list.
+
 Examples:
 omero transfer prepare /home/user/folder_with_files
+omero transfer prepare --filelist /home/user/file_with_paths.txt
 """)
 
 
@@ -206,6 +219,9 @@ def _configure(self, parser):
         )
         folder_help = ("Path to folder with image files")
         prepare.add_argument("folder", type=str, help=folder_help)
+        prepare.add_argument(
+            "--filelist", help="Pass path to a filelist rather than a folder",
+            action="store_true")
 
     @gateway_required
     def pack(self, args):
@@ -546,7 +562,8 @@ def _make_image_map(self, source_map: dict, dest_map: dict) -> dict:
         return imgmap
 
     def __prepare(self, args):
-        populate_xml_folder(args.folder, self.gateway, self.session)
+        populate_xml_folder(args.folder, args.filelist, self.gateway,
+                            self.session)
         return
 
 

test/data/prepare/filelist.txt

@@ -0,0 +1,2 @@
+test_pyramid.ome.tif
+default-plate&plates=1.fake

test/data/prepare/transfer.xml

@@ -2,50 +2,32 @@
     <Plate ID="Plate:1" Name="Plate Name 0">
         <Well Column="0" ID="Well:0_0_0_0" Row="0" Color="255" ExternalDescription="External Description" ExternalIdentifier="External Identifier" Type="Transfection: done">
             <WellSample ID="WellSample:0_0_0_0_0_0" Index="0" PositionX="0.0" PositionY="1.0" Timepoint="2006-05-04T18:13:51">
-                <ImageRef ID="Image:1" />
+                <ImageRef ID="Image:2" />
             </WellSample>
         </Well>
-        <AnnotationRef ID="Annotation:-245185625979654701490316406351781182304" />
+        <AnnotationRef ID="Annotation:-166540319990038501817496698575818747072" />
     </Plate>
-    <Image ID="Image:1" Name="default-plate">
-        <Pixels DimensionOrder="XYZCT" ID="Pixels:1" SizeC="1" SizeT="1" SizeX="512" SizeY="512" SizeZ="1" Type="uint8">
+    <Image ID="Image:1" Name="test_pyramid.tiff">
+        <Pixels DimensionOrder="XYCZT" ID="Pixels:1" SizeC="1" SizeT="1" SizeX="16" SizeY="16" SizeZ="1" Type="uint8">
             <MetadataOnly />
         </Pixels>
-        <AnnotationRef ID="Annotation:-172939569407233550101909030829557976341" />
+        <AnnotationRef ID="Annotation:-86643574848290778120287886298910882783" />
     </Image>
-    <Image ID="Image:2" Name="test_pyramid.tiff">
-        <Pixels DimensionOrder="XYCZT" ID="Pixels:2" SizeC="1" SizeT="1" SizeX="16" SizeY="16" SizeZ="1" Type="uint8">
+    <Image ID="Image:2" Name="default-plate">
+        <Pixels DimensionOrder="XYZCT" ID="Pixels:2" SizeC="1" SizeT="1" SizeX="512" SizeY="512" SizeZ="1" Type="uint8">
             <MetadataOnly />
         </Pixels>
-        <AnnotationRef ID="Annotation:-324306227964107542000952667676993603912" />
-    </Image>
-    <Image ID="Image:3" Name="vsi-ets-test-jpg2k.vsi [001 C405, C488]">
-        <Pixels DimensionOrder="XYCZT" ID="Pixels:3" SizeC="2" SizeT="1" SizeX="1645" SizeY="1682" SizeZ="11" Type="uint16">
-            <MetadataOnly />
-        </Pixels>
-        <AnnotationRef ID="Annotation:-257696975305797818629314594066683015430" />
-    </Image>
-    <Image ID="Image:4" Name="vsi-ets-test-jpg2k.vsi [macro image]">
-        <Pixels DimensionOrder="XYCZT" ID="Pixels:4" SizeC="3" SizeT="1" SizeX="501" SizeY="512" SizeZ="1" Type="uint8">
-            <MetadataOnly />
-        </Pixels>
-        <AnnotationRef ID="Annotation:-145147221356188951573746383297119544119" />
+        <AnnotationRef ID="Annotation:-134337008044623056588408027657468159176" />
     </Image>
     <StructuredAnnotations>
-        <CommentAnnotation ID="Annotation:-172939569407233550101909030829557976341" Namespace="Image:1">
-            <Value>/mnt/c/Users/erick/Documents/GitHub/omero-cli-transfer/test/data/prepare/default-plate&amp;plates=1.fake</Value>
-        </CommentAnnotation>
-        <CommentAnnotation ID="Annotation:-245185625979654701490316406351781182304" Namespace="Plate:1">
-            <Value>/mnt/c/Users/erick/Documents/GitHub/omero-cli-transfer/test/data/prepare/default-plate&amp;plates=1.fake</Value>
-        </CommentAnnotation>
-        <CommentAnnotation ID="Annotation:-324306227964107542000952667676993603912" Namespace="Image:2">
+        <CommentAnnotation ID="Annotation:-86643574848290778120287886298910882783" Namespace="Image:1">
             <Value>/mnt/c/Users/erick/Documents/GitHub/omero-cli-transfer/test/data/prepare/test_pyramid.ome.tif</Value>
         </CommentAnnotation>
-        <CommentAnnotation ID="Annotation:-257696975305797818629314594066683015430" Namespace="Image:3">
-            <Value>/mnt/c/Users/erick/Documents/GitHub/omero-cli-transfer/test/data/prepare/vsi-ets-test-jpg2k.vsi</Value>
+        <CommentAnnotation ID="Annotation:-134337008044623056588408027657468159176" Namespace="Image:2">
+            <Value>/mnt/c/Users/erick/Documents/GitHub/omero-cli-transfer/test/data/prepare/default-plate&amp;plates=1.fake</Value>
         </CommentAnnotation>
-        <CommentAnnotation ID="Annotation:-145147221356188951573746383297119544119" Namespace="Image:4">
-            <Value>/mnt/c/Users/erick/Documents/GitHub/omero-cli-transfer/test/data/prepare/vsi-ets-test-jpg2k.vsi</Value>
+        <CommentAnnotation ID="Annotation:-166540319990038501817496698575818747072" Namespace="Plate:1">
+            <Value>/mnt/c/Users/erick/Documents/GitHub/omero-cli-transfer/test/data/prepare/default-plate&amp;plates=1.fake</Value>
         </CommentAnnotation>
     </StructuredAnnotations>
 </OME>

test/integration/test_prepare.py

@@ -15,7 +15,6 @@
 from ome_types.model import AnnotationRef, ROI, ROIRef, Rectangle
 from ome_types.model.screen import PlateRef
 from ome_types.model.map import M, Map
-from generate_xml import populate_xml_folder
 
 import ezomero
 import pytest
@@ -25,6 +24,10 @@
         "test/data/prepare/",
 ]
 
+TEST_FILELISTS = [
+        "test/data/prepare/filelist.txt"
+]
+
 
 class TestPrepare(CLITest):
 
@@ -80,10 +83,8 @@ def test_prepare_clean(self, folder):
         if Path(folder / 'transfer.xml').exists():
             print('transfer.xml exists! deleting.')
             os.remove(str(folder / 'transfer.xml'))
-        if Path(folder / 'transfer.xml').exists():
-            print('transfer.xml still exists???')
-        _, _ = populate_xml_folder(str(folder),
-                                   self.gw, self.session)
+        args = self.args + ["prepare", str(folder)]
+        self.cli.invoke(args, strict=True)
         assert Path(folder / 'transfer.xml').exists()
         assert os.path.getsize(str(folder / 'transfer.xml')) > 0
         args = self.args + ["unpack", "--folder", str(folder)]
@@ -110,6 +111,23 @@ def test_prepare_edited(self, folder):
         if Path(folder / 'transfer.xml').exists():
             os.remove(str(folder / 'transfer.xml'))
 
+    @pytest.mark.parametrize('filelist', sorted(TEST_FILELISTS))
+    def test_prepare_filelist(self, filelist):
+        folder = Path(filelist).parent
+        if Path(folder / 'transfer.xml').exists():
+            print('transfer.xml exists! deleting.')
+            os.remove(str(folder / 'transfer.xml'))
+        args = self.args + ["prepare", "--filelist", str(filelist)]
+        self.cli.invoke(args, strict=True)
+        assert Path(folder / 'transfer.xml').exists()
+        assert os.path.getsize(str(folder / 'transfer.xml')) > 0
+        args = self.args + ["unpack", "--folder", str(folder)]
+        self.cli.invoke(args, strict=True)
+        self.run_asserts_filelist()
+        self.delete_all()
+        if Path(folder / 'transfer.xml').exists():
+            os.remove(str(folder / 'transfer.xml'))
+
     def run_asserts_clean(self):
         img_ids = ezomero.get_image_ids(self.gw)
         assert len(img_ids) == 3
@@ -119,6 +137,16 @@ def run_asserts_clean(self):
             img_names.append(img.getName())
         assert "vsi-ets-test-jpg2k.vsi [macro image]" in img_names
 
+    def run_asserts_filelist(self):
+        img_ids = ezomero.get_image_ids(self.gw)
+        assert len(img_ids) == 1
+        img_names = []
+        for i in (img_ids):
+            img, _ = ezomero.get_image(self.gw, i, no_pixels=True)
+            img_names.append(img.getName())
+        assert "vsi-ets-test-jpg2k.vsi [macro image]" not in img_names
+        assert "test_pyramid.tiff" in img_names
+
     def run_asserts_edited(self):
         img_ids = ezomero.get_image_ids(self.gw)
         assert len(img_ids) == 0