CommandLineImporter.java

/*
 *   Copyright (C) 2009-2018 University of Dundee & Open Microscopy Environment.
 *   All rights reserved.
 *
 *   Use is subject to license terms supplied in LICENSE.txt
 */

package ome.formats.importer.cli;

import gnu.getopt.Getopt;
import gnu.getopt.LongOpt;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import loci.formats.in.DynamicMetadataOptions;
import loci.formats.in.MetadataLevel;
import loci.formats.meta.MetadataStore;
import ome.formats.OMEROMetadataStoreClient;
import ome.formats.importer.ImportCandidates;
import ome.formats.importer.ImportConfig;
import ome.formats.importer.ImportContainer;
import ome.formats.importer.ImportEvent;
import ome.formats.importer.ImportLibrary;
import ome.formats.importer.OMEROWrapper;
import ome.formats.importer.exclusions.AbstractFileExclusion;
import ome.formats.importer.exclusions.FileExclusion;
import ome.formats.importer.transfers.AbstractFileTransfer;
import ome.formats.importer.transfers.CleanupFailure;
import ome.formats.importer.transfers.FileTransfer;
import ome.formats.importer.transfers.UploadFileTransfer;
import omero.cmd.Response;
import omero.grid.ImportProcessPrx;
import omero.model.Annotation;
import omero.model.CommentAnnotationI;

import org.apache.commons.lang.time.StopWatch;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * The base entry point for the CLI version of the OMERO importer.
 *
 * @author Chris Allan <callan@glencoesoftware.com>
 * @author Josh Moore josh at glencoesoftware.com
 */
public class CommandLineImporter {

    public static final int DEFAULT_WAIT = -1;

    /** Logger for this class. */
    private static Logger log = LoggerFactory.getLogger(CommandLineImporter.class);

    /** StopWatch instance **/
    private final StopWatch sw = new StopWatch();

    /** Name that will be used for usage() */
    private static final String APP_NAME = "importer-cli";

    /** Configuration used by all components */
    public final ImportConfig config;

    /** {@link FileTransfer} mechanism to be used for uploading */
    public final FileTransfer transfer;

    /** {@link FileExclusion} mechanisms for skipping candidates */
    public final List<FileExclusion> exclusions = new ArrayList<FileExclusion>();

    /** Base importer library, this is what we actually use to import. */
    public final ImportLibrary library;

    /** ErrorHandler which is also responsible for uploading files */
    public final ErrorHandler handler;

    /** Bio-Formats reader wrapper customized for OMERO. */
    private final OMEROWrapper reader;

    /** Bio-Formats {@link MetadataStore} implementation for OMERO. */
    private final OMEROMetadataStoreClient store;

    /** Candidates for import */
    private final ImportCandidates candidates;

    /** If true, then only a report on used files will be produced */
    private final boolean getUsedFiles;

    /** Format which should be preferred for standard out messages */
    private ImportOutput importOutput = ImportOutput.ids;

    /**
     * Legacy constructor which uses a {@link UploadFileTransfer}.
     * @param config the import configuration
     * @param paths files or directories to search
     * @param getUsedFiles if only getting a list of used files
     * @throws Exception if the import could not be set up
     */
    public CommandLineImporter(final ImportConfig config, String[] paths,
            boolean getUsedFiles) throws Exception {
        this(config, paths, getUsedFiles, new UploadFileTransfer(), DEFAULT_WAIT);
    }

    /**
     * Legacy constructor without any file exclusions.
     * @param config the import configuration
     * @param paths files or directories to search
     * @param getUsedFiles if only getting a list of used files
     * @param transfer how files are to be transferred to the server
     * @param minutesToWait for how many minutes to wait for an import (negative for indefinitely)
     * @throws Exception if the import could not be set up
     */
    public CommandLineImporter(final ImportConfig config, String[] paths,
            boolean getUsedFiles, FileTransfer transfer, int minutesToWait)
                    throws Exception {
        this(config, paths, getUsedFiles, new UploadFileTransfer(), null, DEFAULT_WAIT);
    }

    /**
     * Main entry class for the application.
     * @param config the import configuration
     * @param paths files or directories to search
     * @param getUsedFiles if only getting a list of used files
     * @param transfer how files are to be transferred to the server
     * @param exclusions mechanisms to be used for skipping candidates
     * @param minutesToWait for how many minutes to wait for an import (negative for indefinitely)
     * @throws Exception if the import could not be set up
     */
    public CommandLineImporter(final ImportConfig config, String[] paths,
            boolean getUsedFiles, FileTransfer transfer,
            List<FileExclusion> exclusions, int minutesToWait)
                    throws Exception {

        this.config = config;
        config.loadAll();

        this.getUsedFiles = getUsedFiles;
        this.reader = new OMEROWrapper(config);
        this.handler = new ErrorHandler(config);
        this.transfer = transfer;
        if (exclusions != null) {
            this.exclusions.addAll(exclusions);
        }
        candidates = new ImportCandidates(reader, paths, handler);

        if (paths == null || paths.length == 0 || getUsedFiles) {

            store = null;
            library = null;

        } else {

            // Ensure that we have all of our required login arguments
            if (!config.canLogin()) {
                // config.requestFromUser(); // stdin if anything missing.
                usage(); // EXITS TODO this should check for a "quiet" flag
            }
            store = config.createStore();
            store.logVersionInfo(config.getIniVersionNumber());
            reader.setMetadataOptions(
                    new DynamicMetadataOptions(MetadataLevel.ALL));

            library = new ImportLibrary(store, reader,
                    transfer, exclusions, minutesToWait);

            if (config.checkUpgrade.get()) {
                config.isUpgradeNeeded();
            } else {
                log.debug("UpgradeCheck disabled.");
            }
        }

        Runtime.getRuntime().addShutdownHook(new Thread() {
            public void run() {
                cleanup();
            }
        });
    }

    /**
     * Look for all {@link ImportProcessPrx} in the current session and close
     * them if they return a non-null {@link Response} (i.e. they are done).
     * @param config the import configuration
     * @return how many import processes could not be accessed
     * @throws Exception in the event of error
     */
    public static int closeCompleted(ImportConfig config) throws Exception {
        config.loadAll();
        OMEROMetadataStoreClient client = config.createStore();
        ImportCloser closer = new ImportCloser(client);
        closer.closeCompleted();
        log.info("{} service(s) processed", closer.getProcessed());
        return closer.getErrors();
    }

    /**
     * Look for all {@link ImportProcessPrx} in the current session and close
     * them if they return a non-null {@link Response} (i.e. they are done).
     * @param config the import configuration
     * @return exit code, {@code 0} for success
     * @throws Exception in the event of error
     */
    public static int waitCompleted(ImportConfig config) throws Exception {
        long wait = 5000L;
        config.loadAll();
        OMEROMetadataStoreClient client = config.createStore();
        while (true) {
            ImportCloser closer = new ImportCloser(client);
            closer.closeCompleted();
            if (closer.getProcessed() == 0) {
                // In this case, there's nothing to do. Exit successfully.
                return 0;
            }
            int closed = closer.getClosed();
            int open = closer.getProcessed() - closed;
            int errs = closer.getErrors();
            if (errs > 0) {
                log.warn("{} open. {} closed. {} errors", open, closed, errs);
            } else {
                log.info("{} open. {} closed.", open, closed);
            }
            try {
                log.debug("Sleeping {} ms", wait);
                Thread.sleep(wait);
            } catch (Exception e) {
                // ignore
            }
        }
    }

    /**
     * Set the current {@link ImportOutput} (defaulting to null if
     * a null value is passed).
     *
     * @param importOutput possibly null enumeration value.
     * @return previous value.
     */
    public ImportOutput setImportOutput(ImportOutput importOutput) {
        ImportOutput old = importOutput;
        if (importOutput == null) {
            this.importOutput = ImportOutput.ids;
        }
        this.importOutput = importOutput;
        return old;
    }

    public int start() {
        boolean successful = true;

        if (getUsedFiles) {
            try {
                switch (importOutput) {
                    case yaml:
                        candidates.printYaml();
                        break;
                    default:
                        candidates.print();
                }
                return candidates.size() == 0 ? 125 : 0;
            } catch (Throwable t) {
                log.error("Error retrieving used files.", t);
                return 1;
            }
        } else if (candidates.size() < 1) {
            if (handler.errorCount() > 0) {
                System.err.println("No imports due to errors!");
                report();
            } else {
                System.err.println("No imports found");
                try {
                    cleanup(); // #5426 Preventing close exceptions.
                } finally {
                    usage();
                }
            }
        } else {
            sw.start();
            LoggingImportMonitor lim = new LoggingImportMonitor();
            lim.setImportOutput(importOutput);
            library.addObserver(lim);
            // error handler has been configured in constructor from main args
            library.addObserver(this.handler);

            successful = library.importCandidates(config, candidates);

            try {
                List<String> paths = new ArrayList<String>();
                for (ImportContainer ic : candidates.getContainers()) {
                    paths.addAll(Arrays.asList(ic.getUsedFiles()));
                }

                // No exceptions are thrown from importCandidates and therefore
                // we must manually check for the number of errors. If there
                // are **ANY** then we refuse to post-process these paths,
                // which primarily only means that MoveFileTransfer will not
                // get a chance to delete the files.
                transfer.afterTransfer(handler.errorCount(), paths);

            } catch (CleanupFailure e) {
                log.error("Failed to cleanup {} files", e.getFailedFiles()
                        .size());
                return 3;
            } finally {
                sw.stop();
                report();
            }
        }

        return successful ? 0 : 2;
    }

    void report() {
        boolean report = config.sendReport.get();
        boolean files = config.sendFiles.get();
        boolean logs = config.sendLogFile.get();
        if (report) {
            handler.update(null, new ImportEvent.DEBUG_SEND(files, logs));
        }
        library.notifyObservers(new ImportEvent.IMPORT_SUMMARY(sw.getTime(),
                handler.errorCount()));
    }

    /**
     * Cleans up after a successful or unsuccessful image import. This method
     * only does the minimum required cleanup, so that it can be called
     * during shutdown.
     */
    public void cleanup() {
        if (store != null) {
            store.logout();
        }
    }

    /**
     * Prints usage to STDERR and exits with return code 1.
     */
    public static void usage() {
        System.err.println(String.format("\n"
            + " Usage:  %s [OPTION]... [path [path ...]]... \n"
            + "   or:   %s [OPTION]... - \n"
            + "\n"
            + "Import any number of files into an OMERO instance.\n"
            + "If \"-\" is the only path, a list of files or directories \n"
            + "is read from standard in. Directories will be searched for \n"
            + "all valid imports.\n"
            + "\n"
            + "Session arguments:\n"
            + "  Mandatory arguments for creating a session are 1- either the OMERO server hostname,\n"
            + "username and password or 2- the OMERO server hostname and a valid session key.\n"
            + "  -s SERVER\tOMERO server hostname\n"
            + "  -u USER\tOMERO username\n"
            + "  -w PASSWORD\tOMERO password\n"
            + "  -k KEY\tOMERO session key (UUID of an active session)\n"
            + "  -p PORT\tOMERO server port (default: 4064)\n"
            + "\n"
            + "Naming arguments:\n"
            + "All naming arguments are optional\n"
            + "  -n NAME\t\t\t\tImage or plate name to use\n"
            + "  -x DESCRIPTION\t\t\tImage or plate description to use\n"
            + "  --name NAME\t\t\t\tImage or plate name to use\n"
            + "  --description DESCRIPTION\t\tImage or plate description to use\n"
            + "\n"
            + "Optional arguments:\n"
            + "  -h\t\t\t\t\tDisplay this help and exit\n"
            + "  -f\t\t\t\t\tDisplay the used files and exit\n"
            + "  -c\t\t\t\t\tContinue importing after errors\n"
            + "  -l READER_FILE\t\t\tUse the list of readers rather than the default\n"
            + "  -d DATASET_ID\t\t\t\tOMERO dataset ID to import image into\n"
            + "  -r SCREEN_ID\t\t\t\tOMERO screen ID to import plate into\n"
            + "  -T TARGET\t\t\t\ttarget for imports\n"

            + "  --report\t\t\t\tReport errors to the OME team\n"
            + "  --upload\t\t\t\tUpload broken files and log file (if any) with report. Required --report\n"
            + "  --logs\t\t\t\tUpload log file (if any) with report. Required --report\n"
            + "  --email EMAIL\t\t\t\tEmail for reported errors. Required --report\n"
            + "  --debug LEVEL\t\t\t\tTurn debug logging on (optional level)\n"
            + "  --annotation-ns ANNOTATION_NS\t\tNamespace to use for subsequent annotation\n"
            + "  --annotation-text ANNOTATION_TEXT\tContent for a text annotation\n"
            + "  --annotation-link ANNOTATION_LINK\tComment annotation ID to link all images to\n"
            + "\n"
            + "Examples:\n"
            + "\n"
            + "  $ %s -s localhost -u user -w password -d 50 foo.tiff\n"
            + "  $ %s -s localhost -u user -w password -d Dataset:50 foo.tiff\n"
            + "  $ %s -f foo.tiff\n"
            + "  $ %s -s localhost -u username -w password -d 50 --debug ALL foo.tiff\n"
            + "\n"
            + "For additional information, see:\n"
            + "https://docs.openmicroscopy.org/latest/omero/users/cli/import.html\n"
            + "Report bugs to <ome-users@lists.openmicroscopy.org.uk>",
            APP_NAME, APP_NAME, APP_NAME, APP_NAME, APP_NAME, APP_NAME));
        System.exit(1);
    }

    /**
     * Prints advanced usage to STDERR and exits with return code 1.
     */
    public static void advUsage() {
        System.err.println("\n"
            + "ADVANCED OPTIONS:\n\n"
            + "  These options are not intended for general use. Make sure you have read the\n"
            + "  documentation regarding them. They may change in future releases.\n\n"
            + "  In-place imports:\n"
            + "  -----------------\n\n"
            + "    --transfer=ARG          \tFile transfer method\n\n"
            + "        General options:    \t\n"
            + "          upload          \t# Default\n"
            + "          upload_rm       \t# Caution! File upload followed by source deletion.\n"
            + "          some.class.Name \t# Use a class on the CLASSPATH.\n\n"
            + "        Server-side options:\t\n"
            + "          ln              \t# Use hard-link.\n"
            + "          ln_s            \t# Use soft-link.\n"
            + "          ln_rm           \t# Caution! Hard-link followed by source deletion.\n"
            + "          cp              \t# Use local copy command.\n"
            + "          cp_rm           \t# Caution! Copy followed by source deletion.\n\n"
            + "\n"
            + "  e.g. $ bin/omero import --transfer=ln_s foo.tiff\n"
            + "       $ ./importer-cli --transfer=ln bar.tiff\n"
            + "       $ CLASSPATH=mycode.jar ./importer-cli --transfer=com.example.MyTransfer baz.tiff\n"
            + "\n"
            + "  Background imports:\n"
            + "  -------------------\n\n"
            + "    --auto-close            \tClose completed imports immediately.\n\n"
            + "    --minutes-wait=ARG      \tChoose how long the importer will wait on server-side processing.\n"
            + "                            \tARG > 0 implies the number of minutes to wait.\n"
            + "                            \tARG = 0 exits immediately. Use a *-completed option to clean up.\n"
            + "                            \tARG < 0 waits indefinitely. This is the default.\n\n"
            + "    --close-completed       \tClose completed imports.\n\n"
            + "    --wait-completed        \tWait for all background imports to complete.\n\n"
            + "\n"
            + "  e.g. $ bin/omero import -- --minutes-wait=0 file1.tiff file2.tiff file3.tiff\n"
            + "       $ ./importer-cli --minutes-wait=0 some_directory/\n"
            + "       $ ./importer-cli --wait-completed # Waits on all 3 imports.\n"
            + "\n"
            + "  File exclusion:\n"
            + "  ---------------\n\n"
            + "    --exclude=filename      \tExclude files based on filename.\n\n"
            + "    --exclude=clientpath    \tExclude files based on the original path.\n\n"
            + "\n"
            + "  e.g. $ bin/omero import --exclude=filename foo.tiff # First-time imports\n"
            + "       $ bin/omero import --exclude=filename foo.tiff # Second-time skips\n"
            + "\n"
            + "  Import speed:\n"
            + "  -------------\n\n"
            + "    --checksum-algorithm=ARG\tChoose a possibly faster algorithm for detecting file corruption,\n"
            + "                            \te.g. Adler-32 (fast), CRC-32 (fast), File-Size-64 (fast),\n"
            + "                            \t     MD5-128, Murmur3-32, Murmur3-128,\n"
            + "                            \t     SHA1-160 (slow, default)\n\n"
            + "  e.g. $ bin/omero import --checksum-algorithm=CRC-32 foo.tiff\n"
            + "       $ ./importer-cli --checksum-algorithm=Murmur3-128 bar.tiff\n\n"
            + "    --no-stats-info\t\tDisable calculation of minima and maxima"
            + " when as part of the Bio-Formats reader metadata\n\n"
            + "  e.g. $ bin/omero import -- --no-stats-info foo.tiff\n"
            + "       $ ./importer-cli --no-stats-info bar.tiff\n\n"
            + "  --no-thumbnails\t\tDo not perform thumbnailing after import\n\n"
            + "  e.g. $ bin/omero import -- --no-thumbnails foo.tiff\n"
            + "       $ ./importer-cli --no-thumbnails bar.tiff\n\n"
            + "    --no-upgrade-check\t\tDisable upgrade check for each import\n"
            + "  e.g. $ bin/omero import -- --no-upgrade-check foo.tiff\n"
            + "       $ ./importer-cli --no-upgrade-check bar.tiff\n\n"
            + "\n"
            + "  Feedback:\n"
            + "  ---------\n\n"
            + "    --qa-baseurl=ARG\tSpecify the base URL for reporting feedback\n"
            + "  e.g. $ bin/omero import broken_image.tif"
            + " -- --email EMAIL --report --upload --logs"
            + " --qa-baseurl=http://qa.example.com\n"
            + "       $ ./importer-cli broken_image.tif"
            + " --email EMAIL --report --upload --logs"
            + " --qa-baseurl=http://qa.openmicroscopy.org.uk/qa\n"
            + "\n"
            + "Report bugs to <ome-users@lists.openmicroscopy.org.uk>");
        System.exit(1);
    }

    /**
     * Takes pairs of namespaces and string and creates comment annotations
     * from each pair.
     * @param namespaces Namespaces to use.
     * @param strings Strings to use.
     * @return List of comment annotations.
     */
    private static List<Annotation> toTextAnnotations(
                   List<String> namespaces, List<String> strings)
    {
        List<Annotation> annotations = new ArrayList<Annotation>();
        CommentAnnotationI annotation;
        if (namespaces.size() != strings.size())
        {
            String ns = null;
            if (namespaces.size() == 1) {
                ns = namespaces.get(0);
            }
            for(int i = 0; i < strings.size(); i++)
            {
                annotation = new CommentAnnotationI();
                if (ns != null) {
                    annotation.setTextValue(omero.rtypes.rstring(ns));
                }
                annotation.setTextValue(omero.rtypes.rstring(strings.get(i)));
                annotations.add(annotation);
            }
        } else {
            for(int i = 0; i < namespaces.size(); i++)
            {
                annotation = new CommentAnnotationI();
                annotation.setNs(omero.rtypes.rstring(namespaces.get(i)));
                annotation.setTextValue(omero.rtypes.rstring(strings.get(i)));
                annotations.add(annotation);
            }
        }
        return annotations;
    }

    /**
     * Command line application entry point which parses CLI arguments and
     * passes them into the importer. Return codes for import are:
     * <ul>
     * <li>0 on success</li>
     * <li>1 on argument parsing failure</li>
     * <li>2 on exception during import</li>
     * </ul>
     *
     * Return codes for the "-f" option (getUsedFiles) are:
     * <ul>
     * <li>0 on success, even if errors exist in the files</li>
     * <li>1 only if an exception propagates up the stack</li>
     * </ul>
     * @param args
     *            Command line arguments.
     * @throws Exception in the event of error
     */
    public static void main(String[] args) throws Exception {

        int minutesToWait = DEFAULT_WAIT;
        FileTransfer transfer = new UploadFileTransfer();
        ImportConfig config = new ImportConfig();

        // Defaults
        config.email.set("");
        config.sendFiles.set(false);
        config.sendLogFile.set(false);
        config.sendReport.set(false);
        config.contOnError.set(false);
        config.parallelUpload.set(1);
        config.parallelFileset.set(1);
        config.debug.set(false);
        config.encryptedConnection.set(false);

        // PLEASE KEEP THIS IN SYNC WITH omero/plugins/import.py

        LongOpt debug = new LongOpt(
                "debug", LongOpt.OPTIONAL_ARGUMENT, null, 1);
        LongOpt report = new LongOpt("report", LongOpt.NO_ARGUMENT, null, 2);
        LongOpt upload = new LongOpt("upload", LongOpt.NO_ARGUMENT, null, 3);
        LongOpt logs = new LongOpt("logs", LongOpt.NO_ARGUMENT, null, 4);
        LongOpt email = new LongOpt(
                "email", LongOpt.REQUIRED_ARGUMENT, null, 5);
        LongOpt name = new LongOpt(
                "name", LongOpt.REQUIRED_ARGUMENT, null, 6);
        LongOpt description = new LongOpt(
                "description", LongOpt.REQUIRED_ARGUMENT, null, 7);
        LongOpt noThumbnails = new LongOpt(
                "no-thumbnails", LongOpt.NO_ARGUMENT, null, 8);
        LongOpt agent = new LongOpt(
                "agent", LongOpt.REQUIRED_ARGUMENT, null, 9);
        LongOpt annotationNamespace =
            new LongOpt("annotation-ns", LongOpt.REQUIRED_ARGUMENT, null, 10);
        LongOpt annotationText =
            new LongOpt("annotation-text", LongOpt.REQUIRED_ARGUMENT,
                        null, 11);
        LongOpt annotationLink =
            new LongOpt("annotation-link", LongOpt.REQUIRED_ARGUMENT,
                        null, 12);

        // ADVANCED OPTIONS
        LongOpt advancedHelp =
                new LongOpt("advanced-help", LongOpt.NO_ARGUMENT, null, 13);
        LongOpt transferOpt =
                new LongOpt("transfer", LongOpt.REQUIRED_ARGUMENT, null, 14);
        LongOpt checksumAlgorithm =
                new LongOpt("checksum-algorithm", LongOpt.REQUIRED_ARGUMENT, null, 15);
        LongOpt minutesWait =
                new LongOpt("minutes-wait", LongOpt.REQUIRED_ARGUMENT, null, 16);
        LongOpt closeCompleted =
                new LongOpt("close-completed", LongOpt.NO_ARGUMENT, null, 17);
        LongOpt waitCompleted =
                new LongOpt("wait-completed", LongOpt.NO_ARGUMENT, null, 18);
        LongOpt autoClose =
                new LongOpt("auto-close", LongOpt.NO_ARGUMENT, null, 19);
        LongOpt exclude =
                new LongOpt("exclude", LongOpt.REQUIRED_ARGUMENT, null, 20);

        LongOpt target =
                new LongOpt("target", LongOpt.REQUIRED_ARGUMENT, null, 21);

        LongOpt qaBaseURL = new LongOpt(
                "qa-baseurl", LongOpt.REQUIRED_ARGUMENT, null, 22);

        LongOpt noStatsInfo =
                new LongOpt("no-stats-info", LongOpt.NO_ARGUMENT, null, 23);
        LongOpt noUpgradeCheck =
                new LongOpt("no-upgrade-check", LongOpt.NO_ARGUMENT, null, 24);

        LongOpt outputFormat =
                new LongOpt("output", LongOpt.REQUIRED_ARGUMENT, null, 25);

        LongOpt encryptedConnection =
                new LongOpt("encrypted", LongOpt.REQUIRED_ARGUMENT, null, 26);

        LongOpt parallelUpload =
                new LongOpt("parallel-upload", LongOpt.REQUIRED_ARGUMENT, null, 27);

        LongOpt parallelFileset =
                new LongOpt("parallel-fileset", LongOpt.REQUIRED_ARGUMENT, null, 28);

        LongOpt readers =
                new LongOpt("readers", LongOpt.REQUIRED_ARGUMENT, null, 29);

        // DEPRECATED OPTIONS
        LongOpt minutesWaitDeprecated =
                new LongOpt("minutes_wait", LongOpt.REQUIRED_ARGUMENT, null, 86);
        LongOpt closeCompletedDeprecated =
                new LongOpt("close_completed", LongOpt.NO_ARGUMENT, null, 87);
        LongOpt waitCompletedDeprecated =
                new LongOpt("wait_completed", LongOpt.NO_ARGUMENT, null, 88);
        LongOpt autoCloseDeprecated =
                new LongOpt("auto_close", LongOpt.NO_ARGUMENT, null, 89);
        LongOpt plateName = new LongOpt(
                "plate_name", LongOpt.REQUIRED_ARGUMENT, null, 90);
        LongOpt plateDescription = new LongOpt(
                "plate_description", LongOpt.REQUIRED_ARGUMENT, null, 91);
        LongOpt noThumbnailsDeprecated = new LongOpt(
                "no_thumbnails", LongOpt.NO_ARGUMENT, null, 92);
        LongOpt checksumAlgorithmDeprecated = new LongOpt(
                "checksum_algorithm", LongOpt.REQUIRED_ARGUMENT, null, 93);
        LongOpt annotationNamespaceDeprecated =
            new LongOpt("annotation_ns", LongOpt.REQUIRED_ARGUMENT, null, 94);
        LongOpt annotationTextDeprecated =
            new LongOpt("annotation_text", LongOpt.REQUIRED_ARGUMENT, null, 95);
        LongOpt annotationLinkDeprecated =
            new LongOpt("annotation_link", LongOpt.REQUIRED_ARGUMENT, null, 96);
        LongOpt plateDescription2 = new LongOpt(
                "plate-description", LongOpt.REQUIRED_ARGUMENT, null, 97);
        LongOpt plateName2= new LongOpt(
                "plate-name", LongOpt.REQUIRED_ARGUMENT, null, 98);

        Getopt g = new Getopt(APP_NAME, args, "cfl:s:u:w:d:r:T:k:x:n:p:h",
                new LongOpt[] { debug, report, upload, logs, email,
                                name, description, noThumbnails,
                                agent, annotationNamespace, annotationText,
                                annotationLink, transferOpt, advancedHelp,
                                checksumAlgorithm, minutesWait,
                                closeCompleted, waitCompleted, autoClose,
                                exclude, target, noStatsInfo,
                                noUpgradeCheck, qaBaseURL,
                                outputFormat, encryptedConnection,
                                parallelUpload, parallelFileset,
                                readers,
                                plateName, plateName2,
                                plateDescription, plateDescription2,
                                noThumbnailsDeprecated,
                                checksumAlgorithmDeprecated,
                                annotationNamespaceDeprecated,
                                annotationTextDeprecated,
                                annotationLinkDeprecated,
                                minutesWaitDeprecated,
                                closeCompletedDeprecated,
                                waitCompletedDeprecated,
                                autoCloseDeprecated
                                });
        int a;
        ImportOutput outputChoice = ImportOutput.ids;

        boolean doCloseCompleted = false;
        boolean doWaitCompleted = false;
        boolean getUsedFiles = false;
        config.agent.set("importer-cli");

        // Create a map for handling conflicting option strings
        Map<String, Boolean> conflictingArguments = new HashMap<String, Boolean>();
        conflictingArguments.put("userSpecifiedName", false);
        conflictingArguments.put("userSpecifiedDescription", false);
        conflictingArguments.put("checksumAlgorithm", false);

        List<String> annotationNamespaces = new ArrayList<String>();
        List<String> textAnnotations = new ArrayList<String>();
        List<Long> annotationIds = new ArrayList<Long>();
        List<FileExclusion> exclusions = new ArrayList<FileExclusion>();
        while ((a = g.getopt()) != -1) {
            switch (a) {
            case 1: {
                config.configureDebug(g.getOptarg());
                break;
            }
            case 2: {
                config.sendReport.set(true);
                break;
            }
            case 3: {
                config.sendFiles.set(true);
                break;
            }
            case 4: {
                config.sendLogFile.set(true);
                break;
            }
            case 5: {
                config.email.set(g.getOptarg());
                break;
            }
            case 6: {
                setArgument(conflictingArguments, "userSpecifiedName");
                config.userSpecifiedName.set(g.getOptarg());
                break;
            }
            case 7: {
                setArgument(conflictingArguments, "userSpecifiedDescription");
                config.userSpecifiedDescription.set(g.getOptarg());
                break;
            }
            case 8: {
                log.info("Skipping thumbnails creation");
                config.doThumbnails.set(false);
                break;
            }
            case 9: {
                config.agent.set(g.getOptarg());
                break;
            }
            case 10: {
                annotationNamespaces.add(g.getOptarg());
                break;
            }
            case 11: {
                textAnnotations.add(g.getOptarg());
                break;
            }
            case 12: {
                annotationIds.add(Long.parseLong(g.getOptarg()));
                break;
            }
            case 13: {
                advUsage();
                break;
            }

            // ADVANCED START -------------------------------------------------
            case 14: {
                String arg = g.getOptarg();
                log.info("Setting transfer to {}", arg);
                transfer = AbstractFileTransfer.createTransfer(arg);
                break;
            }
            case 15: {
                setArgument(conflictingArguments, "checksumAlgorithm");
                String arg = g.getOptarg();
                log.info("Setting checksum algorithm to {}", arg);
                config.checksumAlgorithm.set(arg);
                break;
            }
            case 16:
            case 86: {
                minutesToWait = Integer.parseInt(g.getOptarg());
                log.info("Setting minutes to wait to {}", minutesToWait);
                break;
            }
            case 17:
            case 87: {
                doCloseCompleted = true;
                break;
            }
            case 18:
            case 88: {
                doWaitCompleted = true;
                break;
            }
            case 19:
            case 89: {
                minutesToWait = 0;
                config.autoClose.set(true);
                break;
            }
            case 20: {
                String arg = g.getOptarg();
                log.info("Adding exclusion: {}", arg);
                FileExclusion exclusion = AbstractFileExclusion.createExclusion(arg);
                if (exclusion != null) {
                    exclusions.add(exclusion);
                }
                break;
            }
            case 'T':
            case 21: {
                config.target.set(g.getOptarg());
                break;
            }
            case 22: {
                config.qaBaseURL.set(g.getOptarg());
                break;
            }
            case 23: {
                log.info("Skipping minimum/maximum computation");
                config.noStatsInfo.set(true);
                break;
            }
            case 24: {
                log.info("Disabling upgrade check");
                config.checkUpgrade.set(false);
                break;
            }
            case 25: {
                String outputArg = g.getOptarg();
                log.info("Setting output format: {}", outputArg);
                outputChoice = ImportOutput.valueOf(outputArg);
                break;
            }
            case 26: {
                String encryptedArg = g.getOptarg();
                log.info("Setting encrypted: {}", encryptedArg);
                config.encryptedConnection.set(Boolean.valueOf(encryptedArg));
                break;
            }
            case 27: {
                String parallelFArg = g.getOptarg();
                log.info("Setting parallel upload: {}", parallelFArg);
                config.parallelUpload.set(Integer.valueOf(parallelFArg));
                break;
            }
            case 28: {
                String parallelUArg = g.getOptarg();
                log.info("Setting parallel fileset: {}", parallelUArg);
                config.parallelFileset.set(Integer.valueOf(parallelUArg));
                break;
            }
            // ADVANCED END ---------------------------------------------------
            // DEPRECATED OPTIONS
            case 90:
            case 98: {
                setArgument(conflictingArguments, "userSpecifiedName");
                config.userSpecifiedName.set(g.getOptarg());
                break;
            }
            case 91:
            case 97: {
                setArgument(conflictingArguments, "userSpecifiedDescription");
                config.userSpecifiedDescription.set(g.getOptarg());
                break;
            }
            case 92: {
                log.info("Skipping thumbnails creation");
                config.doThumbnails.set(false);
                break;
            }
            case 93: {
                setArgument(conflictingArguments, "checksumAlgorithm");
                String arg = g.getOptarg();
                log.info("Setting checksum algorithm to {}", arg);
                config.checksumAlgorithm.set(arg);
                break;
            }
            case 94: {
                annotationNamespaces.add(g.getOptarg());
                break;
            }
            case 95: {
                textAnnotations.add(g.getOptarg());
                break;
            }
            case 96: {
                annotationIds.add(Long.parseLong(g.getOptarg()));
                break;
            }
            // END OF DEPRECATED OPTIONS
            case 's': {
                config.hostname.set(g.getOptarg());
                break;
            }
            case 'u': {
                config.username.set(g.getOptarg());
                break;
            }
            case 'w': {
                config.password.set(g.getOptarg());
                break;
            }
            case 'k': {
                config.sessionKey.set(g.getOptarg());
                break;
            }
            case 'p': {
                config.port.set(Integer.parseInt(g.getOptarg()));
                break;
            }
            case 'd': {
                String datasetString = g.getOptarg();
                if (!datasetString.startsWith("Dataset:")) {
                    datasetString = "Dataset:" + Long.valueOf(datasetString).toString();
                }
                config.target.set(datasetString);
                break;
            }
            case 'r': {
                String screenString = g.getOptarg();
                if (!screenString.startsWith("Screen:")) {
                    screenString = "Screen:" + Long.valueOf(screenString).toString();
                }
                config.target.set(screenString);
                break;
            }
            case 'n': {
                setArgument(conflictingArguments, "userSpecifiedName");
                config.userSpecifiedName.set(g.getOptarg());
                break;
            }
            case 'x': {
                setArgument(conflictingArguments, "userSpecifiedDescription");
                config.userSpecifiedDescription.set(g.getOptarg());
                break;
            }
            case 'f': {
                getUsedFiles = true;
                break;
            }
            case 'c': {
                config.contOnError.set(true);
                break;
            }
            case 'l':
            case 29: {
                config.readersPath.set(g.getOptarg());
                break;
            }
            case 'h': {
                usage(); // exits
            }
            default: {
                usage(); // exits
            }
            }
        }

        // Let the user know at what level we're logging
        log.info(String.format(
                "Log levels -- Bio-Formats: %s OMERO.importer: %s",
                ((ch.qos.logback.classic.Logger)LoggerFactory
                    .getLogger("loci")).getLevel(),
                ((ch.qos.logback.classic.Logger)LoggerFactory
                    .getLogger("ome.formats")).getLevel()));

        // Start the importer and import the image we've been given
        String[] rest = new String[args.length - g.getOptind()];
        System.arraycopy(args, g.getOptind(), rest, 0, args.length
                - g.getOptind());

        if (doCloseCompleted || doWaitCompleted) {
            if (rest.length > 0) {
                log.error("Files found with completed option: "+
                        Arrays.toString(rest));
                System.exit(-2); // EARLY EXIT!
            } else if (doCloseCompleted) {
                System.exit(closeCompleted(config)); // EARLY EXIT!
            } else if (doWaitCompleted) {
                System.exit(waitCompleted(config)); // EARLY EXIT!
            }
        }

        List<Annotation> annotations =
            toTextAnnotations(annotationNamespaces, textAnnotations);
        for (Long id: annotationIds)
        {
            CommentAnnotationI unloadedAnnotation =
                new CommentAnnotationI(id, false);
            annotations.add(unloadedAnnotation);
        }
        config.annotations.set(annotations);

        CommandLineImporter c = null;
        int rc = 0;
        try {

            if (rest.length == 1 && "-".equals(rest[0])) {
                rest = stdin();
            }
            c = new CommandLineImporter(config, rest, getUsedFiles,
                    transfer, exclusions, minutesToWait);
            c.setImportOutput(outputChoice);
            rc = c.start();
        } catch (Ice.DNSException dnse) {
            log.error("Failed to look up domain name: {}", dnse.host);
            rc = 2;
        } catch (Throwable t) {
            log.error("Error during import process.", t);
            rc = 2;
        } finally {
            if (c != null) {
                c.cleanup();
            }
        }
        System.exit(rc);
    }

    /**
     * Set a conflicting argument and return the usage if the key is already
     * set
     * @param map map of conflicting properties
     * @param key configuration property to be set
     */
    public static void setArgument(Map <String, Boolean> map, String key) {
        if (map.get(key)) {
            // The property has already been set
            log.error("Conflicting arguments setting {}.", key);
            usage();
        }
        map.put(key, true);
    }

    /**
     * Reads a list of paths from stdin.
     * @return the paths
     */
    static String[] stdin() throws IOException {
        BufferedReader in = new BufferedReader(new InputStreamReader(System.in));
        List<String> files = new ArrayList<String>();
        while (true) {
            String str = in.readLine();
            if (str == null) {
                break;
            } else {
                str = str.trim();
                if (str.length() > 0) {
                    files.add(str);
                }
            }
        }
        return files.toArray(new String[0]);
    }

}