Merge pull request #2055 from mtbc/trac-11877-import-log

fix #11877: move import logs to upload jobs instead of annotations

components/blitz/resources/omero/api/IMetadata.ice

@@ -1,7 +1,7 @@
 /*
  *   $Id$
  *
- *   Copyright 2010 Glencoe Software, Inc. All rights reserved.
+ *   Copyright 2010-2014 Glencoe Software, Inc. All rights reserved.
  *   Use is subject to license terms supplied in LICENSE.txt
  *
  */
@@ -49,6 +49,7 @@ module omero {
                                                                    string rootNodeType,
                                                                    omero::sys::LongList rootNodeIds,
                                                                    omero::sys::Parameters options) throws ServerError;
+                idempotent LongIObjectListMap loadLogFiles(string rootType, omero::sys::LongList ids) throws ServerError;
             };
 
     };

components/blitz/src/ome/formats/importer/ImportLibrary.java

@@ -22,7 +22,6 @@
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -66,19 +65,16 @@
 import omero.grid.ManagedRepositoryPrxHelper;
 import omero.grid.RepositoryMap;
 import omero.grid.RepositoryPrx;
-import omero.model.Annotation;
 import omero.model.ChecksumAlgorithm;
 import omero.model.Dataset;
-import omero.model.FileAnnotation;
-import omero.model.FileAnnotationI;
 import omero.model.Fileset;
 import omero.model.FilesetI;
+import omero.model.IObject;
 import omero.model.OriginalFile;
 import omero.model.Pixels;
 import omero.model.Screen;
-import omero.sys.Parameters;
-import omero.sys.ParametersI;
 
+import org.apache.commons.collections.CollectionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -546,29 +542,24 @@ protected Long loadLogFile() throws ServerError {
             final ImportRequest req = (ImportRequest) handle.getRequest();
             final Long fsId = req.activity.getParent().getId().getValue();
             final IMetadataPrx metadataService = sf.getMetadataService();
-            final List<String> nsToInclude = new ArrayList<String>(
-                    Arrays.asList(omero.constants.namespaces.NSLOGFILE.value));
-            final List<String> nsToExclude = new ArrayList<String>();
-            final List<Long> rootIds = new ArrayList<Long>(Arrays.asList(fsId));
-            final Parameters param = new ParametersI();
-            Map<Long,List<Annotation>> annotationMap = new HashMap<Long,List<Annotation>>();
-            List<Annotation> annotations = new ArrayList<Annotation>();
-            Long ofId = null;
+            final List<Long> rootIds = Collections.singletonList(fsId);
             try {
-                annotationMap = metadataService.loadSpecifiedAnnotationsLinkedTo(
-                        FileAnnotation.class.getName(), nsToInclude, nsToExclude,
-                        Fileset.class.getName(), rootIds, param);
-                if (annotationMap.containsKey(fsId)) {
-                    annotations = annotationMap.get(fsId);
-                    if (annotations.size() != 0) {
-                        FileAnnotation fa = (FileAnnotationI) annotations.get(0);
-                        ofId = fa.getFile().getId().getValue();
+                final Map<Long, List<IObject>> logMap = metadataService.loadLogFiles(Fileset.class.getName(), rootIds);
+                final List<IObject> logs = logMap.get(fsId);
+                if (CollectionUtils.isNotEmpty(logs)) {
+                    for (final IObject log : logs) {
+                        if (log instanceof OriginalFile) {
+                            final Long ofId = log.getId().getValue();
+                            if (ofId != null) {
+                                return ofId;
+                            }
+                        }
                     }
                 }
             } catch (ServerError e) {
-                ofId = null;
+                log.debug("failed to load log file", e);
             }
-            return ofId;
+            return null;
         }
 
         @Override

components/blitz/src/ome/services/blitz/impl/MetadataI.java

@@ -2,7 +2,7 @@
  *  $Id$
  *
  *------------------------------------------------------------------------------
- *  Copyright (C) 2006-2009 University of Dundee. All rights reserved.
+ *  Copyright (C) 2006-2014 University of Dundee. All rights reserved.
  *
  *
  * 	This program is free software; you can redistribute it and/or modify
@@ -20,8 +20,8 @@
  *
  *------------------------------------------------------------------------------
  */
-package ome.services.blitz.impl;
 
+package ome.services.blitz.impl;
 
 import java.util.List;
 
@@ -36,6 +36,7 @@
 import omero.api.AMD_IMetadata_loadAnnotationsUsedNotOwned;
 import omero.api.AMD_IMetadata_loadChannelAcquisitionData;
 import omero.api.AMD_IMetadata_loadInstrument;
+import omero.api.AMD_IMetadata_loadLogFiles;
 import omero.api.AMD_IMetadata_loadSpecifiedAnnotations;
 import omero.api.AMD_IMetadata_loadSpecifiedAnnotationsLinkedTo;
 import omero.api.AMD_IMetadata_loadTagContent;
@@ -203,7 +204,14 @@ public void loadSpecifiedAnnotationsLinkedTo_async(AMD_IMetadata_loadSpecifiedAn
 		 callInvokerOnRawArgs(__cb, __current, annotationType, include, exclude, 
 				 rootNodeType, nodeIds, options);
 	 }
-
+
+    @Override
+    public void loadLogFiles_async(AMD_IMetadata_loadLogFiles __cb,
+            String rootType, List<Long> ids, Current __current)
+                    throws ServerError {
+        callInvokerOnRawArgs(__cb, __current, rootType, ids);
+    }
+
 	protected void map(List<String> annotationTypes) throws ServerError
 	{
 		if (annotationTypes == null)

components/blitz/src/ome/services/blitz/repo/ManagedImportProcessI.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2012-2013 Glencoe Software, Inc. All rights reserved.
+ * Copyright (C) 2012-2014 Glencoe Software, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -37,15 +37,13 @@
 import omero.ServerError;
 import omero.api.RawFileStorePrx;
 import omero.cmd.HandlePrx;
-import omero.constants.CLIENTUUID;
 import omero.grid.ImportLocation;
 import omero.grid.ImportProcessPrx;
 import omero.grid.ImportProcessPrxHelper;
 import omero.grid.ImportRequest;
 import omero.grid.ImportSettings;
 import omero.grid._ImportProcessOperations;
 import omero.grid._ImportProcessTie;
-import omero.model.ChecksumAlgorithm;
 import omero.model.Fileset;
 import omero.model.FilesetJobLink;
 
@@ -281,8 +279,7 @@ public HandlePrx verifyUpload(List<String> hashes, Current __current)
         // Now move on to the metadata import.
         link = fs.getFilesetJobLink(1);
         CheckedPath checkedPath = ((ManagedImportLocationI) location).getLogFile();
-        omero.model.OriginalFile logFile =
-                repo.registerLogFile(repo.getRepoUuid(),  fs.getId().getValue(), checkedPath,__current);
+        final omero.model.OriginalFile logFile = repo.findInDb(checkedPath, "r", __current);
 
         final String reqId = ImportRequest.ice_staticId();
         final ImportRequest req = (ImportRequest)

components/blitz/src/ome/services/blitz/repo/ManagedRepositoryI.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2012-2013 Glencoe Software, Inc. All rights reserved.
+ * Copyright (C) 2012-2014 Glencoe Software, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -36,6 +36,7 @@
 import ome.api.local.LocalAdmin;
 import ome.formats.importer.ImportConfig;
 import ome.formats.importer.ImportContainer;
+import ome.model.core.OriginalFile;
 import ome.services.blitz.gateway.services.util.ServiceUtilities;
 import ome.services.blitz.repo.path.ClientFilePathTransformer;
 import ome.services.blitz.repo.path.FilePathNamingValidator;
@@ -324,6 +325,12 @@ protected ImportProcessPrx createImportProcess(Fileset fs,
 
         fs.linkJob(new IndexingJobI());
 
+        if (location instanceof ManagedImportLocationI) {
+            OriginalFile of = ((ManagedImportLocationI) location).getLogFile().asOriginalFile(IMPORT_LOG_MIMETYPE);
+            of = persistLogFile(of, __current);
+            job.linkOriginalFile((omero.model.OriginalFile) new IceMapper().map(of));
+        }
+
         // Create CheckedPath objects for use by saveFileset
         final int size = fs.sizeOfUsedFiles();
         final List<CheckedPath> checked = new ArrayList<CheckedPath>();

components/blitz/src/ome/services/blitz/repo/PublicRepositoryI.java

@@ -2,7 +2,7 @@
  * ome.services.blitz.repo.PublicRepositoryI
  *
  *------------------------------------------------------------------------------
- *  Copyright (C) 2006-2013 University of Dundee. All rights reserved.
+ *  Copyright (C) 2006-2014 University of Dundee. All rights reserved.
  *
  *
  *  This program is free software; you can redistribute it and/or modify
@@ -57,8 +57,6 @@
 import ome.api.RawFileStore;
 import ome.formats.importer.ImportConfig;
 import ome.formats.importer.OMEROWrapper;
-import ome.model.annotations.FileAnnotation;
-import ome.model.annotations.FilesetAnnotationLink;
 import ome.services.blitz.impl.AbstractAmdServant;
 import ome.services.blitz.impl.ServiceFactoryI;
 import ome.services.blitz.repo.path.FilePathRestrictionInstance;
@@ -101,7 +99,6 @@
 import omero.grid._RepositoryTie;
 import omero.model.ChecksumAlgorithm;
 import omero.model.OriginalFile;
-import omero.model.OriginalFileI;
 import omero.model.enums.ChecksumAlgorithmSHA1160;
 import omero.util.IceMapper;
 
@@ -140,7 +137,10 @@ public void ice_exception(Exception ex) {
     /**
      * Mimetype used to connote a directory {@link OriginalFile} object.
      */
-    public static String DIRECTORY_MIMETYPE = "Directory";
+    public static final String DIRECTORY_MIMETYPE = "Directory";
+
+    /** media type for import logs */
+    public static final String IMPORT_LOG_MIMETYPE = "application/omero-log-file";
 
     private /*final*/ long id;
 
@@ -476,49 +476,31 @@ public Object doWork(Session session, ServiceFactory sf) {
         }
 
     /**
-     * Should be refactored elsewhere.
-     * @param checkedPath 
-     * @param current
+     * Set the repository of the given original file to be this one.
+     * TODO: Should be refactored elsewhere.
+     * @param originalFileId the ID of the log file
+     * @param current the Ice method invocation context
      */
     @Deprecated
-    protected OriginalFile registerLogFile(final String repoUuid, final long filesetId,
-            final CheckedPath checkedPath, Ice.Current current)
+    protected ome.model.core.OriginalFile persistLogFile(final ome.model.core.OriginalFile originalFile, Ice.Current current)
                 throws ServerError {
 
         final Executor executor = this.context.getBean("executor", Executor.class);
         final Map<String, String> ctx = current.ctx;
         final String session = ctx.get(omero.constants.SESSIONUUID.value);
         final String group = ctx.get(omero.constants.GROUP.value);
         final Principal principal = new Principal(session, group, null);
-        final String LOG_FILE_NS =
-                omero.constants.namespaces.NSLOGFILE.value;
 
         try {
-            FilesetAnnotationLink link = (FilesetAnnotationLink)
-                    executor.execute(ctx, principal, new Executor.SimpleWork(this, "setOriginalFileHasherToSHA1", id) {
+            return (ome.model.core.OriginalFile) executor.execute(ctx, principal,
+                    new Executor.SimpleWork(this, "persistLogFile", id) {
                 @Transactional(readOnly = false)
-                public Object doWork(Session session, ServiceFactory sf) {
-
-                    ome.model.core.OriginalFile logFile = null;
-                    try {
-                         logFile = repositoryDao.register(repoUuid,
-                            checkedPath, "text/plain", sf, getSqlAction());
-                    } catch (ServerError se) {
-                        throw new RuntimeException("Failed to register log file", se);
-                    }
-
-                    // use sf to get the services to link Fileset and the OriginalFile
-                    ome.api.IUpdate iUpdate = sf.getUpdateService();
-                    ome.model.annotations.FileAnnotation fa = new ome.model.annotations.FileAnnotation();
-                    fa.setNs(LOG_FILE_NS);
-                    fa.setFile(logFile.proxy());
-                    FilesetAnnotationLink fsl = new FilesetAnnotationLink();
-                    fsl.link(new ome.model.fs.Fileset(filesetId, false), fa);
-                    return iUpdate.saveAndReturnObject(fsl);
+                public ome.model.core.OriginalFile doWork(Session session, ServiceFactory sf) {
+                    final ome.model.core.OriginalFile persisted = sf.getUpdateService().saveAndReturnObject(originalFile);
+                    getSqlAction().setFileRepo(persisted.getId(), repoUuid);
+                    return persisted;
                 }
             });
-            FileAnnotation fs = (FileAnnotation) link.child();
-            return new OriginalFileI(fs.getFile().getId(), false);
         } catch (Exception e) {
             throw (ServerError) new IceMapper().handleException(e, executor.getContext());
         }

components/common/src/ome/api/IMetadata.java

@@ -1,5 +1,5 @@
 /*
- *   Copyright (C) 2006-2011 University of Dundee & Open Microscopy Environment.
+ *   Copyright (C) 2006-2014 University of Dundee & Open Microscopy Environment.
  *   All rights reserved.
  *
  *   Use is subject to license terms supplied in LICENSE.txt
@@ -269,5 +269,13 @@ public <A extends Annotation> Map<Long, Set<A>> loadSpecifiedAnnotationsLinkedTo
     		@NotNull Class rootNodeType,
     		@NotNull @Validate(Long.class) Set<Long> rootNodeIds,
     		Parameters options);
-
+
+    /**
+     * Find the original file IDs for the import logs corresponding to the given Image or Fileset IDs.
+     * @param rootNodeType the root node type, may be {@link omero.model.core.Image} or {@link omero.model.fs.Fileset}
+     * @param ids the IDs of the entities for which the import log original file IDs are required
+     * @return the original file IDs of the import logs
+     */
+    public Map<Long, Set<IObject>> loadLogFiles(@NotNull Class<? extends IObject> rootNodeType,
+            @Validate(Long.class) Set<Long> ids);
 }

components/insight/SRC/org/openmicroscopy/shoola/env/data/OMEROGateway.java

@@ -2,10 +2,10 @@
  * org.openmicroscopy.shoola.env.data.OMEROGateway
  *
  *------------------------------------------------------------------------------
- *  Copyright (C) 2006-2013 University of Dundee. All rights reserved.
+ *  Copyright (C) 2006-2014 University of Dundee. All rights reserved.
  *
  *
- * 	This program is free software; you can redistribute it and/or modify
+ *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
  *  the Free Software Foundation; either version 2 of the License, or
  *  (at your option) any later version.
@@ -8339,4 +8339,30 @@ Roles getSystemRoles(SecurityContext ctx)
         }
         return null;
      }
+
+    /**
+     * Loads the log files linked to the specified objects.
+     *
+     * @param ctx The security context.
+     * @param rootType The type of object to handle.
+     * @param rootIDs The collection of object's identifiers.
+     * @return See above.
+     * @throws DSOutOfServiceException If the connection is broken, or logged in
+     * @throws DSAccessException If an error occurred while trying to 
+     * retrieve data from OMERO service.
+     */
+    Map<Long, List<IObject>> loadLogFiles(SecurityContext ctx,
+            Class<?> rootType, List<Long> rootIDs)
+            throws DSOutOfServiceException, DSAccessException
+    {
+        Connector c = getConnector(ctx, true, false);
+        try {
+            IMetadataPrx service = c.getMetadataService();
+            return service.loadLogFiles(
+                            convertPojos(rootType).getName(), rootIDs);
+        } catch (Throwable t) {
+            handleException(t, "Cannot load log files for " + rootType+".");
+        }
+        return new HashMap();
+    }
 }

components/insight/SRC/org/openmicroscopy/shoola/env/data/OmeroMetadataService.java

@@ -2,10 +2,10 @@
  * org.openmicroscopy.shoola.env.data.OmeroMetadataService 
  *
  *------------------------------------------------------------------------------
- *  Copyright (C) 2006-2013 University of Dundee. All rights reserved.
+ *  Copyright (C) 2006-2014 University of Dundee. All rights reserved.
  *
  *
- * 	This program is free software; you can redistribute it and/or modify
+ *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
  *  the Free Software Foundation; either version 2 of the License, or
  *  (at your option) any later version.
@@ -42,6 +42,7 @@
 import omero.model.FilamentType;
 import omero.model.FilterType;
 import omero.model.Format;
+import omero.model.IObject;
 import omero.model.Illumination;
 import omero.model.Immersion;
 import omero.model.LaserMedium;
@@ -734,4 +735,18 @@ public RequestCallback downloadMetadataFile(SecurityContext ctx, File file,
 		List<String> nsExlcude)
 	throws DSOutOfServiceException, DSAccessException;
 
+	/**
+	 * Loads the log files linked to the specified objects.
+	 *
+	 * @param ctx The security context.
+	 * @param rootType The type of object to handle.
+	 * @param rootIDs The collection of object's identifiers.
+	 * @return See above.
+	 * @throws DSOutOfServiceException If the connection is broken, or logged in
+	 * @throws DSAccessException If an error occurred while trying to 
+	 * retrieve data from OMERO service.
+	 */
+	public Map<Long, List<IObject>> loadLogFiles(SecurityContext ctx,
+	        Class<?> rootType, List<Long> rootIDs)
+	                throws DSOutOfServiceException, DSAccessException;
 }