Add metadata file detection and download links to ee UI

gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataFileService.java

@@ -36,8 +36,8 @@
 public interface ExpressionDataFileService {
 
     String DATA_ARCHIVE_FILE_SUFFIX = ".zip";
-    String DATA_DIR =
-            Settings.getString( "gemma.appdata.home" ) + File.separatorChar + "dataFiles" + File.separatorChar;
+    String DATA_DIR = Settings.getString( "gemma.appdata.home" ) + File.separatorChar + "dataFiles" + File.separatorChar;
+    String METADATA_DIR = Settings.getString( "gemma.appdata.home" ) + File.separatorChar + "metadata" + File.separatorChar;
 
     String DATA_FILE_SUFFIX = ".data.txt";
 

gemma-web/src/main/java/ubic/gemma/web/controller/expression/experiment/ExpressionExperimentDataFetchController.java

@@ -38,15 +38,14 @@
 import ubic.gemma.model.expression.experiment.ExpressionExperiment;
 import ubic.gemma.persistence.service.common.quantitationtype.QuantitationTypeService;
 import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService;
-import ubic.gemma.persistence.util.Settings;
 
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
-import java.util.Collection;
-import java.util.HashSet;
+import java.text.ParseException;
+import java.util.*;
 
 /**
  * For the download of data files from the browser. We can send the 'raw' data for any one quantitation type, with gene
@@ -58,8 +57,13 @@
 @Controller
 public class ExpressionExperimentDataFetchController {
 
-    public static final String DATA_DIR =
-            Settings.getString( "gemma.appdata.home" ) + File.separatorChar + "dataFiles" + File.separatorChar;
+    private static final String[] META_FILES = { //
+            "%s.alignment.metadata",                                    // Alignment metadata
+            "%s.base.metadata",                                         // Base metadata
+            "MultiQCReports" + File.separatorChar + "multiqc_report.html",  // Multi-QC report
+            "configurations" + File.separatorChar                        // Configuration
+    };
+
     @Autowired
     private TaskRunningService taskRunningService;
     @Autowired
@@ -78,33 +82,86 @@ public class ExpressionExperimentDataFetchController {
      */
     @RequestMapping(value = "/getData.html", method = RequestMethod.GET, produces = MediaType.APPLICATION_OCTET_STREAM_VALUE)
     public void downloadFile( HttpServletRequest request, HttpServletResponse response ) throws IOException {
+        String dir = ExpressionDataFileService.DATA_DIR;
+        this.downloadFromDir( request, response, dir );
+    }
 
-        String file = request.getParameter( "file" );
+    @RequestMapping(value = "/getMetaData.html", method = RequestMethod.GET, produces = MediaType.APPLICATION_OCTET_STREAM_VALUE)
+    public void downloadMetaFile( HttpServletRequest request, HttpServletResponse response ) throws IOException {
+        IllegalArgumentException e = new IllegalArgumentException( "The experiment ID parameter is invalid." );
+        ExpressionExperiment ee;
 
-        if ( StringUtils.isBlank( file ) ) {
-            throw new IllegalArgumentException( "The file name cannot be blank" );
+        try {
+
+            String eeId = request.getParameter( "id" );
+            if ( StringUtils.isBlank( eeId ) ) {
+                throw e;
+            }
+
+            ee = expressionExperimentService.load( Long.parseLong( eeId ) );
+        } catch( NumberFormatException ne ){
+            throw e;
         }
 
-        String fullFilePath = ExpressionDataFileService.DATA_DIR + file;
-        File f = new File( fullFilePath );
+        String dir = ExpressionDataFileService.METADATA_DIR + ee.getShortName() + File.separatorChar;
+        this.downloadFromDir( request, response, dir );
+    }
 
-        if ( !f.canRead() ) {
-            throw new IOException( "Cannot read from " + fullFilePath );
+    /**
+     * Scans the metadata directory for any files associated with the given experiment.
+     *
+     * @param eeId the id of the experiment to scan for the metadata for.
+     * @return an array of file names available in the metadata directory for the given experiment.
+     */
+    public String[] getMetadataFileUris( Long eeId ) {
+
+        ExpressionExperiment ee = expressionExperimentService.load( eeId );
+        if ( ee == null ) {
+            throw new IllegalArgumentException( "Experiment with given ID does not exist." );
         }
 
-        // response.setContentType( "application/octet-stream" ); // see Bug4206
-        response.setContentLength( ( int ) f.length() );
-        response.addHeader( "Content-disposition", "attachment; filename=\"" + f.getName() + "\"" );
-        FileInputStream in = new FileInputStream( f );
-        FileCopyUtils.copy( in, response.getOutputStream() );
-        response.flushBuffer();
+        String[] uris = new String[ExpressionExperimentDataFetchController.META_FILES.length];
 
-        try {
-            in.close();
-        } catch ( IOException e ) {
+        int i = 0;
+        for ( String path : ExpressionExperimentDataFetchController.META_FILES ) {
+
+            // Some files are prefixed with the experiments accession
+            String fPath = path.contains( "%" ) ? String.format( path, ee.getShortName() ) : path;
+            File file = new File(
+                    ExpressionDataFileService.METADATA_DIR + ee.getShortName() + File.separatorChar + fPath );
+
+            if ( !file.exists() || !file.canRead() ) {
+                continue;
+            }
+
+            String uri = null;
+
+            if ( file.isDirectory() ) {
+                // We expect this for the Configurations directory
+                File[] files = file.listFiles();
+                if ( files != null && files.length > 0 ) {
+                    List<File> fList = Arrays.asList( files );
+
+                    // Sort by last modified, we only want the newest file
+                    Collections.sort( fList, new Comparator<File>() {
+                        @Override
+                        public int compare( File file, File t1 ) {
+                            return Long.compare( file.lastModified(), t1.lastModified() );
+                        }
+                    } );
 
+                    uri = fPath + fList.get( 0 ).getName();
+                }
+            } else {
+                uri = fPath;
+            }
+
+            if ( uri != null ) {
+                uris[i++] = uri;
+            }
         }
 
+        return uris;
     }
 
     /**
@@ -139,23 +196,52 @@ public String getDiffExpressionDataFile( Long analysisId ) {
     }
 
     public File getOutputFile( String filename ) {
-        String fullFilePath = ExpressionExperimentDataFetchController.DATA_DIR + filename;
+        String fullFilePath = ExpressionDataFileService.DATA_DIR + filename;
         File f = new File( fullFilePath );
         File parentDir = f.getParentFile();
-        if ( !parentDir.exists() )
+        if ( !parentDir.exists() ) {
+            //noinspection ResultOfMethodCallIgnored
             parentDir.mkdirs();
+        }
         return f;
     }
 
     public void setQuantitationTypeService( QuantitationTypeService quantitationTypeService ) {
         this.quantitationTypeService = quantitationTypeService;
     }
 
+    private void downloadFromDir( HttpServletRequest request, HttpServletResponse response, String dir )
+            throws IOException {
+        String file = request.getParameter( "file" );
+
+        if ( StringUtils.isBlank( file ) ) {
+            throw new IllegalArgumentException( "The file name cannot be blank" );
+        }
+
+        File f = new File( dir + file );
+
+        if ( !f.canRead() ) {
+            throw new IOException( "Cannot read from " + dir + file );
+        }
+
+        response.setContentLength( ( int ) f.length() );
+        response.addHeader( "Content-disposition", "attachment; filename=\"" + f.getName() + "\"" );
+        FileInputStream in = new FileInputStream( f );
+        FileCopyUtils.copy( in, response.getOutputStream() );
+        response.flushBuffer();
+
+        try {
+            in.close();
+        } catch ( IOException ignored ) {
+
+        }
+    }
+
     class CoExpressionDataWriterJob extends AbstractTask<TaskResult, ExpressionExperimentDataFetchCommand> {
 
         protected Log log = LogFactory.getLog( this.getClass().getName() );
 
-        public CoExpressionDataWriterJob( ExpressionExperimentDataFetchCommand eeId ) {
+        CoExpressionDataWriterJob( ExpressionExperimentDataFetchCommand eeId ) {
             super( eeId );
         }
 
@@ -191,7 +277,7 @@ class DataWriterJob extends AbstractTask<TaskResult, ExpressionExperimentDataFet
 
         protected Log log = LogFactory.getLog( this.getClass().getName() );
 
-        public DataWriterJob( ExpressionExperimentDataFetchCommand command ) {
+        DataWriterJob( ExpressionExperimentDataFetchCommand command ) {
             super( command );
         }
 
@@ -222,7 +308,7 @@ public TaskResult execute() {
 
             /* if qt is not set, send the data for the filtered matrix */
             QuantitationType qType = null;
-            ExpressionExperiment ee = null;
+            ExpressionExperiment ee;
 
             /* design file */
             if ( eedId != null ) {
@@ -308,7 +394,7 @@ class DiffExpressionDataWriterTask extends AbstractTask<TaskResult, ExpressionEx
 
         protected Log log = LogFactory.getLog( this.getClass().getName() );
 
-        public DiffExpressionDataWriterTask( ExpressionExperimentDataFetchCommand command ) {
+        DiffExpressionDataWriterTask( ExpressionExperimentDataFetchCommand command ) {
             super( command );
         }
 

gemma-web/src/main/webapp/WEB-INF/gemma-servlet.xml

@@ -53,6 +53,7 @@
                 <prop key="/expressionExperiment/editExpressionExperiment.html">expressionExperimentFormController
                 </prop>
                 <prop key="/getData.html">expressionExperimentDataFetchController</prop>
+                <prop key="/getMetaData.html">expressionExperimentDataFetchController</prop>
                 <prop key="/checkJobProgress.html">taskCompletionController</prop>
                 <prop key="/searchCoexpression.html">coexpressionSearchController</prop>
                 <prop key="/gene/*">geneController</prop>
@@ -458,6 +459,7 @@
           class="ubic.gemma.web.controller.expression.experiment.ExpressionExperimentDataFetchController">
         <dwr:remote javascript="ExpressionExperimentDataFetchController">
             <dwr:include method="getDataFile"/>
+            <dwr:include method="getMetadataFileUris"/>
             <dwr:include method="getCoExpressionDataFile"/>
             <dwr:include method="getDiffExpressionDataFile"/>
         </dwr:remote>

gemma-web/src/main/webapp/scripts/api/dwrServices.js

@@ -625,6 +625,10 @@ ExpressionExperimentDataFetchController.getDataFile = function (p0, callback) {
     dwr.engine._execute(ExpressionExperimentDataFetchController._path, 'ExpressionExperimentDataFetchController',
         'getDataFile', p0, callback);
 };
+ExpressionExperimentDataFetchController.getMetadataFileUris = function (p0, callback) {
+    dwr.engine._execute(ExpressionExperimentDataFetchController._path, 'ExpressionExperimentDataFetchController',
+        'getMetadataFileUris', p0, callback);
+};
 ExpressionExperimentDataFetchController.getDiffExpressionDataFile = function (p0, callback) {
     googleAnalyticsTrackPageviewIfConfigured("/Gemma/ExpressionExperimentDataFetchController/getDiffExpressionDataFile?"
         + p0);

gemma-web/src/main/webapp/scripts/api/entities/experiment/ExpressionExperimentDetails.js

@@ -32,7 +32,8 @@ Gemma.ExpressionExperimentDetails = Ext
             /**
              * @memberOf Gemma.ExpressionExperimentDetails
              */
-            renderArrayDesigns: function (arrayDesigns) {
+            renderArrayDesigns: function (ee) {
+                var arrayDesigns = ee.arrayDesigns;
                 var result = '';
                 for (var i = 0; i < arrayDesigns.length; i++) {
                     var ad = arrayDesigns[i];
@@ -47,6 +48,12 @@ Gemma.ExpressionExperimentDetails = Ext
                         result = result + "<br/>";
                     }
                 }
+
+                if (ee.lastArrayDesignUpdateDate) {
+                    result += "<div class='dark-gray v-padded'>The last time a platform associated with this experiment was updated: " +
+                        +Gemma.Renderers.dateTimeRenderer(ee.lastArrayDesignUpdateDate) + "</div>"
+                }
+
                 return result;
             },
             renderCoExpressionLinkCount: function (ee) {
@@ -65,9 +72,6 @@ Gemma.ExpressionExperimentDetails = Ext
 
             },
 
-            /**
-             *
-             */
             renderSourceDatabaseEntry: function (ee) {
                 var result = '';
 
@@ -91,6 +95,37 @@ Gemma.ExpressionExperimentDetails = Ext
 
             },
 
+            renderMetadata: function (ee) {
+
+                ExpressionExperimentDataFetchController.getMetadataFileUris(ee.id, {
+                    callback: function (files) {
+                        var result = "";
+                        var hasFiles = false;
+
+                        files.forEach(function (file) {
+                            if (file != null) {
+                                hasFiles = true;
+                                result +=
+                                    "<div class='v-padded'>" +
+                                    "   <a target='_blank' href='" + ctxBasePath + "/getMetaData.html?id="
+                                    + ee.id + "&file=" + file + "' ext:qtip='Download file " + file
+                                    + "'><i class='gray-blue fa fa-download'></i> " + file + "</a>" +
+                                    "</div>";
+                            }
+                        });
+
+
+                        if (!hasFiles) {
+                            result = "<span class='dark-gray'> Not available for this experiment </span>";
+                        }
+
+                        Ext.getCmp("metadata-row").update(result);
+                    }
+                });
+
+                return "";
+            },
+
             /**
              * Link for samples details page.
              *
@@ -123,7 +158,7 @@ Gemma.ExpressionExperimentDetails = Ext
                 if (ee.needsAttention === true) {
                     result = result + getStatusBadge('exclamation-circle', 'gold', 'in curation', 'The curation of this experiment is not done yet, so the quality and suitability scores may change significantly.')
                 }
-                if (ee.geeq !== null ){
+                if (ee.geeq !== null) {
                     result = result + getGeeqBadges(ee.geeq.publicQualityScore, ee.geeq.publicSuitabilityScore);
                 }
 
@@ -702,16 +737,16 @@ Gemma.ExpressionExperimentDetails = Ext
                                         fieldLabel: 'Profiles',
                                         // id: 'processedExpressionVectorCount-region',
                                         html: '<div id="downloads"> '
-                                        + this.renderProcessedExpressionVectorCount(e)
-                                        + '&nbsp;&nbsp;'
-                                        + '<i>Downloads:</i> &nbsp;&nbsp; <span class="link"  ext:qtip="Download the tab delimited data" onClick="fetchData(true,'
-                                        + e.id
-                                        + ', \'text\', null, null)">Filtered</span> &nbsp;&nbsp;'
-                                        + '<span class="link" ext:qtip="Download the tab delimited data" onClick="fetchData(false,'
-                                        + e.id
-                                        + ', \'text\', null, null)">Unfiltered</span> &nbsp;&nbsp;'
-                                        + '<i class="qtp fa fa-question-circle fa-fw"></i>'
-                                        + '</div>',
+                                            + this.renderProcessedExpressionVectorCount(e)
+                                            + '&nbsp;&nbsp;'
+                                            + '<i>Downloads:</i> &nbsp;&nbsp; <span class="link"  ext:qtip="Download the tab delimited data" onClick="fetchData(true,'
+                                            + e.id
+                                            + ', \'text\', null, null)">Filtered</span> &nbsp;&nbsp;'
+                                            + '<span class="link" ext:qtip="Download the tab delimited data" onClick="fetchData(false,'
+                                            + e.id
+                                            + ', \'text\', null, null)">Unfiltered</span> &nbsp;&nbsp;'
+                                            + '<i class="qtp fa fa-question-circle fa-fw"></i>'
+                                            + '</div>',
                                         width: 400,
                                         listeners: {
                                             'afterrender': function (c) {
@@ -727,8 +762,8 @@ Gemma.ExpressionExperimentDetails = Ext
                                         }
                                     }, {
                                         fieldLabel: 'Platforms',
-                                        html: this.renderArrayDesigns(e.arrayDesigns),
-                                        width: 480
+                                        html: this.renderArrayDesigns(e),
+                                        width: 600
                                     }, {
                                         fieldLabel: 'Coexpr. Links',
                                         html: this.renderCoExpressionLinkCount(e),
@@ -765,10 +800,11 @@ Gemma.ExpressionExperimentDetails = Ext
                                         html: this.renderSourceDatabaseEntry(e)
                                     },
                                     {
-                                        html: 'The last time a platform associated with this experiment was updated: '
-                                        + Gemma.Renderers.dateTimeRenderer(e.lastArrayDesignUpdateDate),
-                                        hidden: !e.lastArrayDesignUpdateDate
-                                    }]
+                                        fieldLabel: 'Metadata',
+                                        id: "metadata-row",
+                                        html: this.renderMetadata(e)
+                                    }
+                                ]
                             }]
                     });