Implemented metagenome file download

bpaotu/bpaotu/urls.py

@@ -55,4 +55,8 @@
         r'^private/site-image-thumbnail/(?P<package_id>[\w-]+)/(?P<resource_id>[\w-]+)/$',
         views.site_image_thumbnail,
         name="site_image_thumbnail"),
+    url(
+        r'^private/metagenome-search/(?P<sample_id>.+)',
+        views.metagenome_search,
+        name="metagenome_search"),
 ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)

bpaotu/bpaotu/views.py

@@ -22,7 +22,7 @@
     AustralianMicrobiomeSampleContextual
 from django.conf import settings
 from django.core.mail import send_mail
-from django.http import (Http404, HttpResponse, JsonResponse,
+from django.http import (Http404, HttpResponse, JsonResponse, HttpResponseServerError,
                          StreamingHttpResponse)
 from django.template import loader
 from django.urls import reverse
@@ -45,10 +45,10 @@
                     MetadataInfo, OntologyInfo, OTUQueryParams, SampleQuery,
                     TaxonomyFilter, TaxonomyOptions, get_sample_ids,
                     SampleSchemaDefinition, make_cache_key, CACHE_7DAYS)
-from .site_images import fetch_image, get_site_image_lookup_table
+from .site_images import fetch_image, get_site_image_lookup_table, make_ckan_remote
 from .spatial import spatial_query
 from .tabular import tabular_zip_file_generator
-from .util import make_timestamp, parse_date, parse_float
+from .util import make_timestamp, parse_date, parse_float, format_sample_id
 
 logger = logging.getLogger("rainbow")
 
@@ -1046,3 +1046,28 @@ def site_image_thumbnail(request, package_id, resource_id):
 
     buf, content_type = fetch_image(package_id, resource_id)
     return HttpResponse(buf.getvalue(), content_type=content_type)
+
+
+@require_CKAN_auth
+@require_GET
+def metagenome_search(request, sample_id):
+    def urls_by_name(resources):
+        names = [res['name'] for res in resources]
+        n = len(os.path.commonprefix(names))
+        short_names = [s[n:].lstrip('._-') for s in names]
+        return {k: (res['url'], res['size']) for k, res in zip(short_names, resources)}
+    try:
+        remote = make_ckan_remote()
+        r = remote.action.package_search(
+            q='type:amdb-metagenomics-analysed AND sample_id:{}'.format(
+                format_sample_id(
+                    re.sub('[^0-9]', '', sample_id))),
+            rows=50000,
+            include_private=True)
+        results = r['results']
+        n = r['count']
+        return JsonResponse(
+            {package['sample_id'].split('/')[-1]: urls_by_name(package['resources'])
+             for package in results})
+    except Exception as e:
+        return HttpResponseServerError(str(e), content_type="text/plain")

frontend/src/api/index.ts

@@ -1,5 +1,5 @@
 import axios from 'axios'
-import { get as _get, map, partial } from 'lodash'
+import { get as _get, map, partial, join } from 'lodash'
 
 import { store } from '../index'
 import '../interfaces'
@@ -192,3 +192,11 @@ export function getBlastSubmission(submissionId) {
     }
   })
 }
+
+export function executeMetagenomeSearch(sample_id) {
+  const url = join(
+    [window.otu_search_config.base_url,
+    'private/metagenome-search',
+    encodeURIComponent(sample_id)], '/')
+  return axios.get(url)
+}

frontend/src/components/animate_helix.tsx

@@ -1,6 +1,13 @@
 import React, { Component } from "react";
 import styled, { keyframes, css } from "styled-components";
 
+export const loadingstyle= {
+  display: 'flex',
+  height: '100%',
+  justifyContent: 'center',
+  alignItems: 'center'
+};
+
 const size = 2
 const time = 2.1
 const timedelay = -0.89
@@ -315,15 +322,15 @@ const Helix = styled.div`
     vertical-align: middle;
 
     &:not(:last-child){
-      margin-right: ${size * 1.62}vh;  
+      margin-right: ${size * 1.62}vh;
     }
 
     &:before, &:after {
       content: "";
       display: inline-block;
       width: ${size}vh;
       height: ${size}vh;
-      border-radius: 50%; 
+      border-radius: 50%;
       position: absolute;
     }
     ${props => getAnimations(time, timedelay, ease_circ, size, props.color1 ? props.color1 : color1, props.color2 ? props.color2 : color2)}

frontend/src/pages/search_page/components/metagenome_modal.tsx

@@ -1,14 +1,39 @@
 import React from 'react';
 import { connect } from 'react-redux'
 import { bindActionCreators } from 'redux'
-import { Modal, ModalBody, ModalHeader, ModalFooter, Button, Input, Label, FormGroup, Form} from 'reactstrap'
+import { Modal, ModalBody, ModalHeader, ModalFooter, Button, Input, Label, FormGroup, Form, Alert} from 'reactstrap'
 import { isString } from 'lodash'
 import { closeMetagenomeModal } from '../reducers/metagenome_modal'
+import { describeSearch } from '../reducers/search'
 import { metagenome_rows } from './metagenome_rows'
-import { values, filter } from 'lodash'
+import { values, filter, map } from 'lodash'
+import AnimateHelix, { loadingstyle } from '../../../components/animate_helix'
 
-function metagenomeDownloadURL(sampleId, fileType) {
-    return "STUB/" + sampleId + '.' + fileType // FIXME. STUB
+// https://www.codegrepper.com/code-examples/javascript/file+size+calculation+based+on+value+to+show+in+kb%2Fmb+in+javascript
+function formatBytes(bytes, decimals = 2) {
+    if (bytes === 0) return '0 Bytes';
+    const k = 1000;
+    const dm = decimals < 0 ? 0 : decimals;
+    const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
+    const i = Math.floor(Math.log(bytes) / Math.log(k));
+    return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
+}
+
+const metagenomeLink = (url, size) => {
+    const segments = new URL(url).pathname.split('/');
+    const tail = segments.pop() || segments.pop(); // Handle potential trailing slash
+    return <>
+        <td>
+            <a href={url}
+                rel="noopener noreferrer"
+                download
+                target="_blank" >
+                {tail}</a>
+        </td>
+        <td>
+            {formatBytes(size)}
+        </td>
+    </>
 }
 
 class MetagenomeModal extends React.Component<any> {
@@ -41,80 +66,136 @@ class MetagenomeModal extends React.Component<any> {
         event.preventDefault();
         const formData = new FormData(event.target)
         const queryString = new URLSearchParams(formData as any).toString();
-
-        // FIXME stub
-        console.log('form submit', this.state, queryString)
+        const searchParams = JSON.stringify(this.props.describeSearch())
+        // FIXME STUB. Download zip file containing fetcher scripts as per bioplatforms.com.au.
+        console.log("MetagenomeModal.handleSubmit stub", searchParams, queryString)
       }
 
+    get_rows(callback) {
+        return metagenome_rows.map((row, index) => {
+            // Note: key={index} is OK here as metagenome_rows is a constant
+            if (isString(row)) {
+                return <tr className="table-secondary" key={"r" + index}>
+                    <th colSpan={5}>{row}</th>
+                </tr>
+            }
+            return <tr key={"r" + index} >
+                <td>{row[0]}</td>
+                <td>{row[1]}</td>
+                <td>{row[2]}</td>
+                {callback(row, index)
+                }
+            </tr>
+        })
+    }
+
+    get_bulk_rows() {
+        return this.get_rows((row, index) => (
+            <td>
+                <FormGroup check>
+                    <Label>
+                        <Input type='checkbox'
+                            name={row[3].toString()}
+                            onChange={this.handleChecboxChange}
+                            checked={this.state.selected[row[3].toString()] || false} />
+                        {row[3]}
+                    </Label>
+                </FormGroup>
+            </td>
+        ))
+    }
+
+    get_sample_rows() {
+        var mg_data = Object.assign(this.props.metagenome_data[this.props.sample_id] || {})
+        const rows = this.get_rows((row, index) => {
+            const fileType = row[3].toString()
+            try {
+                const [url, size] = mg_data[fileType]
+                delete mg_data[fileType]
+                return metagenomeLink(url, size)
+            } catch (e) {
+                return <><td><i>{fileType} not available</i></td><td></td></>
+            }
+        })
+        // Anything left in mg_data?
+        const remaining = map(mg_data, (row) => (
+            <tr key={"x" + row[0]}>
+                <td></td>
+                <td></td>
+                <td></td>
+                {metagenomeLink(row[0], row[1])}
+            </tr>
+        ))
+        if (remaining.length) {
+            return [...rows,
+            <tr className="table-secondary" key='unknown-mg-data'>
+                <th colSpan={5}>Other metagenome data</th>
+            </tr>,
+            ...remaining]
+        } else {
+            return rows
+        }
+    }
+
     render() {
         const n_selected = filter(values(this.state.selected)).length
+        const bulk = (this.props.sample_id === '*')
+
         return (
             <Modal isOpen={this.props.isOpen} scrollable={true} fade={true}>
                 <ModalHeader toggle={this.props.closeMetagenomeModal}>
                     <div>
-                        <span>{'Metagenome download STUB'}</span>
+                        {
+                            this.props.error ?
+                                <Alert color="danger">Error: {this.props.error}</Alert> :
+                                <span>{'Metagenome download'}</span>
+                        }
                     </div>
                 </ModalHeader>
                 <ModalBody>
-                    <Form id='metagenome-download-form' onSubmit={this.handleSubmit}>
-                        {
-                            (this.props.sample_id && this.props.sample_id !== '*') ? (
-                                <p>
-                                    Metagenome files for {this.props.sample_id}
-                                </p>) : ''
-                        }
-                        <table role="table" className="table table-bordered table-striped">
-                            <thead>
-                                <tr className="table-primary">
-                                    <th>Data object type</th>
-                                    <th>Data object description</th>
-                                    <th>Data object methodology</th>
-                                    <th>Download</th>
-                                </tr>
-                            </thead>
-                            <tbody>
-                                {metagenome_rows.map((column, index) => (
-                                    // Note: index is safe as key here because it's a static array
-                                    (isString(column)) ?
-                                        <tr className="table-secondary" key={index}>
-                                            <th colSpan={4}>{column}</th>
-                                        </tr>
-                                        :
-                                        <tr key={index} >
-                                            <td>{column[0]}</td>
-                                            <td>{column[1]}</td>
-                                            <td>{column[2]}</td>
-                                            <td>{
-                                                (this.props.sample_id === '*') ?
-                                                    <FormGroup check>
-                                                        <Label>
-                                                            <Input type='checkbox'
-                                                                name={column[3].toString()}
-                                                                onChange={this.handleChecboxChange}
-                                                                checked={this.state.selected[column[3].toString()] || false} />
-                                                            {column[3]}
-                                                        </Label>
-                                                    </FormGroup>
-                                                    :
-                                                    <a href={metagenomeDownloadURL(this.props.sample_id, column[3])}
-                                                        rel="noopener noreferrer"
-                                                        target="_blank" >
-                                                        Download {column[3]}</a> // FIXME show Mb?
-                                            }</td>
-                                        </tr>
-                                ))}
-                            </tbody>
-                        </table>
-                    </Form>
+                    {
+                        (this.props.isLoading) ?
+                            <div style={loadingstyle}>
+                                <AnimateHelix />
+                            </div>
+                            : (this.props.isOpen) ?
+                                <Form id='metagenome-download-form' onSubmit={this.handleSubmit}>
+                                    {
+                                        (this.props.sample_id && !bulk && !this.props.error) ? (
+                                            <p>
+                                                Metagenome files for sample {this.props.sample_id}
+                                            </p>) : ''
+                                    }
+                                    <table role="table" className="table table-bordered table-striped">
+                                        <thead>
+                                            <tr className="table-primary">
+                                                <th>Data object type</th>
+                                                <th>Data object description</th>
+                                                <th>Data object methodology</th>
+                                                <th>Download</th>
+                                                {(!bulk) && <th>Size</th>}
+                                            </tr>
+                                        </thead>
+                                        <tbody>
+                                            {
+                                                (bulk) ?
+                                                    this.get_bulk_rows() :
+                                                    this.get_sample_rows()
+                                            }
+                                        </tbody>
+                                    </table>
+                                </Form>
+                                : null
+                    }
                 </ModalBody>
                 <ModalFooter>
-                    {(this.props.sample_id === '*') ?
+                    {(bulk) ?
                         <Button
                         type="submit"
                         form='metagenome-download-form'
                         color="primary"
                         disabled={n_selected < 1}>
-                            Download bundle of {n_selected} metagenome files for each of {this.props.rowsCount} samples
+                            Download fetcher for {n_selected} metagenome files for each of {this.props.rowsCount} samples
                         </Button>
                         :
                         ''}
@@ -125,11 +206,15 @@ class MetagenomeModal extends React.Component<any> {
 }
 
 function mapStateToProps(state) {
-    const { sample_id } = state.searchPage.metagenomeModal
+    const { isOpen, isLoading, sample_id, metagenome_data, error} = state.searchPage.metagenomeModal
     return {
         rowsCount: state.searchPage.results.rowsCount,
-        isOpen: !!sample_id,
+        isOpen,
         sample_id,
+        isLoading,
+        metagenome_data,
+        error,
+        describeSearch: () => describeSearch(state)
     }
 }
 

frontend/src/pages/search_page/components/search_results_card.tsx

@@ -56,8 +56,8 @@ const AlertBoxes = props => (
   </div>
 )
 
-const cell_button = (props, action) => (
-   <Button onClick={() => {action(props)}}>{props.value}</Button>
+const cell_button = (cell_props, openMetagenomeModal) => (
+  <Button onClick={() => {openMetagenomeModal(cell_props.row.sample_id)}}>{cell_props.value}</Button>
 )
 
 class _SearchResultsCard extends React.Component<any, any> {
@@ -213,7 +213,7 @@ function mapMgDispatchToProps(dispatch) {
     {
       clearTips,
       showPhinchTip,
-      openMetagenomeModal,
+      openMetagenomeModal: (sample_id) => (openMetagenomeModal(sample_id)),
       openBulkMetagenomeModal
     },
     dispatch