From 9e9d048f8d2893174cb822c4bc9e5e4954b6c491 Mon Sep 17 00:00:00 2001 From: Ross Blair Date: Thu, 14 Sep 2017 10:28:54 -0700 Subject: [PATCH] Add s3-bucket-listing to assets, add template and url route for bucket listing page. Add link to dataset detail page to s3 listing --- open_fmri/assets/js/s3-list.js | 302 ++++++++++++++++++ .../templates/dataset/dataset_detail.html | 2 + open_fmri/templates/s3_list.html | 30 ++ open_fmri/urls.py | 2 + 4 files changed, 336 insertions(+) create mode 100644 open_fmri/assets/js/s3-list.js create mode 100644 open_fmri/templates/s3_list.html diff --git a/open_fmri/assets/js/s3-list.js b/open_fmri/assets/js/s3-list.js new file mode 100644 index 0000000..6963b27 --- /dev/null +++ b/open_fmri/assets/js/s3-list.js @@ -0,0 +1,302 @@ +/* +## Copyright and License + +Copyright 2012-2016 Rufus Pollock. + +Licensed under the MIT license: + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +https://github.com/rufuspollock/s3-bucket-listing +*/ + +if (typeof AUTO_TITLE != 'undefined' && AUTO_TITLE == true) { + document.title = location.hostname; +} + +if (typeof S3_REGION != 'undefined') { + var BUCKET_URL = 'http://' + location.hostname + '.' + S3_REGION + '.amazonaws.com'; // e.g. just 's3' for us-east-1 region + var BUCKET_WEBSITE_URL = location.protocol + '//' + location.hostname; +} + +if (typeof S3BL_IGNORE_PATH == 'undefined' || S3BL_IGNORE_PATH != true) { + var S3BL_IGNORE_PATH = false; +} + +if (typeof BUCKET_URL == 'undefined') { + var BUCKET_URL = location.protocol + '//' + location.hostname; +} + +if (typeof BUCKET_NAME != 'undefined') { + // if bucket_url does not start with bucket_name, + // assume path-style url + if (!~BUCKET_URL.indexOf(location.protocol + '//' + BUCKET_NAME)) { + BUCKET_URL += '/' + BUCKET_NAME; + } +} + +if (typeof BUCKET_WEBSITE_URL == 'undefined') { + var BUCKET_WEBSITE_URL = BUCKET_URL; +} + +if (typeof S3B_ROOT_DIR == 'undefined') { + var S3B_ROOT_DIR = ''; +} + +if (typeof S3B_SORT == 'undefined') { + var S3B_SORT = 'DEFAULT'; +} + +jQuery(function($) { getS3Data(); }); + +// This will sort your file listing by most recently modified. +// Flip the comparator to '>' if you want oldest files first. +function sortFunction(a, b) { + switch (S3B_SORT) { + case "OLD2NEW": + return a.LastModified > b.LastModified ? 1 : -1; + case "NEW2OLD": + return a.LastModified < b.LastModified ? 1 : -1; + case "A2Z": + return a.Key < b.Key ? 1 : -1; + case "Z2A": + return a.Key > b.Key ? 1 : -1; + case "BIG2SMALL": + return a.Size < b.Size ? 1 : -1; + case "SMALL2BIG": + return a.Size > b.Size ? 1 : -1; + } +} +function getS3Data(marker, html) { + var s3_rest_url = createS3QueryUrl(marker); + // set loading notice + $('#listing') + .html('Loading...'); + $.get(s3_rest_url) + .done(function(data) { + // clear loading notice + $('#listing').html(''); + var xml = $(data); + var info = getInfoFromS3Data(xml); + + // Slight modification by FuzzBall03 + // This will sort your file listing based on var S3B_SORT + // See url for example: + // http://esp-link.s3-website-us-east-1.amazonaws.com/ + if (S3B_SORT != 'DEFAULT') { + var sortedFiles = info.files; + sortedFiles.sort(sortFunction); + info.files = sortedFiles; + } + + buildNavigation(info); + + html = typeof html !== 'undefined' ? html + prepareTable(info) : + prepareTable(info); + if (info.nextMarker != "null") { + getS3Data(info.nextMarker, html); + } else { + document.getElementById('listing').innerHTML = + '
' + html + '
'; + } + }) + .fail(function(error, textStatus, errorThrown) { + console.log(textStatus); + console.log(errorThrown); + console.error(error); + $('#listing').html('Error: ' + error + ''); + }); +} + +function buildNavigation(info) { + var root = '' + BUCKET_WEBSITE_URL + ' / '; + if (info.prefix) { + var processedPathSegments = ''; + var content = $.map(info.prefix.split('/'), function(pathSegment) { + processedPathSegments = + processedPathSegments + encodeURIComponent(pathSegment) + '/'; + return '' + pathSegment + + ''; + }); + $('#navigation').html(root + content.join(' / ')); + } else { + $('#navigation').html(root); + } +} + +function createS3QueryUrl(marker) { + var s3_rest_url = BUCKET_URL; + s3_rest_url += '?delimiter=/'; + + // + // Handling paths and prefixes: + // + // 1. S3BL_IGNORE_PATH = false + // Uses the pathname + // {bucket}/{path} => prefix = {path} + // + // 2. S3BL_IGNORE_PATH = true + // Uses ?prefix={prefix} + // + // Why both? Because we want classic directory style listing in normal + // buckets but also allow deploying to non-buckets + // + + var rx = '.*[?&]prefix=' + S3B_ROOT_DIR + '([^&]+)(&.*)?$'; + var prefix = ''; + if (S3BL_IGNORE_PATH == false) { + var prefix = location.pathname.replace(/^\//, S3B_ROOT_DIR); + } + var match = location.search.match(rx); + if (match) { + prefix = S3B_ROOT_DIR + match[1]; + } else { + if (S3BL_IGNORE_PATH) { + var prefix = S3B_ROOT_DIR; + } + } + if (prefix) { + // make sure we end in / + var prefix = prefix.replace(/\/$/, '') + '/'; + s3_rest_url += '&prefix=' + prefix; + } + if (marker) { + s3_rest_url += '&marker=' + marker; + } + return s3_rest_url; +} + +function getInfoFromS3Data(xml) { + var files = $.map(xml.find('Contents'), function(item) { + item = $(item); + // clang-format off + return { + Key: item.find('Key').text(), + LastModified: item.find('LastModified').text(), + Size: bytesToHumanReadable(item.find('Size').text()), + Type: 'file' + } + // clang-format on + }); + var directories = $.map(xml.find('CommonPrefixes'), function(item) { + item = $(item); + // clang-format off + return { + Key: item.find('Prefix').text(), + LastModified: '', + Size: '0', + Type: 'directory' + } + // clang-format on + }); + if ($(xml.find('IsTruncated')[0]).text() == 'true') { + var nextMarker = $(xml.find('NextMarker')[0]).text(); + } else { + var nextMarker = null; + } + // clang-format off + return { + files: files, + directories: directories, + prefix: $(xml.find('Prefix')[0]).text(), + nextMarker: encodeURIComponent(nextMarker) + } + // clang-format on +} + +// info is object like: +// { +// files: .. +// directories: .. +// prefix: ... +// } +function prepareTable(info) { + var files = info.directories.concat(info.files), prefix = info.prefix; + var cols = [45, 30, 15]; + var content = []; + content.push(padRight('Last Modified', cols[1]) + ' ' + + padRight('Size', cols[2]) + 'Key \n'); + content.push(new Array(cols[0] + cols[1] + cols[2] + 4).join('-') + '\n'); + + // add ../ at the start of the dir listing, unless we are already at root dir + if (prefix && prefix !== S3B_ROOT_DIR) { + var up = prefix.replace(/\/$/, '').split('/').slice(0, -1).concat('').join( + '/'), // one directory up + item = + { + Key: up, + LastModified: '', + Size: '', + keyText: '../', + href: S3BL_IGNORE_PATH ? '?prefix=' + up : '../' + }, + row = renderRow(item, cols); + content.push(row + '\n'); + } + + jQuery.each(files, function(idx, item) { + // strip off the prefix + item.keyText = item.Key.substring(prefix.length); + if (item.Type === 'directory') { + if (S3BL_IGNORE_PATH) { + item.href = location.protocol + '//' + location.hostname + + location.pathname + '?prefix=' + item.Key; + } else { + item.href = item.keyText; + } + } else { + item.href = BUCKET_WEBSITE_URL + '/' + encodeURIComponent(item.Key); + item.href = item.href.replace(/%2F/g, '/'); + } + var row = renderRow(item, cols); + if (typeof EXCLUDE_FILE == 'undefined' || EXCLUDE_FILE != item.Key) + content.push(row + '\n'); + }); + + return content.join(''); +} + +function renderRow(item, cols) { + var row = ''; + row += padRight(item.LastModified, cols[1]) + ' '; + row += padRight(item.Size, cols[2]); + row += '' + item.keyText + ''; + return row; +} + +function padRight(padString, length) { + var str = padString.slice(0, length - 3); + if (padString.length > str.length) { + str += '...'; + } + while (str.length < length) { + str = str + ' '; + } + return str; +} + +function bytesToHumanReadable(sizeInBytes) { + var i = -1; + var units = [' kB', ' MB', ' GB']; + do { + sizeInBytes = sizeInBytes / 1024; + i++; + } while (sizeInBytes > 1024); + return Math.max(sizeInBytes, 0.1).toFixed(1) + units[i]; +} diff --git a/open_fmri/templates/dataset/dataset_detail.html b/open_fmri/templates/dataset/dataset_detail.html index 6390251..df567e3 100644 --- a/open_fmri/templates/dataset/dataset_detail.html +++ b/open_fmri/templates/dataset/dataset_detail.html @@ -135,6 +135,8 @@

Papers that Reference this Dataset:

{% endfor %} {% endif %} +

Browse Data on S3

+ {% else %}

The dataset {{ object.project_name }} with accession number {{ object.accession_number }} has been diff --git a/open_fmri/templates/s3_list.html b/open_fmri/templates/s3_list.html new file mode 100644 index 0000000..0d0c6fd --- /dev/null +++ b/open_fmri/templates/s3_list.html @@ -0,0 +1,30 @@ +{% extends "base.html" %} +{% load staticfiles %} +{% block title %} + Datasets +{% endblock %} + +{% block css %} +{{ block.super }} +{% endblock %} + +{% block header-image %} +{% static 'images/banner_view_data.png' %} +{% endblock %} + +{% block content %} +

Data on S3:

+
+{% endblock %} + +{% block javascript %} +{{ block.super }} + + + + + +{% endblock %} diff --git a/open_fmri/urls.py b/open_fmri/urls.py index d5a8900..d7959e7 100644 --- a/open_fmri/urls.py +++ b/open_fmri/urls.py @@ -15,6 +15,8 @@ url(r'^dataset/', include(dataset.urls)), url(r'^contact/', TemplateView.as_view(template_name='contact_form.html'), name='contact_form'), + url(r'^s3-browser/$', TemplateView.as_view(template_name='s3_list.html'), + name='s3_list'), url(r'^$', Index.as_view(), name='index'), url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework')), url(r'^subscribed/', TemplateView.as_view(template_name='subscribe_success.html')),