diff --git a/controllers/direct_upload.js b/controllers/direct_upload.js index 108006f..307b038 100644 --- a/controllers/direct_upload.js +++ b/controllers/direct_upload.js @@ -38,7 +38,7 @@ exports.create = (req, res) => { let uploader = new Uploader(req, res); uploader.upload((id, err) => { if (err) { - debug('Error during upload: %s', JSON.stringify(err)); + debug('Error during upload: %o', err); } else { debug('New compendium %s successfully uploaded', id); diff --git a/controllers/dispatch.js b/controllers/dispatch.js index 88be405..5649a42 100644 --- a/controllers/dispatch.js +++ b/controllers/dispatch.js @@ -45,10 +45,10 @@ exports.dispatch = (req, res) => { // a) direct upload with a file attachment // b) no file attachment = get from share via URL parameter if(req.file) { - debug('Detected file in request, dispatching to direct upload: %s', JSON.stringify(req.file)); + debug('Detected file in request, dispatching to direct upload: %o', req.file); createFromDirectUpload(req, res); } else { - debug('Detected _no_ file in request, dispatching to share upload: %s', JSON.stringify(req.body)); + debug('Detected _no_ file in request, dispatching to share upload: %O', req.body); createFromShare(req, res); } }; diff --git a/controllers/share.js b/controllers/share.js index 79af9dc..f357aea 100644 --- a/controllers/share.js +++ b/controllers/share.js @@ -19,6 +19,7 @@ const config = require('../config/config'); const debug = require('debug')('loader:ctrl:share'); const fs = require('fs'); +const path = require('path'); const Compendium = require('../lib/model/compendium'); const Loader = require('../lib/loader').Loader; @@ -110,6 +111,14 @@ exports.create = (req, res) => { function prepareScieboLoad(req, res) { if (!req.body.path) { // set default value for path ('/') req.body.path = '/'; + } else if (!req.body.path.startsWith('/')) { // add '/' to beginning of path + req.body.path = '/' + req.body.path; + } + + //handle directly submitted zip file + if (req.body.path.endsWith('.zip')){ + req.zipFile = path.basename(req.body.path); + req.body.path = path.dirname(req.body.path); } // only allow sciebo shares, see https://www.sciebo.de/de/login/index.html @@ -127,9 +136,9 @@ function prepareScieboLoad(req, res) { var loader = new Loader(req, res); loader.loadOwncloud((data, err) => { if (err) { - debug('Error during public share load from owncloud: %s', JSON.stringify(err)); + debug('Error during public share load from owncloud: %o', err); } else { - debug('New compendium successfully loaded: %s', JSON.stringify(data)); + debug('New compendium successfully loaded: %o', data); if (config.slack.enable) { let compendium_url = req.protocol + '://' + req.get('host') + '/api/v1/compendium/' + data.id; @@ -164,9 +173,9 @@ function prepareZenodoLoad(req, res) { var loader = new Loader(req, res); loader.loadZenodo((data, err) => { if (err) { - debug('Error during public share load from Zenodo: %s', JSON.stringify(err)); + debug('Error during public share load from Zenodo: %o', err); } else { - debug('New compendium successfully loaded: %s', JSON.stringify(data)); + debug('New compendium successfully loaded: %o', data); if (config.slack.enable) { let compendium_url = req.protocol + '://' + req.get('host') + '/api/v1/compendium/' + data.id; diff --git a/index.js b/index.js index f0d88fd..686aa99 100644 --- a/index.js +++ b/index.js @@ -170,7 +170,7 @@ function initApp(callback) { debug('Error starting slackbot (disabling it now): %s', err); config.slack.enable = false; }, (done) => { - debug('Slack bot enabled and configured - nice! %s', JSON.stringify(done)); + debug('Slack bot enabled and configured - nice! %o', done); }); } @@ -192,9 +192,9 @@ function initApp(callback) { } else { function onFinished(err, output) { if(err) { - debug('error pulling meta image: %s', JSON.stringify(err)); + debug('error pulling meta image: %o', err); } else { - debug('pulled meta tools image: %s', JSON.stringify(output)); + debug('pulled meta tools image: %O', output); } delete docker; } diff --git a/lib/bagit.js b/lib/bagit.js index e74471e..b007816 100644 --- a/lib/bagit.js +++ b/lib/bagit.js @@ -45,7 +45,7 @@ function validateBag(passon) { bag .validate(config.bagit.validation.fast) .then((res) => { - debug('[%s] bag is valid: %s', passon.id, JSON.stringify(res)); + debug('[%s] bag is valid: %o', passon.id, res); passon.bagValid = true; fulfill(passon); }).catch((err) => { diff --git a/lib/loader.js b/lib/loader.js index 0821e7a..df434cd 100644 --- a/lib/loader.js +++ b/lib/loader.js @@ -40,6 +40,7 @@ function Loader(req, res) { content: req.body.content_type, shareURL: encodeURI(this.req.body.share_url), webdav_path: this.req.body.path, + zipFile : this.req.zipFile, user: this.req.user.orcid, req: this.req, res: this.res @@ -63,7 +64,7 @@ function Loader(req, res) { done({ id: passon.id, share_url: passon.shareURL }, null); }) .catch(err => { - debug('Rejection or unhandled failure during load from public share: %s\n%s', JSON.stringify(err), err); + debug('Rejection or unhandled failure during load from public share: %o\n%s', err, err); let status = 500; if (err.status) { status = err.status; @@ -108,7 +109,7 @@ function Loader(req, res) { done({ id: passon.id, share_url: passon.zenodoURL }, null); }) .catch(err => { - debug('Rejection or unhandled failure during load from zenodo: %s\n%s', JSON.stringify(err), err); + debug('Rejection or unhandled failure during load from zenodo: %o\n%s', err, err); let status = 500; if (err.status) { status = err.status; diff --git a/lib/meta.js b/lib/meta.js index 0618ff6..bcd6d0f 100644 --- a/lib/meta.js +++ b/lib/meta.js @@ -35,7 +35,7 @@ module.exports.broker = function (id, metadata_dir, metadata_file, current_mappi metadata_dir + ':' + metadata_dir ]; if(config.fs.volume) { - debug('[%s] volume is configured, overwriting binds configuration (was %s)', id, JSON.stringify(binds)); + debug('[%s] volume is configured, overwriting binds configuration (was %o)', id, binds); // metadata_dir always starts with config.fs.base, mounting more than needed but limiting scope with cmd binds = [ config.fs.volume + ':' + config.fs.base @@ -71,13 +71,13 @@ module.exports.broker = function (id, metadata_dir, metadata_file, current_mappi }; docker.run(config.meta.container.image, cmd, containerLogStream, create_options, start_options, (err, data, container) => { - debug('[%s] container running: %s', id, JSON.stringify(container)); + debug('[%s] container running: %o', id, container); if (err) { debug('[%s] error during metadata brokering:', err); reject(err); } else { if (data.StatusCode === 0) { - debug('[%s] Completed metadata brokering for %s: %s', id, current_mapping, JSON.stringify(data)); + debug('[%s] Completed metadata brokering for %s: %O', id, current_mapping, data); // check if metadata was found, if so return it try { @@ -113,7 +113,7 @@ module.exports.broker = function (id, metadata_dir, metadata_file, current_mappi reject(err); } } else { - debug('[%s] Error during meta container run brokering %s: %s', id, current_mapping, JSON.stringify(data)); + debug('[%s] Error during meta container run brokering %s: %o', id, current_mapping, data); container.logs({ follow: true, stdout: true, diff --git a/lib/slack.js b/lib/slack.js index cde2c34..507e8a5 100644 --- a/lib/slack.js +++ b/lib/slack.js @@ -32,23 +32,23 @@ exports.start = function (err, done) { bot2r = slack.rtm.client(); bot2r.started(function (payload) { - debug('Started... payload from rtm.start: %s', JSON.stringify(payload)); + debug('Started... payload from rtm.start: %o', payload); }); bot2r.channel_joined(function (payload) { - debug('payload from channel joined', JSON.stringify(payload)); + debug('payload from channel joined: %o', payload); }); bot2r.message(function (payload) { - debug('[message] Incoming message: %s', JSON.stringify(payload)); + debug('[message] Incoming message: %o', payload); }); bot2r.goodbye(function (payload) { - debug('[goodbye] Server wants to close the connection soon... %s', JSON.stringify(payload)); + debug('[goodbye] Server wants to close the connection soon... %o', payload); }); bot2r.hello(function (payload) { - debug('[hello] connected to server: %s', JSON.stringify(payload)); + debug('[hello] connected to server: %s', payload); joinChannelAndSayHello(err, done); }); @@ -69,7 +69,7 @@ joinChannelAndSayHello = function (err, done) { err(e); } else { - debug('Response on posting startup message to %s: %s', config.slack.channel.status, JSON.stringify(data)); + debug('Response on posting startup message to %s: %o', config.slack.channel.status, data); done(data); } }); @@ -91,7 +91,7 @@ exports.newDirectUpload = function (compendium_url, orcid) { if (data.ok) { debug('Message send was OK'); } else { - debug('Message send NOT OK. Response for posting new direct upload message: %s', JSON.stringify(data)); + debug('Message send NOT OK. Response for posting new direct upload message: %o', data); } } }); @@ -113,7 +113,7 @@ exports.newShareUpload = function (compendium_url, orcid, share_url) { if (data.ok) { debug('Message send was OK'); } else { - debug('Message send NOT OK. Response for posting new share upload message: %s', JSON.stringify(data)); + debug('Message send NOT OK. Response for posting new share upload message: %o', data); } } }); diff --git a/lib/steps.js b/lib/steps.js index c503a9e..63aa109 100644 --- a/lib/steps.js +++ b/lib/steps.js @@ -62,7 +62,6 @@ function publicShareLoad(passon) { .then(checkContents) .then(checkAndLoadZip) .then(checkAndCopyCompendium) - .then(checkAndLoadDirectory) .then(checkAndLoadFiles) .catch(function (err) { debug('[%s] Error loading public share (rethrow): %s', passon.id, err); @@ -258,7 +257,13 @@ function analyzeContents(passon) { result.bagitCount++; } else if (contents[i].mime === 'application/zip') { result.zipCount++; - result.zipName = contents[i].basename; + if (passon.zipFile) { + //use the file provided in the path parameter (support for zip file names in path parameter) + result.zipName = passon.zipFile; + } else { + //use the file found in the webdav directory + result.zipName = contents[i].basename; + } if (result.zipName === 'webdav') { // return an error message if a zip file is submitted directly debug('Direct file submission is not supported.'); let err = new Error('Direct file submission is not supported. Please submit a shared folder containing the file.'); @@ -335,8 +340,18 @@ function checkAndCopyCompendium(passon) { function checkAndLoadZip(passon) { //Check if a single zip file exists -> load and unzip it return new Promise((fulfill, reject) => { - if (passon.result.zipCount === 1 && passon.result.bagitCount === 0) { - debug('[%s] Single zip file found: %s', passon.id); + if (passon.result.zipCount >= 1 && passon.result.bagitCount === 0) { + if (passon.result.zipCount === 1) { + debug('[%s] Single zip file found: %s', passon.id); + } else if (passon.zipFile){ + debug('[%s] Multiple zip files found: %s. Filename %s was provided.', passon.id, passon.zipFile); + } else { + debug('[%s] Multiple zip files found but no filename provided, aborting.', passon.id); + let error = new Error(); + error.msg = 'Multiple zip files found but no filename provided'; + error.status = 404; + reject(error); + } return passon.client .getFileContents(path.join(passon.webdav_path, passon.result.zipName)) .then(function (zipData) { @@ -363,23 +378,9 @@ function checkAndLoadZip(passon) { }); } -function checkAndLoadDirectory(passon) { - //Check if a single directory exists -> throw error - return new Promise((fulfill, reject) => { - if (passon.result.directoryCount === 1 && passon.result.bagitCount === 0 && passon.result.zipCount !== 1) { - let error = new Error(); - error.msg = 'Single directory found. Use the "path" parameter to point to the compendium directory.'; - error.status = 404; - reject(error); - } else { - fulfill(passon); - } - }); -} - function checkAndLoadFiles(passon) { return new Promise((fulfill, reject) => { - if (passon.result.directoryCount === 0 && passon.result.bagitCount === 0 && passon.result.zipCount === 0) { + if (passon.result.bagitCount === 0 && passon.result.zipCount === 0) { let share = path.join(passon.webdav_path); debug('[%s] Load %s files from share at: %s', passon.id, passon.result.length, share); @@ -621,7 +622,7 @@ function checkEncoding(passon) { } if (invalidFiles.length !== 0) { - debug('[%s] Unsupported encoding found in file(s) %s!', passon.id, JSON.stringify(invalidFiles)); + debug('[%s] Unsupported encoding found in file(s) %o!', passon.id, invalidFiles); let err = new Error('unsupported encoding'); err.status = 422; err.msg = {}; @@ -662,13 +663,13 @@ function extractMetadata(passon) { metaextract_output_dir + ':' + metaextract_output_dir ]; if(config.fs.volume) { - debug('[%s] volume is configured, overwriting binds configuration (was %s)', passon.id, JSON.stringify(binds)); + debug('[%s] volume is configured, overwriting binds configuration (was %o)', passon.id, binds); // passon.compendium_path always starts with config.fs.base binds = [ config.fs.volume + ':' + config.fs.base ]; } - debug('[%s] Binds: %s', passon.id, JSON.stringify(binds)); + debug('[%s] Binds: %o', passon.id, binds); // run metadata extraction container let create_options = Object.assign( @@ -710,7 +711,7 @@ function extractMetadata(passon) { reject(err); } else { if (data.StatusCode === 0) { - debug('[%s] Completed metadata extraction: %s', passon.id, JSON.stringify(data)); + debug('[%s] Completed metadata extraction: %O', passon.id, data); // check if metadata was found, if so put the metadata directory into passon try { fs.readdir(metaextract_output_dir, (err, files) => { @@ -747,7 +748,7 @@ function extractMetadata(passon) { reject(err); } } else { - debug('[%s] Error during meta container run: %s', passon.id, JSON.stringify(data)); + debug('[%s] Error during meta container run: %o', passon.id, data); container.logs({ follow: true, stdout: true, @@ -891,7 +892,7 @@ function cleanup(passon) { } }) ]).then((results) => { - debug('[%s] Finished cleanup: %s', passon.id, JSON.stringify(results)); + debug('[%s] Finished cleanup: %o', passon.id, results); fulfill(passon); }); }); diff --git a/lib/uploader.js b/lib/uploader.js index b2a5890..e137a94 100644 --- a/lib/uploader.js +++ b/lib/uploader.js @@ -59,7 +59,7 @@ function Uploader(req, res) { done(passon.id, null); }) .catch(err => { - debug('[%s] Rejection or unhandled failure during upload: %s', passon.id, JSON.stringify(err)); + debug('[%s] Rejection or unhandled failure during upload: %o', passon.id, err); debug('[%s] Passon object:\n%s', util.inspect(passon, { depth: 1, colors: true })); let status = 500; if (err.status) { @@ -72,7 +72,7 @@ function Uploader(req, res) { res.status(status).send({ error: msg }); done(passon.id, err); }); - } + }; this.respond = (passon) => { return new Promise((fulfill) => { diff --git a/package.json b/package.json index ce5434b..d2466b7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "o2r-loader", - "version": "0.9.0", + "version": "0.10.0", "description": "Node.js implementation for loading compendia and workspaces from external repositories for the [o2r web api](http://o2r.info/o2r-web-api).", "main": "index.js", "scripts": { diff --git a/test/sciebo_erc.js b/test/sciebo_erc.js index dd4b2ea..ebf6da1 100644 --- a/test/sciebo_erc.js +++ b/test/sciebo_erc.js @@ -165,12 +165,11 @@ describe('Sciebo loader with compendia', function () { }).timeout(20000);; }); - // not implemented yet, must be able to load specific file: https://github.com/o2r-project/o2r-loader/issues/14 - describe.skip('create new compendium based on specific zip file in public WebDAV share with multiple zip files', () => { - it('should respond with an error', (done) => { + describe('create new compendium based on specific zip file in public WebDAV share with multiple zip files', () => { + it('should respond with a compendium ID', (done) => { let form = { - share_url: 'https://uni-muenster.sciebo.de/index.php/s/NSqqEdjVIjhPf0N', - path: '/iUaMu', + share_url: 'https://uni-muenster.sciebo.de/s/w9Ste65jjStVlI4', + path: '/newtainer2.zip', content_type: 'compendium' }; @@ -186,12 +185,54 @@ describe('Sciebo loader with compendia', function () { timeout: requestLoadingTimeout }, (err, res, body) => { assert.ifError(err); - assert.equal(res.statusCode, 400); - assert.isUndefined(JSON.parse(body).id, 'returned no id'); - assert.include(JSON.parse(body).error, 'load from share failed'); + assert.equal(res.statusCode, 200); + assert.isObject(JSON.parse(body), 'returned JSON'); + assert.isDefined(JSON.parse(body).id, 'returned id'); + compendium_id = JSON.parse(body).id; done(); }); }).timeout(20000); + + it('should download the files in the share and make them available via API', (done) => { + let form = { + share_url: 'https://uni-muenster.sciebo.de/s/w9Ste65jjStVlI4', + path: '/newtainer2.zip', + content_type: 'compendium' + }; + + let j = request.jar(); + let ck = request.cookie('connect.sid=' + cookie); + j.setCookie(ck, global.test_host); + + request({ + uri: global.test_host + '/api/v1/compendium', + method: 'POST', + jar: j, + form: form, + timeout: requestLoadingTimeout + }, (err, res, body) => { + assert.ifError(err); + let compendium_id = JSON.parse(body).id; + + request({ + uri: global.test_host_read + '/api/v1/compendium/' + compendium_id, + method: 'GET', + jar: j, + form: form, + timeout: requestLoadingTimeout + }, (err, res, body) => { + assert.ifError(err); + assert.equal(res.statusCode, 200); + let response = JSON.parse(body); + + assert.include(JSON.stringify(response), 'data/data/document.Rmd'); + assert.include(JSON.stringify(response), 'data/data/document.tex'); + assert.include(JSON.stringify(response), 'data/data/bagtainer.yml'); + + done(); + }); + }); + }).timeout(20000);; }); }); diff --git a/test/sciebo_workspace.js b/test/sciebo_workspace.js index 74618ab..c2c7afa 100644 --- a/test/sciebo_workspace.js +++ b/test/sciebo_workspace.js @@ -211,6 +211,73 @@ describe('Sciebo loader with workspaces', function () { }); }).timeout(20000); }); + + describe('create new compendium based on a workspace in a public WebDAV that contains a directory', () => { + it('should respond with a compendium ID', (done) => { + let form = { + share_url: 'https://uni-muenster.sciebo.de/s/Dct9JUTgnhAtZaM', + path: '/', + content_type: 'workspace' + }; + + let j = request.jar(); + let ck = request.cookie('connect.sid=' + cookie); + j.setCookie(ck, global.test_host); + + request({ + uri: global.test_host + '/api/v1/compendium', + method: 'POST', + jar: j, + form: form, + timeout: requestLoadingTimeout + }, (err, res, body) => { + assert.ifError(err); + assert.equal(res.statusCode, 200); + assert.isObject(JSON.parse(body), 'returned JSON'); + assert.isDefined(JSON.parse(body).id, 'returned id'); + done(); + }); + }).timeout(20000); + + it('should download the files in the workspace and list them via the API', (done) => { + let form = { + share_url: 'https://uni-muenster.sciebo.de/s/Dct9JUTgnhAtZaM', + content_type: 'workspace' + }; + + let j = request.jar(); + let ck = request.cookie('connect.sid=' + cookie); + j.setCookie(ck, global.test_host); + + request({ + uri: global.test_host + '/api/v1/compendium', + method: 'POST', + jar: j, + form: form, + timeout: requestLoadingTimeout + }, (err, res, body) => { + assert.ifError(err); + let compendium_id = JSON.parse(body).id; + + request({ + uri: global.test_host_read + '/api/v1/compendium/' + compendium_id, + method: 'GET', + jar: j, + timeout: requestLoadingTimeout + }, (err, res, body) => { + assert.ifError(err); + assert.equal(res.statusCode, 200); + let response = JSON.parse(body); + + assert.include(JSON.stringify(response), 'data/main.Rmd'); + assert.include(JSON.stringify(response), 'data/display.html'); + assert.include(JSON.stringify(response), 'data/subDirectory/data.csv'); + + done(); + }); + }); + }).timeout(20000); + }); }); describe('Sciebo loader with invalid requests', () => {