From 494e78c82741b6a8b03a3c0e046963f83a0e47f4 Mon Sep 17 00:00:00 2001 From: LukasLohoff Date: Thu, 22 Mar 2018 16:39:56 +0100 Subject: [PATCH 01/20] read id from compendium file for compendia --- config/config.js | 3 + lib/loader.js | 4 + lib/steps.js | 59 ++++++++++++-- lib/uploader.js | 1 + package-lock.json | 191 ++++++++++++++++++++++++++-------------------- package.json | 1 + 6 files changed, 171 insertions(+), 88 deletions(-) diff --git a/config/config.js b/config/config.js index 5be7915..274d472 100644 --- a/config/config.js +++ b/config/config.js @@ -84,6 +84,9 @@ c.bagit.validation = {}; c.bagit.validation.fast = false; c.bagit.validation.failUpload = true; +c.bagtainer = {}; +c.bagtainer.id_regex = /^[^-_.\n\r][a-zA-Z0-9\._-]*[^-_.\n\r]$/; + // metadata extraction options c.meta = {}; c.meta.container = {}; diff --git a/lib/loader.js b/lib/loader.js index df434cd..8cc5860 100644 --- a/lib/loader.js +++ b/lib/loader.js @@ -53,6 +53,8 @@ function Loader(req, res) { .then(steps.checkEncoding) .then(steps.detectBag) .then(validateBag) + .then(steps.detectCompendium) + .then(steps.fetchCompendiumID) .then(steps.extractMetadata) .then(steps.loadMetadata) .then(steps.brokerMetadata) @@ -98,6 +100,8 @@ function Loader(req, res) { .then(steps.detectBag) .then(steps.getTextFiles) .then(steps.checkEncoding) + .then(steps.detectCompendium) + .then(steps.fetchCompendiumID) .then(steps.extractMetadata) .then(steps.loadMetadata) .then(steps.brokerMetadata) diff --git a/lib/steps.js b/lib/steps.js index 63aa109..dcbcd07 100644 --- a/lib/steps.js +++ b/lib/steps.js @@ -36,6 +36,7 @@ const util = require('util'); const Docker = require('dockerode'); const Stream = require('stream'); const meta = require('../lib/meta'); +const yaml = require('js-yaml'); var docker = new Docker(); @@ -548,17 +549,16 @@ function detectCompendium(passon) { return new Promise((fulfill, reject) => { debug('[%s] Detecting compendium', passon.id); - let detectionFile = null; if (passon.isBag) { - detectionFile = path.join(passon.compendium_path, config.bagit.payloadDirectory, config.compendium.detectionFileName); + passon.detectionFile = path.join(passon.compendium_path, config.bagit.payloadDirectory, config.compendium.detectionFileName); } else { - detectionFile = path.join(passon.compendium_path, config.compendium.detectionFileName); + passon.detectionFile = path.join(passon.compendium_path, config.compendium.detectionFileName); } try { - fs.accessSync(detectionFile); + fs.accessSync(passon.detectionFile); - debug('[%s] Found %s - it\'s a compendium! Content was labeled as "%s"', passon.id, detectionFile, passon.content); + debug('[%s] Found %s - it\'s a compendium! Content was labeled as "%s"', passon.id, passon.detectionFile, passon.content); passon.isCompendium = true; fulfill(passon); } catch (error) { @@ -576,6 +576,54 @@ function detectCompendium(passon) { }); } +function fetchCompendiumID(passon) { + return new Promise((fulfill, reject) => { + if (passon.isCompendium) { + debug('[%s] Reading compendium ID from %s', passon.id, config.compendium.detectionFileName); + + let compendiumID = null; + + // Get document, or throw exception on error + try { + let doc = yaml.safeLoad(fs.readFileSync(passon.detectionFile, 'utf8')); + compendiumID = doc.id; + debug('[%s] Successfully read compendium metadata: %O', passon.id, doc); + + } catch (error) { + debug('[%s] Could not read compendium detection file but content is %s: %s', passon.id, passon.content, error); + + error.msg = 'content_type is ' + passon.content + ' but no compendium provided.'; + error.status = 400; + reject(error); + } + + if (!compendiumID){ + debug('[%s] No ID specified for compendium', passon.id); + let error = new Error('No id found in compendium detection file'); + error.status = 400; + reject(error); + } else { + // Validate compendium id + let idRegex = new RegExp(config.bagtainer.id_regex); + if (idRegex.test(compendiumID)) { + debug('[%s] Assigned new ID %s to compendium', passon.id, compendiumID); + passon.id = compendiumID; + fulfill(passon); + } else { + debug('[%s] Invalid ID %s specified for compendium in %s', passon.id, compendiumID, passon.detectionFile); + let error = new Error('Invalid id found in compendium detection file'); + error.status = 400; + reject(error); + } + } + + } else { + debug('[%s] Not an compendium, keeping generated ID', passon.id); + fulfill(passon); + } + }); +} + function getTextFiles(passon) { return new Promise((fulfill, reject) => { debug('[%s] Getting text files', passon.id); @@ -906,6 +954,7 @@ module.exports = { detectCompendium: detectCompendium, getTextFiles: getTextFiles, checkEncoding: checkEncoding, + fetchCompendiumID : fetchCompendiumID, extractMetadata: extractMetadata, loadMetadata: loadMetadata, brokerMetadata: brokerMetadata, diff --git a/lib/uploader.js b/lib/uploader.js index e137a94..0af3851 100644 --- a/lib/uploader.js +++ b/lib/uploader.js @@ -48,6 +48,7 @@ function Uploader(req, res) { .then(steps.detectBag) .then(validateBag) .then(steps.detectCompendium) + .then(steps.fetchCompendiumID) .then(steps.extractMetadata) .then(steps.loadMetadata) .then(steps.brokerMetadata) diff --git a/package-lock.json b/package-lock.json index 01934fe..4073f3b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "o2r-loader", - "version": "0.8.0", + "version": "0.10.0", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -77,6 +77,14 @@ "readable-stream": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.1.5.tgz" } }, + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "requires": { + "sprintf-js": "1.0.3" + } + }, "array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -198,7 +206,7 @@ "debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "integrity": "sha1-XRKFFd8TT/Mn6QpMk/Tgd6U2NB8=", "requires": { "ms": "2.0.0" } @@ -405,7 +413,7 @@ "debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "integrity": "sha1-XRKFFd8TT/Mn6QpMk/Tgd6U2NB8=", "requires": { "ms": "2.0.0" } @@ -460,7 +468,7 @@ "content-type": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", - "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" + "integrity": "sha1-4TjMdeBAxyexlm/l5fjJruJW/js=" }, "cookie": { "version": "0.3.1", @@ -492,7 +500,7 @@ "boom": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/boom/-/boom-5.2.0.tgz", - "integrity": "sha512-Z5BTk6ZRe4tXXQlkqftmsAUANpXmuwlsF5Oov8ThoMbQRzdGTA1ngYRW160GexgOgjsFOKJz0LYhoNi+2AMBUw==", + "integrity": "sha1-XdnabuOl8wIHdDYpDLcX0/SlTgI=", "requires": { "hoek": "4.2.0" } @@ -515,7 +523,7 @@ "debug": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", - "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", + "integrity": "sha1-W7WgZyYotkFJVmuhaBnmFRjGcmE=", "requires": { "ms": "2.0.0" } @@ -593,10 +601,6 @@ "type-detect": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.3.tgz" } }, - "deepmerge": { - "version": "https://registry.npmjs.org/deepmerge/-/deepmerge-1.5.1.tgz", - "integrity": "sha1-wFO/Bv1ydvGZT3DAmgdgy2GlYjc=" - }, "defaults": { "version": "https://registry.npmjs.org/defaults/-/defaults-1.0.3.tgz", "integrity": "sha1-xlYFHpgX2f8I7YgUd/P+QBnz730=", @@ -631,7 +635,7 @@ "detect-character-encoding": { "version": "0.5.1", "resolved": "https://registry.npmjs.org/detect-character-encoding/-/detect-character-encoding-0.5.1.tgz", - "integrity": "sha512-Dc2MB8TZ1iB1ZyXUVOqhiG+rR/YQZo7KlyfO2jioiA5s7Fo339FRvQXLRn7p/heQHxvV8XZe3wfBKkTIuv+PlA==", + "integrity": "sha1-UnPb1HDpx/wZDK+j4kMOUN77Rvo=", "requires": { "bindings": "1.3.0", "nan": "https://registry.npmjs.org/nan/-/nan-2.6.2.tgz" @@ -664,7 +668,7 @@ "diff": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/diff/-/diff-3.3.1.tgz", - "integrity": "sha512-MKPHZDMB0o6yHyDryUOScqZibp914ksXwAMYMTHj6KO8UeKsRYNJD3oNCKjTqZon+V488P7N/HzXF8t7ZR95ww==", + "integrity": "sha1-qoVnpu7QPFMfyJ0/cRzQ5SWd7HU=", "dev": true }, "directory-tree": { @@ -674,7 +678,7 @@ "docker-modem": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-1.0.3.tgz", - "integrity": "sha512-D55iUS6GrV7ohFurvAl1PYqEf8k/x3+xWxtOFoKOs8JM2xyT1yBbZ216Nxjk4F0TboR5iqVVITwPiH/AXeP9Ig==", + "integrity": "sha1-cMqd5kCy/DfkiFltEVcs7eYKx1o=", "requires": { "JSONStream": "0.10.0", "debug": "2.6.9", @@ -685,7 +689,7 @@ "debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "integrity": "sha1-XRKFFd8TT/Mn6QpMk/Tgd6U2NB8=", "requires": { "ms": "2.0.0" } @@ -716,7 +720,7 @@ "dockerode": { "version": "2.5.3", "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-2.5.3.tgz", - "integrity": "sha512-LQKXR5jyI+G/+5OhZCi40m0ArY4j46g7Tl71Vtn10Ekt5TiyDzZAoqXOCS6edQpEuGbdFgSDJxleFqLxACpKJg==", + "integrity": "sha1-Wsw8Bx96JuRDpbWM83U+8vPCvAQ=", "requires": { "concat-stream": "1.5.2", "docker-modem": "1.0.3", @@ -805,13 +809,6 @@ "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.1.tgz", "integrity": "sha1-eePVhlU0aQn+bw9Fpd5oEDspTSA=" }, - "encoding": { - "version": "https://registry.npmjs.org/encoding/-/encoding-0.1.12.tgz", - "integrity": "sha1-U4tm8+5izRq1HsMjgp0flIDHS+s=", - "requires": { - "iconv-lite": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.15.tgz" - } - }, "end-of-stream": { "version": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.0.tgz", "integrity": "sha1-epDYM+/abPpurA9JSduw+tOmMgY=", @@ -839,6 +836,11 @@ "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", "dev": true }, + "esprima": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.0.tgz", + "integrity": "sha512-oftTcaMu/EGrEIu904mWteKIv8vMuOgGYo7EhVJJN00R/EED9DCua/xxHRdYnKtcECzVg7xOWhflvJMnqcFZjw==" + }, "etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", @@ -884,7 +886,7 @@ "debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "integrity": "sha1-XRKFFd8TT/Mn6QpMk/Tgd6U2NB8=", "requires": { "ms": "2.0.0" } @@ -910,7 +912,7 @@ "setprototypeof": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", - "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==" + "integrity": "sha1-0L2FU2iHtv58DYGMuWLZ2RxU5lY=" }, "statuses": { "version": "1.3.1", @@ -931,7 +933,7 @@ "express-session": { "version": "1.15.6", "resolved": "https://registry.npmjs.org/express-session/-/express-session-1.15.6.tgz", - "integrity": "sha512-r0nrHTCYtAMrFwZ0kBzZEXa1vtPVrw0dKvGSrKP4dahwBQ1BJpF2/y1Pp4sCD/0kvxV4zZeclyvfmw0B4RMJQA==", + "integrity": "sha1-R7QWDIj0KrcP6KUI4xy/92dXqwo=", "requires": { "cookie": "0.3.1", "cookie-signature": "1.0.6", @@ -947,7 +949,7 @@ "debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "integrity": "sha1-XRKFFd8TT/Mn6QpMk/Tgd6U2NB8=", "requires": { "ms": "2.0.0" } @@ -1007,7 +1009,7 @@ "debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "integrity": "sha1-XRKFFd8TT/Mn6QpMk/Tgd6U2NB8=", "requires": { "ms": "2.0.0" } @@ -1045,12 +1047,13 @@ "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" }, "fs-extra": { - "version": "https://registry.npmjs.org/fs-extra/-/fs-extra-4.0.2.tgz", - "integrity": "sha1-+RcExT0bRh+JNFKwwwfZmXZHq2s=", + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-4.0.3.tgz", + "integrity": "sha512-q6rbdDd1o2mAnQreO7YADIxf/Whx4AHBiRf6d+/cVT8h44ss+lHgxf1FemcqDnQt9X3ct4McHr+JMGlYSsK7Cg==", "requires": { "graceful-fs": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.11.tgz", - "jsonfile": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", - "universalify": "https://registry.npmjs.org/universalify/-/universalify-0.1.1.tgz" + "jsonfile": "4.0.0", + "universalify": "0.1.1" } }, "fs.realpath": { @@ -1137,7 +1140,7 @@ "growl": { "version": "1.10.3", "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.3.tgz", - "integrity": "sha512-hKlsbA5Vu3xsh1Cg3J7jSmX/WaW6A5oBeqzM88oNbCRQFz+zUaXm6yxS4RVytp1scBoJzSYl4YAEOQIt6O8V1Q==", + "integrity": "sha1-GSa6kM8+3+KttJJ/WIC8IsZseQ8=", "dev": true }, "har-schema": { @@ -1167,7 +1170,7 @@ "hawk": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/hawk/-/hawk-6.0.2.tgz", - "integrity": "sha512-miowhl2+U7Qle4vdLqDdPt9m09K6yZhkLDTWGoUiUzrQCn+mHHSmfJgAyGaLRZbPmTqfFFjRV1QWCW0VWUJBbQ==", + "integrity": "sha1-r02RTrBl+bXOTZ0RwcshJu7MMDg=", "requires": { "boom": "4.3.1", "cryptiles": "3.1.2", @@ -1196,7 +1199,7 @@ "hooks-fixed": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/hooks-fixed/-/hooks-fixed-2.0.2.tgz", - "integrity": "sha512-YurCM4gQSetcrhwEtpQHhQ4M7Zo7poNGqY4kQGeBS6eZtOcT3tnNs01ThFa0jYBByAiYt1MjMjP/YApG0EnAvQ==" + "integrity": "sha1-IAdtqgfnfYphBog84/FyLgURQLA=" }, "htmlparser2": { "version": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.9.2.tgz", @@ -1238,10 +1241,6 @@ "sshpk": "1.13.1" } }, - "iconv-lite": { - "version": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.15.tgz", - "integrity": "sha1-/iZaIYrGpXz+hUkn6dBMGYJe3es=" - }, "ieee754": { "version": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.8.tgz", "integrity": "sha1-vjPUCsEO8ZJnAfbwii2G+/0a0+Q=" @@ -1296,6 +1295,15 @@ "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" }, + "js-yaml": { + "version": "3.11.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.11.0.tgz", + "integrity": "sha512-saJstZWv7oNeOyBh3+Dx1qWzhW0+e6/8eDzo7p5rDFqxntSztloLtuKu+Ejhtq82jsilwOIZYsCz+lIjthg1Hw==", + "requires": { + "argparse": "1.0.10", + "esprima": "4.0.0" + } + }, "jsbn": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", @@ -1318,7 +1326,8 @@ "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" }, "jsonfile": { - "version": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=", "requires": { "graceful-fs": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.11.tgz" @@ -1366,6 +1375,11 @@ "version": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" }, + "merge": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/merge/-/merge-1.2.0.tgz", + "integrity": "sha1-dTHjnUlJwoGma4xabgJl6LBYlNo=" + }, "merge-descriptors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", @@ -1379,7 +1393,7 @@ "mime": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/mime/-/mime-1.4.1.tgz", - "integrity": "sha512-KI1+qOZu5DcW6wayYHSzR/tXKCDC5Om4s1z2QJjDULzLcmf3DvzS7oluY4HCTrc+9FiKmWUgeNLg7W3uIQvxtQ==" + "integrity": "sha1-Eh+evEnjdm8xGnbh+hyAA8SwOqY=" }, "mime-db": { "version": "https://registry.npmjs.org/mime-db/-/mime-db-1.26.0.tgz", @@ -1395,7 +1409,7 @@ "mimos": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/mimos/-/mimos-4.0.0.tgz", - "integrity": "sha512-JvlvRLqGIlk+AYypWrbrDmhsM+6JVx/xBM5S3AMwTBz1trPCEoPN/swO2L4Wu653fL7oJdgk8DMQyG/Gq3JkZg==", + "integrity": "sha1-duPScShDHLZIL9FbIEdXGa1ialo=", "requires": { "hoek": "5.0.2", "mime-db": "https://registry.npmjs.org/mime-db/-/mime-db-1.26.0.tgz" @@ -1524,7 +1538,7 @@ "mongojs": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/mongojs/-/mongojs-2.4.1.tgz", - "integrity": "sha512-R344Q8ufjcqyFHO1CrxYboUBrEJwmsvMBtI8wsjCZq90mh/lzT0PBleAD6d1f8s07zeHSM2ebeu3OwMC4wxQlg==", + "integrity": "sha1-bDZYXrv3yC4lesrDPWaFF69VFhI=", "dev": true, "requires": { "each-series": "1.0.0", @@ -1627,7 +1641,7 @@ "muri": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/muri/-/muri-1.3.0.tgz", - "integrity": "sha512-FiaFwKl864onHFFUV/a2szAl7X0fxVlSKNdhTf+BM8i8goEgYut8u5P9MqQqIYwvaMxjzVESsoEm/2kfkFH1rg==" + "integrity": "sha1-rszz22TFaqfFs04A+Vt4eFJ6RyE=" }, "nan": { "version": "https://registry.npmjs.org/nan/-/nan-2.6.2.tgz", @@ -1639,12 +1653,9 @@ "integrity": "sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk=" }, "node-fetch": { - "version": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz", - "integrity": "sha1-mA9vcthSEaU0fGsrwYxbhMPrR+8=", - "requires": { - "encoding": "https://registry.npmjs.org/encoding/-/encoding-0.1.12.tgz", - "is-stream": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz" - } + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.0.0.tgz", + "integrity": "sha1-mCu6Q+zU8pIqKcwYamu7C7c/y6Y=" }, "node-gyp": { "version": "https://registry.npmjs.org/node-gyp/-/node-gyp-3.6.2.tgz", @@ -1837,10 +1848,11 @@ "qs": { "version": "6.5.1", "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz", - "integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A==" + "integrity": "sha1-NJzfbu+J7EXBLX1es/wMhwNDptg=" }, "querystringify": { - "version": "https://registry.npmjs.org/querystringify/-/querystringify-1.0.0.tgz", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-1.0.0.tgz", "integrity": "sha1-YoYkIRLFtxL6ZU5SZlK/ahP/Bcs=" }, "random-bytes": { @@ -1906,7 +1918,7 @@ "request": { "version": "2.83.0", "resolved": "https://registry.npmjs.org/request/-/request-2.83.0.tgz", - "integrity": "sha512-lR3gD69osqm6EYLk9wB/G1W/laGWjzH90t1vEa2xuxHD5KUrSzp9pUSfTm+YC5Nxt2T8nMPEvKlhbQayU7bgFw==", + "integrity": "sha1-ygtl2gLtYpNYh4COb1EDgQNOM1Y=", "requires": { "aws-sign2": "0.7.0", "aws4": "1.6.0", @@ -1950,14 +1962,15 @@ "require_optional": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/require_optional/-/require_optional-1.0.1.tgz", - "integrity": "sha512-qhM/y57enGWHAe3v/NcwML6a3/vfESLe/sGM2dII+gEO0BpKRUkWZow/tyloNqJyN6kXSl3RyyM8Ll5D/sJP8g==", + "integrity": "sha1-TPNaQkf2TKPfjC7yCMxJSxyo/C4=", "requires": { "resolve-from": "2.0.0", "semver": "https://registry.npmjs.org/semver/-/semver-5.3.0.tgz" } }, "requires-port": { - "version": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", "integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8=" }, "resolve-from": { @@ -1983,11 +1996,12 @@ "safe-buffer": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz", - "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg==" + "integrity": "sha1-iTMSr2myEj3vcfV4iQAWce6yyFM=" }, "sax": { - "version": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", - "integrity": "sha1-KBYjTiN4vdxOU1T6tcqold9xANk=" + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" }, "seek-bzip": { "version": "https://registry.npmjs.org/seek-bzip/-/seek-bzip-1.0.5.tgz", @@ -2012,7 +2026,7 @@ "send": { "version": "0.16.1", "resolved": "https://registry.npmjs.org/send/-/send-0.16.1.tgz", - "integrity": "sha512-ElCLJdJIKPk6ux/Hocwhk7NFHpI3pVm/IZOYWqUmoxcgeyM+MpxHHKhb8QmlJDX1pU6WrgaHBkVNm73Sv7uc2A==", + "integrity": "sha1-pw4coh0TgsEdDZ9iMd6ygQgNerM=", "requires": { "debug": "2.6.9", "depd": "1.1.1", @@ -2032,7 +2046,7 @@ "debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "integrity": "sha1-XRKFFd8TT/Mn6QpMk/Tgd6U2NB8=", "requires": { "ms": "2.0.0" } @@ -2052,7 +2066,7 @@ "serve-static": { "version": "1.13.1", "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.13.1.tgz", - "integrity": "sha512-hSMUZrsPa/I09VYFJwa627JJkNs0NrfL1Uzuup+GqHfToR2KcsXFymXSV90hoyw3M+msjFuQly+YzIH/q0MGlQ==", + "integrity": "sha1-TFfVNASnYdjy58HooYpH2/J4pxk=", "requires": { "encodeurl": "1.0.1", "escape-html": "1.0.3", @@ -2115,6 +2129,11 @@ "resolved": "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz", "integrity": "sha1-bIOv82kvphJW4M0ZfgXp3hV2kaY=" }, + "sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" + }, "sshpk": { "version": "1.13.1", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.13.1.tgz", @@ -2174,7 +2193,7 @@ "supports-color": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-4.4.0.tgz", - "integrity": "sha512-rKC3+DyXWgK0ZLKwmRsrkyHVZAjNkfzeehuFWdGGcqGDTZFH73+RH6S/RDAAxl9GusSjZSUWYLmT9N5pzXFOXQ==", + "integrity": "sha1-iD992rwWUUKyphQn8zUt7RldGj4=", "dev": true, "requires": { "has-flag": "2.0.0" @@ -2280,7 +2299,7 @@ "uid-safe": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/uid-safe/-/uid-safe-2.1.5.tgz", - "integrity": "sha512-KPHm4VL5dDXKz01UuEd88Df+KzynaohSL9fBh096KWAxSKZQDI2uBrVqtvRM4rwrIrRRKsdLNML/lnaaVSRioA==", + "integrity": "sha1-Kz1cckDo/C5Y+Komnl7knAhXvTo=", "requires": { "random-bytes": "1.0.0" } @@ -2294,7 +2313,8 @@ } }, "universalify": { - "version": "https://registry.npmjs.org/universalify/-/universalify-0.1.1.tgz", + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.1.tgz", "integrity": "sha1-+nG63UQ3r0wUiEHjs7Fl+enlkLc=" }, "unpipe": { @@ -2303,15 +2323,17 @@ "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" }, "url-join": { - "version": "https://registry.npmjs.org/url-join/-/url-join-2.0.2.tgz", - "integrity": "sha1-wHJ1aWetJLi1nldBVRyqx49QuLc=" + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/url-join/-/url-join-4.0.0.tgz", + "integrity": "sha1-TTNA6AfTdzvamZH4MFrNzCpmXSo=" }, "url-parse": { - "version": "https://registry.npmjs.org/url-parse/-/url-parse-1.1.9.tgz", - "integrity": "sha1-xn8dd11R8KGJEd17P/rSe7nlvRk=", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.2.0.tgz", + "integrity": "sha512-DT1XbYAfmQP65M/mE6OALxmXzZ/z1+e5zk2TcSKe/KiYbNGZxgtttzC0mR/sjopbpOXcbniq7eIKmocJnUWlEw==", "requires": { - "querystringify": "https://registry.npmjs.org/querystringify/-/querystringify-1.0.0.tgz", - "requires-port": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz" + "querystringify": "1.0.0", + "requires-port": "1.0.0" } }, "util": { @@ -2339,7 +2361,7 @@ "uuid": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.1.0.tgz", - "integrity": "sha512-DIWtzUkw04M4k3bf1IcpS2tngXEL26YUD2M0tMDUpnUrz2hgzUBlD55a4FjdLGPvfHxS6uluGWvaVEqgBcVa+g==" + "integrity": "sha1-PdPT55Crwk17DToDT/q6vijrvAQ=" }, "validator": { "version": "9.1.1", @@ -2362,14 +2384,15 @@ } }, "webdav": { - "version": "https://registry.npmjs.org/webdav/-/webdav-1.1.0.tgz", - "integrity": "sha1-dXwA3L0XrRryeoXoDOP2Y5RLW9M=", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/webdav/-/webdav-1.5.0.tgz", + "integrity": "sha512-jgy8pZSEfz+KQTao7tOD3I6G0Sv7RkUyuiGuW5CF+R3+9IzldSJOv2f40qksoAuINQOXFvXUJNINAEheb+Ii4A==", "requires": { - "deepmerge": "https://registry.npmjs.org/deepmerge/-/deepmerge-1.5.1.tgz", - "node-fetch": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz", - "url-join": "https://registry.npmjs.org/url-join/-/url-join-2.0.2.tgz", - "url-parse": "https://registry.npmjs.org/url-parse/-/url-parse-1.1.9.tgz", - "xml2js": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz" + "merge": "1.2.0", + "node-fetch": "2.0.0", + "url-join": "4.0.0", + "url-parse": "1.2.0", + "xml2js": "0.4.19" } }, "which": { @@ -2391,16 +2414,18 @@ "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, "xml2js": { - "version": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", - "integrity": "sha1-aGwg8hMgnpSr8NG88e+qKRx4J6c=", + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", "requires": { - "sax": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", - "xmlbuilder": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.4.tgz" + "sax": "1.2.4", + "xmlbuilder": "9.0.7" } }, "xmlbuilder": { - "version": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.4.tgz", - "integrity": "sha1-UZy0ymhtAFqEINNJbz8MruzKWA8=" + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" }, "xtend": { "version": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", diff --git a/package.json b/package.json index d2466b7..a02cd86 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "express-session": "^1.15.6", "fs-extra": "^4.0.2", "htmlparser2": "^3.9.2", + "js-yaml": "^3.11.0", "mimos": "^4.0.0", "mongoose": "^4.13.2", "mongoose-timestamp": "^0.6.0", From 42407efec1700654e642d9eee846be2048d91304 Mon Sep 17 00:00:00 2001 From: LukasLohoff Date: Thu, 22 Mar 2018 16:40:17 +0100 Subject: [PATCH 02/20] bump minor version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a02cd86..1c1cd7d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "o2r-loader", - "version": "0.10.0", + "version": "0.10.1", "description": "Node.js implementation for loading compendia and workspaces from external repositories for the [o2r web api](http://o2r.info/o2r-web-api).", "main": "index.js", "scripts": { From 89c87f3fcbaee98b164aa0fe1c99dca24860cf7d Mon Sep 17 00:00:00 2001 From: LukasLohoff Date: Fri, 23 Mar 2018 14:58:41 +0100 Subject: [PATCH 03/20] add test cases for existing id and invalid id in compendium --- test/direct_erc.js | 48 +++++++++++++++++++++++++++++++++++++++++++++- test/sciebo_erc.js | 6 +++--- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/test/direct_erc.js b/test/direct_erc.js index 3440674..e29196c 100644 --- a/test/direct_erc.js +++ b/test/direct_erc.js @@ -21,7 +21,7 @@ const request = require('request'); const fs = require('fs'); const config = require('../config/config'); -require("./setup") +require("./setup"); const cookie_o2r = 's:C0LIrsxGtHOGHld8Nv2jedjL4evGgEHo.GMsWD5Vveq0vBt7/4rGeoH5Xx7Dd2pgZR9DvhKCyDTY'; const requestLoadingTimeout = 15000; const createCompendiumPostRequest = require('./util').createCompendiumPostRequest; @@ -149,6 +149,52 @@ describe('Direct upload of ERC', function () { }).timeout(requestLoadingTimeout); }); + describe('POST /api/v1/compendium with invalid id in bag', () => { + it('should fail the upload because bag ID is invalid (contains invalid chars)', (done) => { + let req = createCompendiumPostRequest('./test/erc/invalid_id', cookie_o2r); + + request(req, (err, res, body) => { + assert.ifError(err); + assert.equal(res.statusCode, 400); + assert.include(body, 'Invalid id found in compendium detection file'); + done(); + }); + }).timeout(requestLoadingTimeout); + + it('should not tell about internal server configuration in the error message', (done) => { + let req = createCompendiumPostRequest('./test/erc/invalid_id', cookie_o2r); + + request(req, (err, res, body) => { + assert.ifError(err); + assert.notInclude(body, config.fs.base); + done(); + }); + }).timeout(requestLoadingTimeout); + }); + + describe('POST /api/v1/compendium with two compendia with the same ID', () => { + it('should respond with HTTP 200 OK for the first compendium', (done) => { + let req = createCompendiumPostRequest('./test/erc/executable', cookie_o2r); + + request(req, (err, res, body) => { + assert.ifError(err); + assert.equal(res.statusCode, 200); + done(); + }); + }).timeout(requestLoadingTimeout); + + it('should fail the second upload because bag ID is already used', (done) => { + let req = createCompendiumPostRequest('./test/erc/invalid_id', cookie_o2r); + + request(req, (err, res, body) => { + assert.ifError(err); + assert.equal(res.statusCode, 400); + assert.include(body, 'ID already exists'); + done(); + }); + }).timeout(requestLoadingTimeout); + }); + describe.skip('POST /api/v1/compendium with virus', () => { it('upload compendium should fail and return an error message about infected files', (done) => { let req = createCompendiumPostRequest('./test/erc/virustainer', cookie); diff --git a/test/sciebo_erc.js b/test/sciebo_erc.js index ebf6da1..ae85e12 100644 --- a/test/sciebo_erc.js +++ b/test/sciebo_erc.js @@ -20,7 +20,7 @@ const assert = require('chai').assert; const request = require('request'); const config = require('../config/config'); -require("./setup") +require("./setup"); const cookie = 's:C0LIrsxGtHOGHld8Nv2jedjL4evGgEHo.GMsWD5Vveq0vBt7/4rGeoH5Xx7Dd2pgZR9DvhKCyDTY'; const cookie_plain = 's:yleQfdYnkh-sbj9Ez--_TWHVhXeXNEgq.qRmINNdkRuJ+iHGg5woRa9ydziuJ+DzFG9GnAZRvaaM'; const requestLoadingTimeout = 20000; @@ -94,7 +94,7 @@ describe('Sciebo loader with compendia', function () { done(); }); }); - }).timeout(20000);; + }).timeout(20000); }); describe('create new compendium based on public WebDAV share with one zip file', () => { @@ -162,7 +162,7 @@ describe('Sciebo loader with compendia', function () { done(); }); }); - }).timeout(20000);; + }).timeout(20000); }); describe('create new compendium based on specific zip file in public WebDAV share with multiple zip files', () => { From c6e0ac12d276f2e33fd979f98db83d331531b13c Mon Sep 17 00:00:00 2001 From: LukasLohoff Date: Fri, 23 Mar 2018 14:59:07 +0100 Subject: [PATCH 04/20] add test ERC with invalid id --- test/erc/invalid_id/bag-info.txt | 3 +++ test/erc/invalid_id/bagit.txt | 2 ++ test/erc/invalid_id/data/Dockerfile | 3 +++ test/erc/invalid_id/data/erc.yml | 3 +++ test/erc/invalid_id/data/test.txt | 1 + test/erc/invalid_id/manifest-md5.txt | 3 +++ test/erc/invalid_id/tagmanifest-md5.txt | 3 +++ 7 files changed, 18 insertions(+) create mode 100644 test/erc/invalid_id/bag-info.txt create mode 100644 test/erc/invalid_id/bagit.txt create mode 100644 test/erc/invalid_id/data/Dockerfile create mode 100644 test/erc/invalid_id/data/erc.yml create mode 100644 test/erc/invalid_id/data/test.txt create mode 100644 test/erc/invalid_id/manifest-md5.txt create mode 100644 test/erc/invalid_id/tagmanifest-md5.txt diff --git a/test/erc/invalid_id/bag-info.txt b/test/erc/invalid_id/bag-info.txt new file mode 100644 index 0000000..069ad03 --- /dev/null +++ b/test/erc/invalid_id/bag-info.txt @@ -0,0 +1,3 @@ +Payload-Oxum: 122.3 +Bagging-Date: 2017-09-20 +Bag-Size: 0.8 KB diff --git a/test/erc/invalid_id/bagit.txt b/test/erc/invalid_id/bagit.txt new file mode 100644 index 0000000..c4aebb4 --- /dev/null +++ b/test/erc/invalid_id/bagit.txt @@ -0,0 +1,2 @@ +BagIt-Version: 0.97 +Tag-File-Character-Encoding: UTF-8 diff --git a/test/erc/invalid_id/data/Dockerfile b/test/erc/invalid_id/data/Dockerfile new file mode 100644 index 0000000..66c4f88 --- /dev/null +++ b/test/erc/invalid_id/data/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine +MAINTAINER +CMD ["uname", "-a" ] \ No newline at end of file diff --git a/test/erc/invalid_id/data/erc.yml b/test/erc/invalid_id/data/erc.yml new file mode 100644 index 0000000..6ae4bc9 --- /dev/null +++ b/test/erc/invalid_id/data/erc.yml @@ -0,0 +1,3 @@ +--- +id: -kIbe_bWnPlx +spec_version: "1" diff --git a/test/erc/invalid_id/data/test.txt b/test/erc/invalid_id/data/test.txt new file mode 100644 index 0000000..40f990a --- /dev/null +++ b/test/erc/invalid_id/data/test.txt @@ -0,0 +1 @@ +step 04 - build image diff --git a/test/erc/invalid_id/manifest-md5.txt b/test/erc/invalid_id/manifest-md5.txt new file mode 100644 index 0000000..e2d3f86 --- /dev/null +++ b/test/erc/invalid_id/manifest-md5.txt @@ -0,0 +1,3 @@ +a48f626014a144f1cf4a9b3b8cac955e data/Dockerfile +274c9deef9d7a23aae93b83af1aae677 data/erc.yml +eadd49d3a3facebd5baa60d519505662 data/test.txt diff --git a/test/erc/invalid_id/tagmanifest-md5.txt b/test/erc/invalid_id/tagmanifest-md5.txt new file mode 100644 index 0000000..4883c6c --- /dev/null +++ b/test/erc/invalid_id/tagmanifest-md5.txt @@ -0,0 +1,3 @@ +f3803d06c2dcae6f32217c4d162122aa bag-info.txt +4442ae1092385ab634d11fbfcfc13957 manifest-md5.txt +9e5ad981e0d29adc278f6a294b8c2aca bagit.txt From 3e5a3e2e8a987e8bc719c8786bef37d2986084f0 Mon Sep 17 00:00:00 2001 From: LukasLohoff Date: Fri, 23 Mar 2018 14:59:40 +0100 Subject: [PATCH 05/20] update readme with info about public sciebo ERCs --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 3bca822..4d92da3 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,17 @@ TEST_HOST=http://localhost:80 npm test ``` For this, the loader has to run locally or as part of a docker-compose configuration. +### Public shares + +The tests for public shares (`sciebo_erc.js`, `sciebo_workspace.js` and `zenodo.js`) use ERC uploaded to the respective services. + +They can be found at + +* Sciebo: [public link](https://uni-muenster.sciebo.de/index.php/s/h5tNYXsS1Bsv4qr) | [private link](https://uni-muenster.sciebo.de/f/749265161) +* Zenodo: https://sandbox.zenodo.org/deposit/69114 + +For information on which share URL belongs to which compendium, see the file `README` in the [`integration_test_shares`](https://uni-muenster.sciebo.de/index.php/s/h5tNYXsS1Bsv4qr) folder. + ## Dockerfile The file `Dockerfile` describes the Docker image published at [Docker Hub](https://hub.docker.com/r/o2rproject/o2r-loader/). From 2f40090c90bcaa39b43831397ab5a13bb9482f1e Mon Sep 17 00:00:00 2001 From: LukasLohoff Date: Fri, 23 Mar 2018 15:00:02 +0100 Subject: [PATCH 06/20] check for existing compendium id when uploading --- lib/steps.js | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/lib/steps.js b/lib/steps.js index dcbcd07..af34164 100644 --- a/lib/steps.js +++ b/lib/steps.js @@ -583,7 +583,7 @@ function fetchCompendiumID(passon) { let compendiumID = null; - // Get document, or throw exception on error + // Get compendium yml, or throw exception on error try { let doc = yaml.safeLoad(fs.readFileSync(passon.detectionFile, 'utf8')); compendiumID = doc.id; @@ -591,7 +591,6 @@ function fetchCompendiumID(passon) { } catch (error) { debug('[%s] Could not read compendium detection file but content is %s: %s', passon.id, passon.content, error); - error.msg = 'content_type is ' + passon.content + ' but no compendium provided.'; error.status = 400; reject(error); @@ -606,9 +605,27 @@ function fetchCompendiumID(passon) { // Validate compendium id let idRegex = new RegExp(config.bagtainer.id_regex); if (idRegex.test(compendiumID)) { - debug('[%s] Assigned new ID %s to compendium', passon.id, compendiumID); - passon.id = compendiumID; - fulfill(passon); + + // Check if id found in compendium file already exists + Compendium.findOne({ id: compendiumID }).select('id user').exec((err, compendium) => { + // eslint-disable-next-line no-eq-null, eqeqeq + if (err) { + debug('[%s] Error querying compendium with id %s', passon.id, compendiumID); + err.message = 'Error querying compendium'; + err.status = 400; + reject(err); + } + if (compendium === null) { + debug('[%s] Assigned new ID %s to compendium', passon.id, compendiumID); + passon.id = compendiumID; + fulfill(passon); + } else { + debug('[%s] Compendium with ID %s already exists; user %s', passon.id, compendiumID, compendium.user); + let error = new Error('Error fetching ID from compendium, ID already exists'); + error.status = 400; + reject(error); + } + }); } else { debug('[%s] Invalid ID %s specified for compendium in %s', passon.id, compendiumID, passon.detectionFile); let error = new Error('Invalid id found in compendium detection file'); @@ -616,7 +633,6 @@ function fetchCompendiumID(passon) { reject(error); } } - } else { debug('[%s] Not an compendium, keeping generated ID', passon.id); fulfill(passon); From 1e6c982457d821e5190a4b9d08d409e872a2d96e Mon Sep 17 00:00:00 2001 From: LukasLohoff Date: Fri, 23 Mar 2018 16:16:05 +0100 Subject: [PATCH 07/20] fix error messages --- lib/steps.js | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/steps.js b/lib/steps.js index af34164..229511e 100644 --- a/lib/steps.js +++ b/lib/steps.js @@ -590,15 +590,17 @@ function fetchCompendiumID(passon) { debug('[%s] Successfully read compendium metadata: %O', passon.id, doc); } catch (error) { - debug('[%s] Could not read compendium detection file but content is %s: %s', passon.id, passon.content, error); - error.msg = 'content_type is ' + passon.content + ' but no compendium provided.'; + debug('[%s] Could not read compendium detection file %s: %s', passon.id, passon.detectionFile, error); + error.msg = 'Could not read compendium detection file'; error.status = 400; reject(error); } if (!compendiumID){ debug('[%s] No ID specified for compendium', passon.id); - let error = new Error('No id found in compendium detection file'); + let message = 'No id found in compendium detection file'; + let error = new Error(message); + error.msg = message; error.status = 400; reject(error); } else { @@ -621,14 +623,18 @@ function fetchCompendiumID(passon) { fulfill(passon); } else { debug('[%s] Compendium with ID %s already exists; user %s', passon.id, compendiumID, compendium.user); - let error = new Error('Error fetching ID from compendium, ID already exists'); + let message = 'Error fetching ID from compendium, ID already exists'; + let error = new Error(message); + error.msg = message; error.status = 400; reject(error); } }); } else { debug('[%s] Invalid ID %s specified for compendium in %s', passon.id, compendiumID, passon.detectionFile); - let error = new Error('Invalid id found in compendium detection file'); + let message = 'Invalid id found in compendium detection file'; + let error = new Error(message); + error.msg = message; error.status = 400; reject(error); } From e9ce41335a79b713a97e119b7edf00f9053f31d9 Mon Sep 17 00:00:00 2001 From: LukasLohoff Date: Fri, 23 Mar 2018 16:17:54 +0100 Subject: [PATCH 08/20] delete compendia before each test --- test/direct_erc.js | 15 ++++++++++++++- test/sciebo_erc.js | 7 +++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/test/direct_erc.js b/test/direct_erc.js index e29196c..84a50a4 100644 --- a/test/direct_erc.js +++ b/test/direct_erc.js @@ -20,6 +20,7 @@ const assert = require('chai').assert; const request = require('request'); const fs = require('fs'); const config = require('../config/config'); +const mongojs = require('mongojs'); require("./setup"); const cookie_o2r = 's:C0LIrsxGtHOGHld8Nv2jedjL4evGgEHo.GMsWD5Vveq0vBt7/4rGeoH5Xx7Dd2pgZR9DvhKCyDTY'; @@ -28,7 +29,19 @@ const createCompendiumPostRequest = require('./util').createCompendiumPostReques describe('Direct upload of ERC', function () { + var db = mongojs('localhost/muncher', ['compendia']); + beforeEach(function(done) { + db.compendia.drop(function (err, doc) { + done(); + }); + }); + describe('POST /api/v1/compendium response with executable ERC', () => { + before(function(done) { + db.compendia.drop(function (err, doc) { + done(); + }); + }); it('should respond with HTTP 200 OK', (done) => { let req = createCompendiumPostRequest('./test/erc/executable', cookie_o2r); @@ -184,7 +197,7 @@ describe('Direct upload of ERC', function () { }).timeout(requestLoadingTimeout); it('should fail the second upload because bag ID is already used', (done) => { - let req = createCompendiumPostRequest('./test/erc/invalid_id', cookie_o2r); + let req = createCompendiumPostRequest('./test/erc/executable', cookie_o2r); request(req, (err, res, body) => { assert.ifError(err); diff --git a/test/sciebo_erc.js b/test/sciebo_erc.js index ae85e12..fd1e87c 100644 --- a/test/sciebo_erc.js +++ b/test/sciebo_erc.js @@ -19,6 +19,7 @@ const assert = require('chai').assert; const request = require('request'); const config = require('../config/config'); +const mongojs = require('mongojs'); require("./setup"); const cookie = 's:C0LIrsxGtHOGHld8Nv2jedjL4evGgEHo.GMsWD5Vveq0vBt7/4rGeoH5Xx7Dd2pgZR9DvhKCyDTY'; @@ -27,6 +28,12 @@ const requestLoadingTimeout = 20000; describe('Sciebo loader with compendia', function () { + var db = mongojs('localhost/muncher', ['compendia']); + beforeEach(function(done) { + db.compendia.drop(function (err, doc) { + done(); + }); + }); let compendium_id = ''; describe('create new compendium based on public WebDAV share with bagit.txt', () => { From d5d8146f3450462faac61e0f781c843e31e73ed8 Mon Sep 17 00:00:00 2001 From: LukasLohoff Date: Mon, 26 Mar 2018 16:59:16 +0200 Subject: [PATCH 09/20] update tests --- test/direct_erc.js | 27 ++++++++++----------------- test/sciebo_erc.js | 8 ++++---- test/zenodo.js | 16 +++++++++++----- 3 files changed, 25 insertions(+), 26 deletions(-) diff --git a/test/direct_erc.js b/test/direct_erc.js index 84a50a4..439b96e 100644 --- a/test/direct_erc.js +++ b/test/direct_erc.js @@ -37,11 +37,6 @@ describe('Direct upload of ERC', function () { }); describe('POST /api/v1/compendium response with executable ERC', () => { - before(function(done) { - db.compendia.drop(function (err, doc) { - done(); - }); - }); it('should respond with HTTP 200 OK', (done) => { let req = createCompendiumPostRequest('./test/erc/executable', cookie_o2r); @@ -72,13 +67,14 @@ describe('Direct upload of ERC', function () { }); }).timeout(requestLoadingTimeout); - it('should give a response including the id field', (done) => { + it('should give a response including the id specified in erc.yml', (done) => { let req = createCompendiumPostRequest('./test/erc/executable', cookie_o2r); request(req, (err, res, body) => { assert.ifError(err); assert.isDefined(JSON.parse(body).id, 'returned id'); assert.property(JSON.parse(body), 'id'); + assert.equal(JSON.parse(body).id, 'KIbebWnPlx'); done(); }); }).timeout(requestLoadingTimeout); @@ -187,23 +183,20 @@ describe('Direct upload of ERC', function () { describe('POST /api/v1/compendium with two compendia with the same ID', () => { it('should respond with HTTP 200 OK for the first compendium', (done) => { + let req = createCompendiumPostRequest('./test/erc/executable', cookie_o2r); request(req, (err, res, body) => { assert.ifError(err); assert.equal(res.statusCode, 200); - done(); - }); - }).timeout(requestLoadingTimeout); - - it('should fail the second upload because bag ID is already used', (done) => { - let req = createCompendiumPostRequest('./test/erc/executable', cookie_o2r); + let req2 = createCompendiumPostRequest('./test/erc/executable', cookie_o2r); - request(req, (err, res, body) => { - assert.ifError(err); - assert.equal(res.statusCode, 400); - assert.include(body, 'ID already exists'); - done(); + request(req2, (err, res, body) => { + assert.ifError(err); + assert.equal(res.statusCode, 400); + assert.include(body, 'ID already exists'); + done(); + }); }); }).timeout(requestLoadingTimeout); }); diff --git a/test/sciebo_erc.js b/test/sciebo_erc.js index fd1e87c..d8e815a 100644 --- a/test/sciebo_erc.js +++ b/test/sciebo_erc.js @@ -37,7 +37,7 @@ describe('Sciebo loader with compendia', function () { let compendium_id = ''; describe('create new compendium based on public WebDAV share with bagit.txt', () => { - it('should respond with a compendium ID', (done) => { + it('should respond with the compendium ID specified in erc.yml', (done) => { let form = { share_url: 'https://uni-muenster.sciebo.de/index.php/s/7Y7U4HC8GzJr5b9', content_type: 'compendium' @@ -58,7 +58,7 @@ describe('Sciebo loader with compendia', function () { assert.equal(res.statusCode, 200); assert.isObject(JSON.parse(body), 'returned JSON'); assert.isDefined(JSON.parse(body).id, 'returned id'); - compendium_id = JSON.parse(body).id; + assert.equal(JSON.parse(body).id, '6afbdbc29965'); done(); }); }).timeout(20000); @@ -96,7 +96,7 @@ describe('Sciebo loader with compendia', function () { assert.include(JSON.stringify(response), 'data/data/test2.Rmd'); assert.include(JSON.stringify(response), 'data/data/test.txt'); - assert.include(JSON.stringify(response), 'data/data/bagtainer.yml'); + assert.include(JSON.stringify(response), 'data/data/erc.yml'); done(); }); @@ -234,7 +234,7 @@ describe('Sciebo loader with compendia', function () { assert.include(JSON.stringify(response), 'data/data/document.Rmd'); assert.include(JSON.stringify(response), 'data/data/document.tex'); - assert.include(JSON.stringify(response), 'data/data/bagtainer.yml'); + assert.include(JSON.stringify(response), 'data/data/erc.yml'); done(); }); diff --git a/test/zenodo.js b/test/zenodo.js index 8986fb7..bf5b494 100644 --- a/test/zenodo.js +++ b/test/zenodo.js @@ -19,18 +19,24 @@ const assert = require('chai').assert; const request = require('request'); const config = require('../config/config'); +const mongojs = require('mongojs'); -require("./setup") +require("./setup"); const cookie = 's:C0LIrsxGtHOGHld8Nv2jedjL4evGgEHo.GMsWD5Vveq0vBt7/4rGeoH5Xx7Dd2pgZR9DvhKCyDTY'; const requestLoadingTimeout = 20000; describe('Zenodo loader', function () { - - var compendium_id = ''; + const db = mongojs('localhost/muncher', ['compendia']); + beforeEach(function(done) { + db.compendia.drop(function (err, doc) { + done(); + }); + }); + let compendium_id = ''; describe('create new compendium based on a zenodo record', () => { - it('zenodo record: should respond with a compendium ID', (done) => { + it('zenodo record: should respond with the compendium ID specified in erc.yml', (done) => { let form = { share_url: 'https://sandbox.zenodo.org/record/69114', content_type: 'compendium' @@ -51,7 +57,7 @@ describe('Zenodo loader', function () { assert.equal(res.statusCode, 200); assert.isObject(JSON.parse(body), 'returned JSON'); assert.isDefined(JSON.parse(body).id, 'returned id'); - compendium_id = JSON.parse(body).id; + assert.equal(JSON.parse(body).id, '66b173cb682d6'); done(); }); }).timeout(30000); From 936f29234e1f42f4fff82daef994b4044aea2cbe Mon Sep 17 00:00:00 2001 From: LukasLohoff Date: Mon, 26 Mar 2018 17:49:56 +0200 Subject: [PATCH 10/20] udpate controllers for id detection --- lib/loader.js | 3 +++ lib/steps.js | 36 +++++++++++++++++++++++++++++++++++- lib/uploader.js | 1 + 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/lib/loader.js b/lib/loader.js index 8cc5860..77fe975 100644 --- a/lib/loader.js +++ b/lib/loader.js @@ -55,6 +55,7 @@ function Loader(req, res) { .then(validateBag) .then(steps.detectCompendium) .then(steps.fetchCompendiumID) + .then(steps.moveCompendiumFiles) .then(steps.extractMetadata) .then(steps.loadMetadata) .then(steps.brokerMetadata) @@ -85,6 +86,7 @@ function Loader(req, res) { let passon = { id: randomstring.generate(config.id_length), + content: req.body.content_type, zenodoURL: encodeURI(this.req.body.share_url), zenodoID: this.req.params.zenodoID, baseURL: this.req.params.baseURL, @@ -102,6 +104,7 @@ function Loader(req, res) { .then(steps.checkEncoding) .then(steps.detectCompendium) .then(steps.fetchCompendiumID) + .then(steps.moveCompendiumFiles) .then(steps.extractMetadata) .then(steps.loadMetadata) .then(steps.brokerMetadata) diff --git a/lib/steps.js b/lib/steps.js index 229511e..cd77d1e 100644 --- a/lib/steps.js +++ b/lib/steps.js @@ -37,6 +37,7 @@ const Docker = require('dockerode'); const Stream = require('stream'); const meta = require('../lib/meta'); const yaml = require('js-yaml'); +const randomstring = require('randomstring'); var docker = new Docker(); @@ -640,12 +641,44 @@ function fetchCompendiumID(passon) { } } } else { - debug('[%s] Not an compendium, keeping generated ID', passon.id); + debug('[%s] Not a compendium, keeping generated ID', passon.id); fulfill(passon); } }); } +function moveCompendiumFiles(passon) { + return new Promise((fulfill, reject) => { + + if (passon.isCompendium){ + let updatedPath = path.join(config.fs.compendium, passon.id); + debug('[%s] Copying compendium files from %s to %s due to ID specified in compendium file', passon.id, passon.compendium_path, updatedPath); + + let cmd = 'mv ' + passon.compendium_path + ' ' + updatedPath; + exec(cmd, (error, stdout, stderr) => { + if (error || stderr) { + debug(error, stderr, stdout); + let errors = error.message.split(':'); + let message = errorMessageHelper(errors[errors.length - 1]); + error.msg = 'moving compendium files to new location failed: ' + message; + error.status = 500; + reject(error); + } else { + debug('[%s] Moving compendium files finished: %s', passon.id, stdout); + passon.compendium_path = updatedPath; + passon.detectionFile = path.join(passon.compendium_path, config.bagit.detectionFileName); + fulfill(passon); + } + }); + + } else { + debug('[%s] Not a compendium, files do not need to be moved', passon.id, ); + fulfill(passon); + } + + }); +} + function getTextFiles(passon) { return new Promise((fulfill, reject) => { debug('[%s] Getting text files', passon.id); @@ -977,6 +1010,7 @@ module.exports = { getTextFiles: getTextFiles, checkEncoding: checkEncoding, fetchCompendiumID : fetchCompendiumID, + moveCompendiumFiles : moveCompendiumFiles, extractMetadata: extractMetadata, loadMetadata: loadMetadata, brokerMetadata: brokerMetadata, diff --git a/lib/uploader.js b/lib/uploader.js index 0af3851..ce76114 100644 --- a/lib/uploader.js +++ b/lib/uploader.js @@ -49,6 +49,7 @@ function Uploader(req, res) { .then(validateBag) .then(steps.detectCompendium) .then(steps.fetchCompendiumID) + .then(steps.moveCompendiumFiles) .then(steps.extractMetadata) .then(steps.loadMetadata) .then(steps.brokerMetadata) From 398c02470a92561dc9a88aa70ecc429663ef7c41 Mon Sep 17 00:00:00 2001 From: LukasLohoff Date: Mon, 26 Mar 2018 17:54:52 +0200 Subject: [PATCH 11/20] fix tests --- test/direct_erc.js | 9 ++++++--- test/sciebo_erc.js | 20 +++++++++++++++++++- test/zenodo.js | 20 ++++++++++---------- 3 files changed, 35 insertions(+), 14 deletions(-) diff --git a/test/direct_erc.js b/test/direct_erc.js index 439b96e..de0a5fc 100644 --- a/test/direct_erc.js +++ b/test/direct_erc.js @@ -36,6 +36,10 @@ describe('Direct upload of ERC', function () { }); }); + after(function(done) { + db.close(); + }); + describe('POST /api/v1/compendium response with executable ERC', () => { it('should respond with HTTP 200 OK', (done) => { let req = createCompendiumPostRequest('./test/erc/executable', cookie_o2r); @@ -126,9 +130,8 @@ describe('Direct upload of ERC', function () { let response = JSON.parse(body); assert.property(response, 'metadata'); assert.property(response.metadata, 'o2r'); - assert.property(response.metadata.o2r, 'ercIdentifier'); - //assert.propertyVal(response.metadata.o2r, 'ercIdentifier', 'KIbebWnPlx'); - //assert.propertyVal(response.metadata.o2r, 'title', 'This is the title: it contains a colon'); + assert.property(response.metadata.o2r, 'publication_date'); + assert.propertyVal(response.metadata.id, 'KIbebWnPlx'); done(); }); }); diff --git a/test/sciebo_erc.js b/test/sciebo_erc.js index d8e815a..f2e2f80 100644 --- a/test/sciebo_erc.js +++ b/test/sciebo_erc.js @@ -20,6 +20,8 @@ const assert = require('chai').assert; const request = require('request'); const config = require('../config/config'); const mongojs = require('mongojs'); +const path = require('path'); +const exec = require('child_process').exec; require("./setup"); const cookie = 's:C0LIrsxGtHOGHld8Nv2jedjL4evGgEHo.GMsWD5Vveq0vBt7/4rGeoH5Xx7Dd2pgZR9DvhKCyDTY'; @@ -29,11 +31,27 @@ const requestLoadingTimeout = 20000; describe('Sciebo loader with compendia', function () { var db = mongojs('localhost/muncher', ['compendia']); + beforeEach(function(done) { + // 1. Delete database compendium collection db.compendia.drop(function (err, doc) { - done(); + // 2. Delete compendium files + let cmd = 'rm -rf ' + path.join(config.fs.compendium, '*'); + exec(cmd, (error, stdout, stderr) => { + if (error || stderr) { + console.log(error); + } else { + done(); + } + }); }); }); + + after(function (done) { + db.close(); + done(); + }); + let compendium_id = ''; describe('create new compendium based on public WebDAV share with bagit.txt', () => { diff --git a/test/zenodo.js b/test/zenodo.js index bf5b494..850be15 100644 --- a/test/zenodo.js +++ b/test/zenodo.js @@ -38,7 +38,7 @@ describe('Zenodo loader', function () { describe('create new compendium based on a zenodo record', () => { it('zenodo record: should respond with the compendium ID specified in erc.yml', (done) => { let form = { - share_url: 'https://sandbox.zenodo.org/record/69114', + share_url: 'https://sandbox.zenodo.org/record/212213', content_type: 'compendium' }; @@ -64,7 +64,7 @@ describe('Zenodo loader', function () { it('zenodo record, additional "filename" parameter: should respond with a compendium ID', (done) => { let form = { - share_url: 'https://sandbox.zenodo.org/record/69114', + share_url: 'https://sandbox.zenodo.org/record/212213', filename: 'metatainer.zip', content_type: 'compendium' }; @@ -91,7 +91,7 @@ describe('Zenodo loader', function () { it('zenodo record, "doi" parameter: should respond with a compendium ID', (done) => { let form = { - doi: '10.5072/zenodo.69114', // intenally taken apart and using sandbox + doi: '10.5072/zenodo.212213', // intenally taken apart and using sandbox content_type: 'compendium' }; @@ -117,7 +117,7 @@ describe('Zenodo loader', function () { it('zenodo record, "doi.org" as "share_url": should respond with a compendium ID', (done) => { let form = { - share_url: 'http://doi.org/10.5072/zenodo.69114', // page not found, internally using only id in sandbox.zenodo.org + share_url: 'http://doi.org/10.5072/zenodo.212213', // page not found, internally using only id in sandbox.zenodo.org filename: 'metatainer.zip', content_type: 'compendium' }; @@ -144,7 +144,7 @@ describe('Zenodo loader', function () { it('zenodo record, "zenodo_record_id" parameter: should respond with a compendium ID', (done) => { let form = { - zenodo_record_id: '69114', + zenodo_record_id: '212213', content_type: 'compendium' }; @@ -172,7 +172,7 @@ describe('Zenodo loader', function () { describe('No new compendium based on invalid parameters', () => { it('invalid zenodo URL: should respond with an error 422', (done) => { let form = { - share_url: 'htts?///sandbox.zenodo.org/record/69114', + share_url: 'htts?///sandbox.zenodo.org/record/212213', filename: 'metatainer.zip', content_type: 'compendium', }; @@ -198,7 +198,7 @@ describe('Zenodo loader', function () { it('host not allowed (not a zenodo record): should respond with an error 403', (done) => { let form = { - share_url: 'https://sandbox.ODONEZ.org/record/69114', + share_url: 'https://sandbox.ODONEZ.org/record/212213', filename: 'metatainer.zip', content_type: 'compendium', }; @@ -299,7 +299,7 @@ describe('Zenodo loader', function () { it('filename not found: should respond with an error 500', (done) => { let form = { - share_url: 'https://sandbox.zenodo.org/record/69114', + share_url: 'https://sandbox.zenodo.org/record/212213', filename: 'not_existing_file.xyz', content_type: 'compendium', }; @@ -323,7 +323,7 @@ describe('Zenodo loader', function () { it('filename not found: should respond with a useful but not talkative error message', (done) => { let form = { - share_url: 'https://sandbox.zenodo.org/record/69114', + share_url: 'https://sandbox.zenodo.org/record/212213', filename: 'not_existing_file.xyz', content_type: 'compendium', }; @@ -343,7 +343,7 @@ describe('Zenodo loader', function () { let response = JSON.parse(body); assert.isUndefined(response.id, 'returned no id'); assert.include(response.error, 'download failed:'); - assert.include(response.error, 'sandbox.zenodo.org/record/69114/files/not_existing_file.xyz'); + assert.include(response.error, 'sandbox.zenodo.org/record/212213/files/not_existing_file.xyz'); assert.notInclude(JSON.stringify(response.error), config.fs.base); done(); }); From abdc520d5be7b21bffaf8edd23b7fdc04d53db3c Mon Sep 17 00:00:00 2001 From: LukasLohoff Date: Tue, 27 Mar 2018 12:46:14 +0200 Subject: [PATCH 12/20] delete compendium database and storage before tests --- test/direct_erc.js | 20 ++++++++++++++++---- test/sciebo_erc.js | 2 +- test/sciebo_workspace.js | 2 +- test/zenodo.js | 18 +++++++++++++++++- 4 files changed, 35 insertions(+), 7 deletions(-) diff --git a/test/direct_erc.js b/test/direct_erc.js index de0a5fc..8590a54 100644 --- a/test/direct_erc.js +++ b/test/direct_erc.js @@ -21,6 +21,8 @@ const request = require('request'); const fs = require('fs'); const config = require('../config/config'); const mongojs = require('mongojs'); +const path = require('path'); +const exec = require('child_process').exec; require("./setup"); const cookie_o2r = 's:C0LIrsxGtHOGHld8Nv2jedjL4evGgEHo.GMsWD5Vveq0vBt7/4rGeoH5Xx7Dd2pgZR9DvhKCyDTY'; @@ -31,13 +33,23 @@ const createCompendiumPostRequest = require('./util').createCompendiumPostReques describe('Direct upload of ERC', function () { var db = mongojs('localhost/muncher', ['compendia']); beforeEach(function(done) { + // 1. Delete database compendium collection db.compendia.drop(function (err, doc) { - done(); + // 2. Delete compendium files + let cmd = 'rm -rf ' + path.join(config.fs.compendium, '*'); + exec(cmd, (error, stdout, stderr) => { + if (error || stderr) { + assert.ifError(error); + } else { + done(); + } + }); }); }); - after(function(done) { - db.close(); + after(function (done) { + db.close(); + done(); }); describe('POST /api/v1/compendium response with executable ERC', () => { @@ -131,7 +143,7 @@ describe('Direct upload of ERC', function () { assert.property(response, 'metadata'); assert.property(response.metadata, 'o2r'); assert.property(response.metadata.o2r, 'publication_date'); - assert.propertyVal(response.metadata.id, 'KIbebWnPlx'); + assert.propertyVal(response, 'id', 'KIbebWnPlx'); done(); }); }); diff --git a/test/sciebo_erc.js b/test/sciebo_erc.js index f2e2f80..4cec51b 100644 --- a/test/sciebo_erc.js +++ b/test/sciebo_erc.js @@ -39,7 +39,7 @@ describe('Sciebo loader with compendia', function () { let cmd = 'rm -rf ' + path.join(config.fs.compendium, '*'); exec(cmd, (error, stdout, stderr) => { if (error || stderr) { - console.log(error); + assert.ifError(error); } else { done(); } diff --git a/test/sciebo_workspace.js b/test/sciebo_workspace.js index c2c7afa..776a02b 100644 --- a/test/sciebo_workspace.js +++ b/test/sciebo_workspace.js @@ -20,7 +20,7 @@ const assert = require('chai').assert; const request = require('request'); const config = require('../config/config'); -require("./setup") +require("./setup"); const cookie = 's:C0LIrsxGtHOGHld8Nv2jedjL4evGgEHo.GMsWD5Vveq0vBt7/4rGeoH5Xx7Dd2pgZR9DvhKCyDTY'; const cookie_plain = 's:yleQfdYnkh-sbj9Ez--_TWHVhXeXNEgq.qRmINNdkRuJ+iHGg5woRa9ydziuJ+DzFG9GnAZRvaaM'; const requestLoadingTimeout = 20000; diff --git a/test/zenodo.js b/test/zenodo.js index 850be15..65f8f91 100644 --- a/test/zenodo.js +++ b/test/zenodo.js @@ -20,6 +20,8 @@ const assert = require('chai').assert; const request = require('request'); const config = require('../config/config'); const mongojs = require('mongojs'); +const path = require('path'); +const exec = require('child_process').exec; require("./setup"); const cookie = 's:C0LIrsxGtHOGHld8Nv2jedjL4evGgEHo.GMsWD5Vveq0vBt7/4rGeoH5Xx7Dd2pgZR9DvhKCyDTY'; @@ -29,10 +31,24 @@ const requestLoadingTimeout = 20000; describe('Zenodo loader', function () { const db = mongojs('localhost/muncher', ['compendia']); beforeEach(function(done) { + // 1. Delete database compendium collection db.compendia.drop(function (err, doc) { - done(); + // 2. Delete compendium files + let cmd = 'rm -rf ' + path.join(config.fs.compendium, '*'); + exec(cmd, (error, stdout, stderr) => { + if (error || stderr) { + assert.ifError(error); + } else { + done(); + } + }); }); }); + + after(function (done) { + db.close(); + done(); + }); let compendium_id = ''; describe('create new compendium based on a zenodo record', () => { From b808aced0c15f03b2ca5267b2a49747d01ba76f3 Mon Sep 17 00:00:00 2001 From: LukasLohoff Date: Tue, 27 Mar 2018 14:50:33 +0200 Subject: [PATCH 13/20] update slack dependency to 11.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1c1cd7d..3fce097 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "recursive-readdir": "^2.2.1", "request": "^2.83.0", "response-time": "^2.3.2", - "slack": "^9.0.9", + "slack": "^11.0.0", "util": "^0.10.3", "validator": "^9.1.1", "webdav": "^1.1.0", From 6311e26931826ab238676b2bbe48b71c4592219c Mon Sep 17 00:00:00 2001 From: LukasLohoff Date: Tue, 27 Mar 2018 14:58:11 +0200 Subject: [PATCH 14/20] increase major version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 3fce097..cf94ebc 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "o2r-loader", - "version": "0.10.1", + "version": "0.11.0", "description": "Node.js implementation for loading compendia and workspaces from external repositories for the [o2r web api](http://o2r.info/o2r-web-api).", "main": "index.js", "scripts": { From ac63a6ba9225426ffa5648e35a0991d1034e9dce Mon Sep 17 00:00:00 2001 From: LukasLohoff Date: Tue, 27 Mar 2018 15:11:35 +0200 Subject: [PATCH 15/20] rename ERC detectionFile to configurationFile --- config/config.js | 2 +- lib/steps.js | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/config/config.js b/config/config.js index 274d472..0ce136b 100644 --- a/config/config.js +++ b/config/config.js @@ -74,7 +74,7 @@ c.user.level.create_compendium = 100; // compendium configuration c.compendium = {}; c.compendium.supportedContentTypes = ["compendium", "workspace"]; -c.compendium.detectionFileName = 'erc.yml'; +c.compendium.configurationFileName = 'erc.yml'; c.compendium.supportedVersions = ['0.1', '1']; c.bagit = {}; diff --git a/lib/steps.js b/lib/steps.js index cd77d1e..a468310 100644 --- a/lib/steps.js +++ b/lib/steps.js @@ -551,20 +551,20 @@ function detectCompendium(passon) { debug('[%s] Detecting compendium', passon.id); if (passon.isBag) { - passon.detectionFile = path.join(passon.compendium_path, config.bagit.payloadDirectory, config.compendium.detectionFileName); + passon.configurationFile = path.join(passon.compendium_path, config.bagit.payloadDirectory, config.compendium.configurationFileName); } else { - passon.detectionFile = path.join(passon.compendium_path, config.compendium.detectionFileName); + passon.configurationFile = path.join(passon.compendium_path, config.compendium.configurationFileName); } try { - fs.accessSync(passon.detectionFile); + fs.accessSync(passon.configurationFile); - debug('[%s] Found %s - it\'s a compendium! Content was labeled as "%s"', passon.id, passon.detectionFile, passon.content); + debug('[%s] Found %s - it\'s a compendium! Content was labeled as "%s"', passon.id, passon.configurationFile, passon.content); passon.isCompendium = true; fulfill(passon); } catch (error) { if (passon.content === 'compendium') { - debug('[%s] Could not find compendium detection file but content is %s: %s', passon.id, passon.content, error); + debug('[%s] Could not find compendium configuration file but content is %s: %s', passon.id, passon.content, error); error.msg = 'content_type is ' + passon.content + ' but no compendium provided.'; error.status = 400; @@ -580,18 +580,18 @@ function detectCompendium(passon) { function fetchCompendiumID(passon) { return new Promise((fulfill, reject) => { if (passon.isCompendium) { - debug('[%s] Reading compendium ID from %s', passon.id, config.compendium.detectionFileName); + debug('[%s] Reading compendium ID from %s', passon.id, config.compendium.configurationFileName); let compendiumID = null; // Get compendium yml, or throw exception on error try { - let doc = yaml.safeLoad(fs.readFileSync(passon.detectionFile, 'utf8')); + let doc = yaml.safeLoad(fs.readFileSync(passon.configurationFile, 'utf8')); compendiumID = doc.id; debug('[%s] Successfully read compendium metadata: %O', passon.id, doc); } catch (error) { - debug('[%s] Could not read compendium detection file %s: %s', passon.id, passon.detectionFile, error); + debug('[%s] Could not read compendium detection file %s: %s', passon.id, passon.configurationFile, error); error.msg = 'Could not read compendium detection file'; error.status = 400; reject(error); @@ -632,7 +632,7 @@ function fetchCompendiumID(passon) { } }); } else { - debug('[%s] Invalid ID %s specified for compendium in %s', passon.id, compendiumID, passon.detectionFile); + debug('[%s] Invalid ID %s specified for compendium in %s', passon.id, compendiumID, passon.configurationFile); let message = 'Invalid id found in compendium detection file'; let error = new Error(message); error.msg = message; @@ -666,7 +666,7 @@ function moveCompendiumFiles(passon) { } else { debug('[%s] Moving compendium files finished: %s', passon.id, stdout); passon.compendium_path = updatedPath; - passon.detectionFile = path.join(passon.compendium_path, config.bagit.detectionFileName); + passon.configurationFile = path.join(passon.compendium_path, config.bagit.detectionFileName); fulfill(passon); } }); From 85a1b3cc07511bfe2df76ca806bc9d4f0021e9d4 Mon Sep 17 00:00:00 2001 From: LukasLohoff Date: Tue, 27 Mar 2018 15:52:52 +0200 Subject: [PATCH 16/20] update steps.js with PR suggestions by @nuest --- lib/steps.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/steps.js b/lib/steps.js index a468310..300eb82 100644 --- a/lib/steps.js +++ b/lib/steps.js @@ -19,7 +19,6 @@ const config = require('../config/config'); const debug = require('debug')('loader'); const exec = require('child_process').exec; const errorMessageHelper = require('../lib/error-message'); -const nodemailer = require('nodemailer'); const Compendium = require('../lib/model/compendium'); const path = require('path'); @@ -37,7 +36,6 @@ const Docker = require('dockerode'); const Stream = require('stream'); const meta = require('../lib/meta'); const yaml = require('js-yaml'); -const randomstring = require('randomstring'); var docker = new Docker(); @@ -606,8 +604,7 @@ function fetchCompendiumID(passon) { reject(error); } else { // Validate compendium id - let idRegex = new RegExp(config.bagtainer.id_regex); - if (idRegex.test(compendiumID)) { + if (config.bagtainer.id_regex.test(compendiumID)) { // Check if id found in compendium file already exists Compendium.findOne({ id: compendiumID }).select('id user').exec((err, compendium) => { @@ -619,7 +616,7 @@ function fetchCompendiumID(passon) { reject(err); } if (compendium === null) { - debug('[%s] Assigned new ID %s to compendium', passon.id, compendiumID); + debug('[%s] Assigned ID %s from configuration file to compendium', passon.id, compendiumID); passon.id = compendiumID; fulfill(passon); } else { @@ -657,6 +654,7 @@ function moveCompendiumFiles(passon) { let cmd = 'mv ' + passon.compendium_path + ' ' + updatedPath; exec(cmd, (error, stdout, stderr) => { if (error || stderr) { + debug('Error copying compendium files: %O', {error, stderr, stdout}); debug(error, stderr, stdout); let errors = error.message.split(':'); let message = errorMessageHelper(errors[errors.length - 1]); From 645f6437c5a60c76618a3c930daeed95c1d37b9c Mon Sep 17 00:00:00 2001 From: LukasLohoff Date: Tue, 27 Mar 2018 16:55:13 +0200 Subject: [PATCH 17/20] fix dockerfile --- Dockerfile | 30 ++++++++---------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/Dockerfile b/Dockerfile index eb8b2ed..00a73cd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,33 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. # -FROM node:8-alpine -# FROM alpine:3.6 does not work because of https://github.com/sonicdoe/detect-character-encoding/issues/8 +FROM node:8-slim +# FROM alpine:3.6 and node:8-alpine does not work because of https://github.com/sonicdoe/detect-character-encoding/issues/8 # Python, based on frolvlad/alpine-python3 -RUN apk add --no-cache \ - python2 \ - && python2 -m ensurepip \ - && rm -r /usr/lib/python*/ensurepip \ - && pip install --upgrade pip setuptools setuptools_scm \ - && if [ ! -e /usr/bin/pip ]; then ln -s pip /usr/bin/pip ; fi \ - && if [[ ! -e /usr/bin/python ]]; then ln -sf /usr/bin/python2 /usr/bin/python; fi \ - && rm -r /root/.cache - -# Add Alpine mirrors, replacing default repositories with edge ones, based on https://github.com/jfloff/alpine-python/blob/master/3.4/Dockerfile -RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" > /etc/apk/repositories \ - && echo "http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \ - && echo "http://dl-cdn.alpinelinux.org/alpine/edge/main" >> /etc/apk/repositories - -RUN apk add --no-cache \ +RUN apt-get update && apt-get install -y \ + python \ + python-pip \ unzip \ - dumb-init \ - # needed for loading from external http(s) sources - wget \ - openssl \ # needed for npm install gyp make \ g++ \ + && wget https://github.com/Yelp/dumb-init/releases/download/v1.2.1/dumb-init_1.2.1_amd64.deb \ + && dpkg -i dumb-init_*.deb \ && pip install bagit # Install app @@ -46,7 +32,7 @@ WORKDIR /loader COPY package.json package.json RUN npm install --production -RUN apk del \ +RUN apt-get purge -y \ make \ g++ \ && rm -rf /var/cache From ed3203a15a66383b158f30f12d15beb720f6e6ea Mon Sep 17 00:00:00 2001 From: LukasLohoff Date: Wed, 28 Mar 2018 15:09:24 +0200 Subject: [PATCH 18/20] fix permission issue when deleting compedia --- test/direct_erc.js | 34 +++++++++++++++++++++++----------- test/sciebo_erc.js | 31 +++++++++++++++++++++---------- test/zenodo.js | 33 +++++++++++++++++++++++---------- 3 files changed, 67 insertions(+), 31 deletions(-) diff --git a/test/direct_erc.js b/test/direct_erc.js index 8590a54..fb39b0c 100644 --- a/test/direct_erc.js +++ b/test/direct_erc.js @@ -18,9 +18,10 @@ /* eslint-env mocha */ const assert = require('chai').assert; const request = require('request'); -const fs = require('fs'); const config = require('../config/config'); const mongojs = require('mongojs'); +const fs = require('fs-extra'); +const env = process.env; const path = require('path'); const exec = require('child_process').exec; @@ -30,21 +31,32 @@ const requestLoadingTimeout = 15000; const createCompendiumPostRequest = require('./util').createCompendiumPostRequest; -describe('Direct upload of ERC', function () { +describe.only('Direct upload of ERC', function () { var db = mongojs('localhost/muncher', ['compendia']); + beforeEach(function(done) { // 1. Delete database compendium collection - db.compendia.drop(function (err, doc) { - // 2. Delete compendium files - let cmd = 'rm -rf ' + path.join(config.fs.compendium, '*'); - exec(cmd, (error, stdout, stderr) => { - if (error || stderr) { - assert.ifError(error); - } else { + if(env.TRAVIS === "true") { + db.compendia.drop(function (err, doc) { + // 2. Delete compendium files + let cmd = 'docker exec testloader rm -rf ' + path.join(config.fs.compendium, '*'); + exec(cmd, (error, stdout, stderr) => { + if (error || stderr) { + assert.ifError(error); + } else { + done(); + } + }); + }); + } else { + db.compendia.drop(function (err, doc) { + // 2. Delete compendium files + fs.emptyDir(config.fs.compendium, err => { + if (err) assert.ifError(err); done(); - } + }); }); - }); + } }); after(function (done) { diff --git a/test/sciebo_erc.js b/test/sciebo_erc.js index 4cec51b..f138bc9 100644 --- a/test/sciebo_erc.js +++ b/test/sciebo_erc.js @@ -20,6 +20,8 @@ const assert = require('chai').assert; const request = require('request'); const config = require('../config/config'); const mongojs = require('mongojs'); +const fs = require('fs-extra'); +const env = process.env; const path = require('path'); const exec = require('child_process').exec; @@ -28,23 +30,32 @@ const cookie = 's:C0LIrsxGtHOGHld8Nv2jedjL4evGgEHo.GMsWD5Vveq0vBt7/4rGeoH5Xx7Dd2 const cookie_plain = 's:yleQfdYnkh-sbj9Ez--_TWHVhXeXNEgq.qRmINNdkRuJ+iHGg5woRa9ydziuJ+DzFG9GnAZRvaaM'; const requestLoadingTimeout = 20000; - describe('Sciebo loader with compendia', function () { var db = mongojs('localhost/muncher', ['compendia']); beforeEach(function(done) { // 1. Delete database compendium collection - db.compendia.drop(function (err, doc) { - // 2. Delete compendium files - let cmd = 'rm -rf ' + path.join(config.fs.compendium, '*'); - exec(cmd, (error, stdout, stderr) => { - if (error || stderr) { - assert.ifError(error); - } else { + if(env.TRAVIS === "true") { + db.compendia.drop(function (err, doc) { + // 2. Delete compendium files + let cmd = 'docker exec testloader rm -rf ' + path.join(config.fs.compendium, '*'); + exec(cmd, (error, stdout, stderr) => { + if (error || stderr) { + assert.ifError(error); + } else { + done(); + } + }); + }); + } else { + db.compendia.drop(function (err, doc) { + // 2. Delete compendium files + fs.emptyDir(config.fs.compendium, err => { + if (err) assert.ifError(err); done(); - } + }); }); - }); + } }); after(function (done) { diff --git a/test/zenodo.js b/test/zenodo.js index 65f8f91..68f81fb 100644 --- a/test/zenodo.js +++ b/test/zenodo.js @@ -20,6 +20,8 @@ const assert = require('chai').assert; const request = require('request'); const config = require('../config/config'); const mongojs = require('mongojs'); +const fs = require('fs-extra'); +const env = process.env; const path = require('path'); const exec = require('child_process').exec; @@ -29,20 +31,31 @@ const requestLoadingTimeout = 20000; describe('Zenodo loader', function () { - const db = mongojs('localhost/muncher', ['compendia']); + var db = mongojs('localhost/muncher', ['compendia']); + beforeEach(function(done) { // 1. Delete database compendium collection - db.compendia.drop(function (err, doc) { - // 2. Delete compendium files - let cmd = 'rm -rf ' + path.join(config.fs.compendium, '*'); - exec(cmd, (error, stdout, stderr) => { - if (error || stderr) { - assert.ifError(error); - } else { + if(env.TRAVIS === "true") { + db.compendia.drop(function (err, doc) { + // 2. Delete compendium files + let cmd = 'docker exec testloader rm -rf ' + path.join(config.fs.compendium, '*'); + exec(cmd, (error, stdout, stderr) => { + if (error || stderr) { + assert.ifError(error); + } else { + done(); + } + }); + }); + } else { + db.compendia.drop(function (err, doc) { + // 2. Delete compendium files + fs.emptyDir(config.fs.compendium, err => { + if (err) assert.ifError(err); done(); - } + }); }); - }); + } }); after(function (done) { From 4d11837048113414f2e2ec3a829b8e6c2025f228 Mon Sep 17 00:00:00 2001 From: LukasLohoff Date: Wed, 28 Mar 2018 15:09:56 +0200 Subject: [PATCH 19/20] remove describe.only --- test/direct_erc.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/direct_erc.js b/test/direct_erc.js index fb39b0c..1188d65 100644 --- a/test/direct_erc.js +++ b/test/direct_erc.js @@ -31,7 +31,7 @@ const requestLoadingTimeout = 15000; const createCompendiumPostRequest = require('./util').createCompendiumPostRequest; -describe.only('Direct upload of ERC', function () { +describe('Direct upload of ERC', function () { var db = mongojs('localhost/muncher', ['compendia']); beforeEach(function(done) { From c31a666ac71716742c626f14cbb514d3ee7de04c Mon Sep 17 00:00:00 2001 From: LukasLohoff Date: Wed, 28 Mar 2018 15:20:15 +0200 Subject: [PATCH 20/20] inc loader timeout for sciebo --- test/sciebo_erc.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/sciebo_erc.js b/test/sciebo_erc.js index f138bc9..88978ee 100644 --- a/test/sciebo_erc.js +++ b/test/sciebo_erc.js @@ -28,7 +28,7 @@ const exec = require('child_process').exec; require("./setup"); const cookie = 's:C0LIrsxGtHOGHld8Nv2jedjL4evGgEHo.GMsWD5Vveq0vBt7/4rGeoH5Xx7Dd2pgZR9DvhKCyDTY'; const cookie_plain = 's:yleQfdYnkh-sbj9Ez--_TWHVhXeXNEgq.qRmINNdkRuJ+iHGg5woRa9ydziuJ+DzFG9GnAZRvaaM'; -const requestLoadingTimeout = 20000; +const requestLoadingTimeout = 30000; describe('Sciebo loader with compendia', function () { var db = mongojs('localhost/muncher', ['compendia']);