Use id from erc.yml

Dockerfile

@@ -12,41 +12,27 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-FROM node:8-alpine
-# FROM alpine:3.6 does not work because of https://github.com/sonicdoe/detect-character-encoding/issues/8
+FROM node:8-slim
+# FROM alpine:3.6 and node:8-alpine does not work because of https://github.com/sonicdoe/detect-character-encoding/issues/8
 
 # Python, based on frolvlad/alpine-python3
-RUN apk add --no-cache \
-  python2 \
-  && python2 -m ensurepip \
-  && rm -r /usr/lib/python*/ensurepip \
-  && pip install --upgrade pip setuptools setuptools_scm \
-  && if [ ! -e /usr/bin/pip ]; then ln -s pip /usr/bin/pip ; fi \
-  && if [[ ! -e /usr/bin/python ]]; then ln -sf /usr/bin/python2 /usr/bin/python; fi \
-  && rm -r /root/.cache
-
-# Add Alpine mirrors, replacing default repositories with edge ones, based on https://github.com/jfloff/alpine-python/blob/master/3.4/Dockerfile
-RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" > /etc/apk/repositories \
-  && echo "http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \
-  && echo "http://dl-cdn.alpinelinux.org/alpine/edge/main" >> /etc/apk/repositories
-
-RUN apk add --no-cache \
+RUN apt-get update && apt-get install -y \
+    python \
+    python-pip \
     unzip \
-    dumb-init \
-    # needed for loading from external http(s) sources
-    wget \
-    openssl \
     # needed for npm install gyp
     make \
     g++ \
+  && wget https://github.com/Yelp/dumb-init/releases/download/v1.2.1/dumb-init_1.2.1_amd64.deb \
+  && dpkg -i dumb-init_*.deb \
   && pip install bagit
 
 # Install app
 WORKDIR /loader
 COPY package.json package.json
 RUN npm install --production
 
-RUN apk del \
+RUN apt-get purge -y \
   make \
   g++ \
   && rm -rf /var/cache

README.md

@@ -106,6 +106,17 @@ TEST_HOST=http://localhost:80 npm test
 ```
 For this, the loader has to run locally or as part of a docker-compose configuration.
 
+### Public shares
+
+The tests for public shares (`sciebo_erc.js`, `sciebo_workspace.js` and `zenodo.js`) use ERC uploaded to the respective services.
+
+They can be found at
+
+* Sciebo: [public link](https://uni-muenster.sciebo.de/index.php/s/h5tNYXsS1Bsv4qr) | [private link](https://uni-muenster.sciebo.de/f/749265161)
+* Zenodo: https://sandbox.zenodo.org/deposit/69114
+
+For information on which share URL belongs to which compendium, see the file `README` in the [`integration_test_shares`](https://uni-muenster.sciebo.de/index.php/s/h5tNYXsS1Bsv4qr) folder.
+
 ## Dockerfile
 
 The file `Dockerfile` describes the Docker image published at [Docker Hub](https://hub.docker.com/r/o2rproject/o2r-loader/).

config/config.js

@@ -74,7 +74,7 @@ c.user.level.create_compendium = 100;
 // compendium configuration
 c.compendium = {};
 c.compendium.supportedContentTypes = ["compendium", "workspace"];
-c.compendium.detectionFileName = 'erc.yml';
+c.compendium.configurationFileName = 'erc.yml';
 c.compendium.supportedVersions = ['0.1', '1'];
 
 c.bagit = {};
@@ -84,6 +84,9 @@ c.bagit.validation = {};
 c.bagit.validation.fast = false;
 c.bagit.validation.failUpload = true;
 
+c.bagtainer = {};
+c.bagtainer.id_regex = /^[^-_.\n\r][a-zA-Z0-9\._-]*[^-_.\n\r]$/;
+
 // metadata extraction options
 c.meta = {};
 c.meta.container = {};

lib/loader.js

@@ -53,6 +53,9 @@ function Loader(req, res) {
             .then(steps.checkEncoding)
             .then(steps.detectBag)
             .then(validateBag)
+            .then(steps.detectCompendium)
+            .then(steps.fetchCompendiumID)
+            .then(steps.moveCompendiumFiles)
             .then(steps.extractMetadata)
             .then(steps.loadMetadata)
             .then(steps.brokerMetadata)
@@ -83,6 +86,7 @@ function Loader(req, res) {
 
         let passon = {
             id: randomstring.generate(config.id_length),
+            content: req.body.content_type,
             zenodoURL: encodeURI(this.req.body.share_url),
             zenodoID: this.req.params.zenodoID,
             baseURL: this.req.params.baseURL,
@@ -98,6 +102,9 @@ function Loader(req, res) {
             .then(steps.detectBag)
             .then(steps.getTextFiles)
             .then(steps.checkEncoding)
+            .then(steps.detectCompendium)
+            .then(steps.fetchCompendiumID)
+            .then(steps.moveCompendiumFiles)
             .then(steps.extractMetadata)
             .then(steps.loadMetadata)
             .then(steps.brokerMetadata)

lib/steps.js

@@ -19,7 +19,6 @@ const config = require('../config/config');
 const debug = require('debug')('loader');
 const exec = require('child_process').exec;
 const errorMessageHelper = require('../lib/error-message');
-const nodemailer = require('nodemailer');
 
 const Compendium = require('../lib/model/compendium');
 const path = require('path');
@@ -36,6 +35,7 @@ const util = require('util');
 const Docker = require('dockerode');
 const Stream = require('stream');
 const meta = require('../lib/meta');
+const yaml = require('js-yaml');
 
 var docker = new Docker();
 
@@ -548,22 +548,21 @@ function detectCompendium(passon) {
     return new Promise((fulfill, reject) => {
         debug('[%s] Detecting compendium', passon.id);
 
-        let detectionFile = null;
         if (passon.isBag) {
-            detectionFile = path.join(passon.compendium_path, config.bagit.payloadDirectory, config.compendium.detectionFileName);
+            passon.configurationFile = path.join(passon.compendium_path, config.bagit.payloadDirectory, config.compendium.configurationFileName);
         } else {
-            detectionFile = path.join(passon.compendium_path, config.compendium.detectionFileName);
+            passon.configurationFile = path.join(passon.compendium_path, config.compendium.configurationFileName);
         }
 
         try {
-            fs.accessSync(detectionFile);
+            fs.accessSync(passon.configurationFile);
 
-            debug('[%s] Found %s - it\'s a compendium! Content was labeled as "%s"', passon.id, detectionFile, passon.content);
+            debug('[%s] Found %s - it\'s a compendium! Content was labeled as "%s"', passon.id, passon.configurationFile, passon.content);
             passon.isCompendium = true;
             fulfill(passon);
         } catch (error) {
             if (passon.content === 'compendium') {
-                debug('[%s] Could not find compendium detection file but content is %s: %s', passon.id, passon.content, error);
+                debug('[%s] Could not find compendium configuration file but content is %s: %s', passon.id, passon.content, error);
 
                 error.msg = 'content_type is ' + passon.content + ' but no compendium provided.';
                 error.status = 400;
@@ -576,6 +575,108 @@ function detectCompendium(passon) {
     });
 }
 
+function fetchCompendiumID(passon) {
+    return new Promise((fulfill, reject) => {
+        if (passon.isCompendium) {
+            debug('[%s] Reading compendium ID from %s', passon.id, config.compendium.configurationFileName);
+
+            let compendiumID = null;
+
+            // Get compendium yml, or throw exception on error
+            try {
+                let doc = yaml.safeLoad(fs.readFileSync(passon.configurationFile, 'utf8'));
+                compendiumID = doc.id;
+                debug('[%s] Successfully read compendium metadata: %O', passon.id, doc);
+
+            } catch (error) {
+                debug('[%s] Could not read compendium detection file %s: %s', passon.id, passon.configurationFile, error);
+                error.msg = 'Could not read compendium detection file';
+                error.status = 400;
+                reject(error);
+            }
+
+            if (!compendiumID){
+                debug('[%s] No ID specified for compendium', passon.id);
+                let message = 'No id found in compendium detection file';
+                let error = new Error(message);
+                error.msg = message;
+                error.status = 400;
+                reject(error);
+            } else {
+                // Validate compendium id
+                if (config.bagtainer.id_regex.test(compendiumID)) {
+
+                    // Check if id found in compendium file already exists
+                    Compendium.findOne({ id: compendiumID }).select('id user').exec((err, compendium) => {
+                        // eslint-disable-next-line no-eq-null, eqeqeq
+                        if (err) {
+                            debug('[%s] Error querying compendium with id %s', passon.id, compendiumID);
+                            err.message = 'Error querying compendium';
+                            err.status = 400;
+                            reject(err);
+                        }
+                        if (compendium === null) {
+                            debug('[%s] Assigned ID %s from configuration file to compendium', passon.id, compendiumID);
+                            passon.id = compendiumID;
+                            fulfill(passon);
+                        } else {
+                            debug('[%s] Compendium with ID %s already exists; user %s', passon.id, compendiumID, compendium.user);
+                            let message = 'Error fetching ID from compendium, ID already exists';
+                            let error = new Error(message);
+                            error.msg = message;
+                            error.status = 400;
+                            reject(error);
+                        }
+                    });
+                } else {
+                    debug('[%s] Invalid ID %s specified for compendium in %s', passon.id, compendiumID, passon.configurationFile);
+                    let message = 'Invalid id found in compendium detection file';
+                    let error = new Error(message);
+                    error.msg = message;
+                    error.status = 400;
+                    reject(error);
+                }
+            }
+        } else {
+            debug('[%s] Not a compendium, keeping generated ID', passon.id);
+            fulfill(passon);
+        }
+    });
+}
+
+function moveCompendiumFiles(passon) {
+    return new Promise((fulfill, reject) => {
+
+        if (passon.isCompendium){
+            let updatedPath = path.join(config.fs.compendium, passon.id);
+            debug('[%s] Copying compendium files from %s to %s due to ID specified in compendium file', passon.id, passon.compendium_path, updatedPath);
+
+            let cmd = 'mv ' + passon.compendium_path + ' ' + updatedPath;
+            exec(cmd, (error, stdout, stderr) => {
+                if (error || stderr) {
+                    debug('Error copying compendium files: %O', {error, stderr, stdout});
+                    debug(error, stderr, stdout);
+                    let errors = error.message.split(':');
+                    let message = errorMessageHelper(errors[errors.length - 1]);
+                    error.msg = 'moving compendium files to new location failed: ' + message;
+                    error.status = 500;
+                    reject(error);
+                } else {
+                    debug('[%s] Moving compendium files finished: %s', passon.id, stdout);
+                    passon.compendium_path = updatedPath;
+                    passon.configurationFile = path.join(passon.compendium_path, config.bagit.detectionFileName);
+                    fulfill(passon);
+                }
+            });
+
+        } else {
+            debug('[%s] Not a compendium, files do not need to be moved', passon.id, );
+            fulfill(passon);
+        }
+
+    });
+}
+
 function getTextFiles(passon) {
     return new Promise((fulfill, reject) => {
         debug('[%s] Getting text files', passon.id);
@@ -906,6 +1007,8 @@ module.exports = {
     detectCompendium: detectCompendium,
     getTextFiles: getTextFiles,
     checkEncoding: checkEncoding,
+    fetchCompendiumID : fetchCompendiumID,
+    moveCompendiumFiles : moveCompendiumFiles,
     extractMetadata: extractMetadata,
     loadMetadata: loadMetadata,
     brokerMetadata: brokerMetadata,

lib/uploader.js

@@ -48,6 +48,8 @@ function Uploader(req, res) {
             .then(steps.detectBag)
             .then(validateBag)
             .then(steps.detectCompendium)
+            .then(steps.fetchCompendiumID)
+            .then(steps.moveCompendiumFiles)
             .then(steps.extractMetadata)
             .then(steps.loadMetadata)
             .then(steps.brokerMetadata)