Merge branch 'master' into build_dist

api/odk/controllers/archive-form.js

@@ -1,6 +1,8 @@
 var fs = require('fs');
 var fse = require('fs-extra');
 var path = require('path');
+
+const { syncDataDir } = require('../helpers/aws-sync');
 var settings = require('../../../settings.js');
 
 
@@ -14,7 +16,11 @@ const moveFiles = (archiveDir, formName) => {
         path.join(settings.dataDir, 'forms', i),
         path.join(archiveDir, 'forms', i),
         renameError => {
-          if (renameError) console.log(`Error when moving file ${i}.`);
+          if (renameError) {
+            console.log(`Error when moving file ${i}.`);
+          } else {
+            syncDataDir();
+          };
         }
       )
     );
@@ -28,7 +34,11 @@ const moveSubmissions = (submissionDir, archiveDir, formName) => {
     submissionDir,
     path.join(archiveDir, 'submissions', formName),
     renameError => {
-      if (renameError) console.log('It was not possible to move the submissions directory.');
+      if (renameError) {
+        console.log('It was not possible to move the submissions directory.');
+      } else {
+        syncDataDir();
+      };
     }
   );
 }

api/odk/controllers/delete-form.js

@@ -1,6 +1,7 @@
 var fs = require('fs');
 var path = require('path');
 var settings = require('../../../settings.js');
+const { syncDataDir } = require('../helpers/aws-sync');
 
 module.exports = (req, res, next) => {
   const formName = req.params.formName;
@@ -24,7 +25,7 @@ module.exports = (req, res, next) => {
       [xlsFile, xlsxFile, xmlFile].map(file => {
         fs.stat(file, (statError, stats) => {
           if (statError) {
-            errors.push(file)
+            errors.push(file);
           } else {
             fs.unlink(file, unlinkError => {
               if (unlinkError) errors.push(file);
@@ -39,9 +40,9 @@ module.exports = (req, res, next) => {
           {detail: "It was not possible to find or delete the form files"}
         );
       } else {
+        syncDataDir();
         res.status(200).json({detail: "Form deleted successfully"});
       }
     }
-  }
-  );
+  });
 };

api/odk/controllers/restore-form.js

@@ -1,6 +1,8 @@
 var fs = require('fs');
 var fse = require('fs-extra');
 var path = require('path');
+
+const { syncDataDir } = require('../helpers/aws-sync');
 var settings = require('../../../settings.js');
 
 
@@ -37,6 +39,8 @@ module.exports = (req, res, next) => {
                 return res.status(500).json(
                   {detail: "Something went wrong during the restoration process."}
                 );
+              } else {
+                syncDataDir();
               }
             }
           )
@@ -45,7 +49,15 @@ module.exports = (req, res, next) => {
       // move submissions dir
       fs.readdir(path.join(archiveDir, 'submissions', formName), (err, items) => {
         if (!err) {
-          fse.move(path.join(archiveDir, 'submissions', formName), submissionDir);
+          fse.move(
+            path.join(archiveDir, 'submissions', formName),
+            submissionDir,
+            moveError => {
+              if (!moveError) {
+                syncDataDir();
+              }
+            }
+          );
         }
       }
       );

api/odk/controllers/save-form.js

@@ -1,34 +1,47 @@
 var fs = require('fs');
 var path = require('path');
 var mkdirp = require('mkdirp');
+
+const {
+  syncDataDir
+} = require('../helpers/aws-sync');
 var settings = require('../../../settings.js');
 
-module.exports = function (req, res, next) {
-    var submission = req.submission;
-    var ext = submission.geojson ? '.geojson' : '.json';
-    var json = JSON.stringify(submission.json, null, '  ');
-    var xml = submission.xml; // Original XML for debug purposes is nice.
 
-    var dir = settings.dataDir + '/submissions/' + submission.formId + '/' + submission.instanceId;
-    var jsonFileName = dir + '/data' + ext;
-    var xmlFileName = dir + '/data' + '.xml';
+module.exports = function(req, res, next) {
+  var submission = req.submission;
+  var ext = submission.geojson ? '.geojson' : '.json';
+  var json = JSON.stringify(submission.json, null, '  ');
+  var xml = submission.xml; // Original XML for debug purposes is nice.
+
+  var dir = settings.dataDir + '/submissions/' + submission.formId + '/' + submission.instanceId;
+  var jsonFileName = dir + '/data' + ext;
+  var xmlFileName = dir + '/data' + '.xml';
 
-    mkdirp(path.dirname(jsonFileName), function (err) {
-        if (err) {
-            console.error(err);
-            res.status(500).json({status: 500, err: err});
-            return;
-        }
-        fs.writeFile(xmlFileName, xml, function(err) {
-            if (err) console.error(err);
-        });
-        fs.writeFile(jsonFileName, json, function (err) {
-            if (err) {
-                console.error(err);
-                res.status(500).json({status: 500, err: err});
-                return;
-            }
-            res.status(201).json({saved: jsonFileName});
-        });
+  mkdirp(path.dirname(jsonFileName), function(err) {
+    if (err) {
+      console.error(err);
+      res.status(500).json({status: 500, err: err});
+      return;
+    }
+    fs.writeFile(xmlFileName, xml, function(err) {
+      if (err) {
+        console.error(err);
+      } else {
+        syncDataDir();
+      }
+    });
+    fs.writeFile(jsonFileName, json, function(err) {
+      if (err) {
+        console.error(err);
+        res.status(500).json({status: 500, err: err});
+        return;
+      } else {
+        syncDataDir();
+      }
+      res.status(201).json({
+        saved: jsonFileName
+      });
     });
+  });
 };

api/odk/controllers/upload-form.js

@@ -12,6 +12,7 @@ const PythonShell = require('python-shell');
 const tempy = require('tempy');
 
 const settings = require('../../../settings');
+const { syncDataDir } = require('../helpers/aws-sync');
 const { getForms, loadXForm } = require('../../../util/xform');
 
 const formsDir = settings.dataDir + '/forms/';
@@ -221,6 +222,8 @@ module.exports = function (req, res, next) {
                   err: err,
                   msg: `Unable to move ${xform.title} to the forms directory.`
                 });
+              } else {
+                syncDataDir();
               }
 
               cleanup();

api/odk/helpers/aws-sync.js

@@ -0,0 +1,63 @@
+var s3 = require('@monolambda/s3');
+var path = require('path');
+
+var settings = require('../../../settings');
+
+const AWSBUCKETPREFIX = process.env.AWSBUCKETPREFIX ? process.env.AWSBUCKETPREFIX : '/';
+
+const syncDataDir = () => {
+  if (process.env.ENABLES3SYNC) {
+    var client = s3.createClient({
+      s3Options: {
+        accessKeyId: process.env.AWSKEYID,
+        secretAccessKey: process.env.AWSSECRETKEY
+      }
+    });
+    var params = {
+      localDir: settings.dataDir,
+      deleteRemoved: true,
+      s3Params: {
+        Bucket: process.env.AWSBUCKETNAME,
+        Prefix: AWSBUCKETPREFIX
+      }
+    };
+    var uploader = client.uploadDir(params);
+    uploader.on('error', function(err) {
+      console.error("unable to sync:", err.stack);
+    });
+    uploader.on('end', function() {
+      console.log("done uploading");
+    });
+  }
+};
+
+const downloadDataDir = () => {
+  if (process.env.ENABLES3SYNC) {
+    var client = s3.createClient({
+      s3Options: {
+        accessKeyId: process.env.AWSKEYID,
+        secretAccessKey: process.env.AWSSECRETKEY
+      }
+    });
+    var params = {
+      localDir: settings.dataDir,
+      deleteRemoved: true,
+      s3Params: {
+        Bucket: process.env.AWSBUCKETNAME,
+        Prefix: AWSBUCKETPREFIX
+      }
+    };
+    var uploader = client.downloadDir(params);
+    uploader.on('error', function(err) {
+      console.error("unable to sync:", err.stack);
+    });
+    uploader.on('end', function() {
+      console.log("Download from AWS S3 done");
+    });
+  }
+};
+
+module.exports = {
+  syncDataDir,
+  downloadDataDir
+};

build_frontend.js

@@ -1,3 +1,5 @@
-const args = [ 'build' ];
+var spawn = require('child_process').spawnSync;
 const opts = { stdio: 'inherit', cwd: 'frontend', shell: true };
-require('child_process').spawn('yarn', args, opts);
+
+const install = spawn('yarn', [ 'install' ], opts);
+const build = spawn('yarn', [ 'build' ], opts);

docs/development-installation.md

@@ -58,6 +58,29 @@ To start the server in the development mode, use `npm startdev`, it will make th
 
 The frontend builds are ignored by the `master` branch git and available on the `dist` branch. To make a new build and push it to the `dist` branch, use `npm pushbuild`
 
+### Data Sync
+
+If you want to enable AWS S3 sync, in a way to have a backup of forms and
+submissions files in a S3 bucket, set the following environment variables:
+
+```sh
+export ENABLES3SYNC=1
+export AWSKEYID=<your AWS access key ID>
+export AWSSECRETKEY=<your AWS secret access key>
+export AWSBUCKETNAME=<a S3 bucket name>
+export AWSBUCKETPREFIX=<(optional, bucket root is the default prefix) subdirectory where the files should be stored in the S3 bucket>
+```
+
+This will make the data be syncronized to your S3 bucket after each API request
+that modifies the data. The first variable enables/disables the S3
+sync and need to receive the value 1 or 0.
+
+If you need to get the data stored in a S3 Bucket to put it in your server,
+execute: `npm get_from_s3`.
+
+To assure that the files were synced to AWS before turning off a server, execute
+`npm send_to_s3`
+
 ### NodeJS Version Problems
 
 We are using node version 4.*. If you are having problems with another

frontend/src/components/forms/uploadForm.js

@@ -54,7 +54,7 @@ class UploadForm extends React.Component {
       }
       {
         this.state.error && <Callout title="Error!" intent="danger" className="upload-result">
-            Some error ocurred while uploading your file(s).
+            Some error occurred while uploading your file(s).
           </Callout>
       }
       <Upload {...uploaderProps} ref="inner">