Relocate persistent data files.

These need to be moved out of the gateway tree to better handle
upgrades and for data persistence when using Docker.

.gitignore

@@ -5,9 +5,10 @@
 .nyc_output/
 /build
 OZW_Log.txt
+config/local.js
 db.sqlite3
 node_modules/
 static/js/lib/stm_web.min.js
-static/uploads/floorplan.svg
+static/uploads
 zwcfg_0x*.xml
 zwscene.xml

README.md

@@ -118,13 +118,15 @@ $ yarn
 
  Add SSL certificate:
 
- If you don't plan on using Mozilla's provided tunneling service to set up a `*.mozilla-iot.org` domain, you can use your own SSL certificate. The HTTPS server looks for `privatekey.pem` and `certificate.pem`. You can use a real certificate or generate a self-signed one by following the steps below.
+ If you don't plan on using Mozilla's provided tunneling service to set up a `*.mozilla-iot.org` domain, you can use your own SSL certificate. The HTTPS server looks for `privatekey.pem` and `certificate.pem` in the directory specified by `ssltunnel.directory` in your config. You can use a real certificate or generate a self-signed one by following the steps below.
 
  ```
- $ mkdir -p ssl
- $ openssl genrsa -out ssl/privatekey.pem 2048
- $ openssl req -new -sha256 -key ssl/privatekey.pem -out ssl/csr.pem
- $ openssl x509 -req -in ssl/csr.pem -signkey ssl/privatekey.pem -out ssl/certificate.pem
+ $ MOZIOT_HOME="${MOZIOT_HOME:=${HOME}/mozilla-iot}"
+ $ SSL_DIR="${MOZIOT_HOME}/data/ssl"
+ $ [ ! -d "${SSL_DIR}" ] && mkdir -p "${SSL_DIR}"
+ $ openssl genrsa -out "${SSL_DIR}/privatekey.pem" 2048
+ $ openssl req -new -sha256 -key "${SSL_DIR}/privatekey.pem" -out "${SSL_DIR}/csr.pem"
+ $ openssl x509 -req -in "${SSL_DIR}/csr.pem" -signkey "${SSL_DIR}/privatekey.pem" -out "${SSL_DIR}/certificate.pem"
 ```
 
  Start the web server:

config/default.js

@@ -8,13 +8,18 @@
 
 'use strict';
 
+const os = require('os');
+const home = os.homedir();
+
 module.exports = {
   // Expose CLI
   cli: true,
 
+  userConfigDir: `${home}/mozilla-iot/data/config`,
+
   ports: {
     https: 4443,
-    http: 8080
+    http: 8080,
   },
   // Whether the gateway is behind port forwarding and should use simplified
   // port-free urls
@@ -25,7 +30,7 @@ module.exports = {
     listUrl: 'https://raw.githubusercontent.com/mozilla-iot/addon-list/master/list.json',
   },
   database: {
-    filename: './db.sqlite3',
+    filename: `${home}/mozilla-iot/data/config/db.sqlite3`,
     removeBeforeOpen: false,
   },
   ipc: {
@@ -44,7 +49,7 @@ module.exports = {
     },
   },
   uploads: {
-    directory: '../static/uploads/' // Directory to store uploads in
+    directory: `${home}/mozilla-iot/data/static/uploads/`, // Directory to store uploads in
   },
   authentication: {
     defaultUser: null,
@@ -55,7 +60,8 @@ module.exports = {
     domain: 'mozilla-iot.org',
     pagekite_cmd: './pagekite.py',
     port: 443,
-    certemail: 'certificate@mozilla-iot.org'
+    certemail: 'certificate@mozilla-iot.org',
+    directory: `${home}/mozilla-iot/data/ssl`,
   },
-  bcryptRounds: 2
+  bcryptRounds: 2,
 };

config/test.js

@@ -26,15 +26,4 @@ module.exports = {
   ipc: {
     protocol: 'inproc',
   },
-  settings: {
-    defaults: {
-    },
-  },
-  uploads: {
-    directory: '../../static/uploads/' // Directory to store uploads in
-  },
-  authentication: {
-    defaultUser: null,
-  },
-  bcryptRounds: 2
 };

package.json

@@ -38,6 +38,7 @@
     "greenlock": "^2.1.15",
     "jsonwebtoken": "^8.1.0",
     "le-challenge-dns": "https://github.com/mozilla-iot/le-challenge-dns",
+    "mkdirp": "^0.5.1",
     "mustache-express": "^1.2.5",
     "nanomsg": "^3.3.0",
     "nocache": "^2.0.0",

run-app.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+MOZIOT_HOME="${MOZIOT_HOME:=${HOME}/mozilla-iot}"
+
 run_app() {
   export NVM_DIR="$HOME/.nvm"
   [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"  # This loads nvm
@@ -16,4 +18,4 @@ run_app() {
   npm start
 }
 
-run_app > ${HOME}/mozilla-iot/gateway/run-app.log 2>&1
+run_app > "${MOZIOT_HOME}/data/log/run-app.log" 2>&1

src/app.js

@@ -19,6 +19,7 @@ const GetOpt = require('node-getopt');
 const config = require('config');
 const path = require('path');
 const mustacheExpress = require('mustache-express');
+const mkdirp = require('mkdirp');
 const addonManager = require('./addon-manager');
 const db = require('./db');
 const Router = require('./router');
@@ -34,6 +35,19 @@ require('./log-timestamps');
 // This is then used in other places (like src/addons/plugin/ipc.js)
 require('./app-instance');
 
+// Relocate the database before opening it, if necessary.
+// This has been moved out of the gateway directory for Docker data persistence
+// purposes.
+const dbPath = config.get('database.filename');
+const dbDir = path.dirname(dbPath);
+if (!fs.existsSync(dbPath)) {
+  mkdirp.sync(dbDir);
+}
+
+if (fs.existsSync('db.sqlite3')) {
+  fs.renameSync('db.sqlite3', dbPath);
+}
+
 // Open the database
 db.open();
 
@@ -57,20 +71,69 @@ if (fs.existsSync('notunnel')) {
 }
 
 // Move certificates, if necessary. If this throws an error, let it bubble up.
-if (!fs.existsSync('ssl')) {
-  fs.mkdirSync('ssl');
+// These have been moved out of the gateway directory for Docker data
+// persistence purposes.
+const sslDir = config.get('ssltunnel.directory');
+if (!fs.existsSync(sslDir)) {
+  mkdirp.sync(sslDir);
 }
 
 if (fs.existsSync('privatekey.pem')) {
-  fs.renameSync('privatekey.pem', path.join('ssl', 'privatekey.pem'));
+  fs.renameSync('privatekey.pem', path.join(sslDir, 'privatekey.pem'));
+} else if (fs.existsSync('ssl/privatekey.pem')) {
+  fs.renameSync('ssl/privatekey.pem', path.join(sslDir, 'privatekey.pem'));
 }
 
 if (fs.existsSync('certificate.pem')) {
-  fs.renameSync('certificate.pem', path.join('ssl', 'certificate.pem'));
+  fs.renamesync('certificate.pem', path.join(sslDir, 'certificate.pem'));
+} else if (fs.existsSync('ssl/certificate.pem')) {
+  fs.renameSync('ssl/certificate.pem', path.join(sslDir, 'certificate.pem'));
 }
 
 if (fs.existsSync('chain.pem')) {
-  fs.renameSync('chain.pem', path.join('ssl', 'chain.pem'));
+  fs.renameSync('chain.pem', path.join(sslDir, 'chain.pem'));
+} else if (fs.existsSync('ssl/chain.pem')) {
+  fs.renameSync('ssl/chain.pem', path.join(sslDir, 'chain.pem'));
+}
+
+if (fs.existsSync('ssl')) {
+  fs.rmdirSync('ssl');
+}
+
+// Move old uploads, if necessary.
+// This has been moved out of the gateway directory for Docker data persistence
+// purposes.
+const uploadDir = config.get('uploads.directory');
+if (!fs.existsSync(uploadDir)) {
+  mkdirp.sync(uploadDir);
+}
+
+const oldUploadDir = `${__dirname}/../static/uploads`;
+if (fs.existsSync(oldUploadDir) && fs.lstatSync(oldUploadDir).isDirectory()) {
+  const fnames = fs.readdirSync(oldUploadDir);
+  for (const fname of fnames) {
+    fs.renameSync(path.join(oldUploadDir, fname), path.join(uploadDir, fname));
+  }
+
+  fs.rmdirSync(oldUploadDir);
+  fs.symlinkSync(uploadDir, oldUploadDir);
+}
+
+// Create a user config if one doesn't exist.
+const userConfig = config.get('userConfigDir');
+if (!fs.existsSync(userConfig)) {
+  mkdirp.sync(userConfig);
+}
+
+const userConfigPath = path.join(userConfig, 'local.js');
+if (!fs.existsSync(userConfigPath)) {
+  fs.writeFileSync(
+    userConfigPath, '\'use strict\';\n\nmodule.exports = {\n};');
+}
+
+const localConfigPath = `${__dirname}/../config/local.js`;
+if (!fs.existsSync(localConfigPath)) {
+  fs.symlinkSync(userConfigPath, localConfigPath);
 }
 
 let httpServer = http.createServer();
@@ -86,11 +149,11 @@ function createHttpsServer() {
 
   // HTTPS server configuration
   const options = {
-    key: fs.readFileSync(path.join('ssl', 'privatekey.pem')),
-    cert: fs.readFileSync(path.join('ssl', 'certificate.pem'))
+    key: fs.readFileSync(path.join(sslDir, 'privatekey.pem')),
+    cert: fs.readFileSync(path.join(sslDir, 'certificate.pem'))
   };
-  if (fs.existsSync(path.join('ssl', 'chain.pem'))) {
-    options.ca = fs.readFileSync(path.join('ssl', 'chain.pem'));
+  if (fs.existsSync(path.join(sslDir, 'chain.pem'))) {
+    options.ca = fs.readFileSync(path.join(sslDir, 'chain.pem'));
   }
   return https.createServer(options);
 }

src/controllers/settings_controller.js

@@ -180,9 +180,10 @@ SettingsController.post('/subscribe', async (request, response) => {
     console.log('success', results);
 
     // ok. we got the certificates. let's save them
-    fs.writeFileSync(path.join('ssl', 'certificate.pem'), results.cert);
-    fs.writeFileSync(path.join('ssl', 'privatekey.pem'), results.privkey);
-    fs.writeFileSync(path.join('ssl', 'chain.pem'), results.chain);
+    const sslDir = config.get('ssltunnel.directory');
+    fs.writeFileSync(path.join(sslDir, 'certificate.pem'), results.cert);
+    fs.writeFileSync(path.join(sslDir, 'privatekey.pem'), results.privkey);
+    fs.writeFileSync(path.join(sslDir, 'chain.pem'), results.chain);
 
     // now we associate user's emails with the subdomain, unless it was
     // reclaimed.

src/controllers/uploads_controller.js

@@ -16,7 +16,7 @@ const path = require('path');
 const config = require('config');
 const Constants = require('../constants');
 
-const UPLOADS_PATH = path.join(__dirname, config.get('uploads.directory'));
+const UPLOADS_PATH = config.get('uploads.directory');
 const FLOORPLAN_PATH = path.join(UPLOADS_PATH, 'floorplan.svg');
 const FALLBACK_FLOORPLAN_PATH = path.join(Constants.STATIC_PATH,
                                           'images',

src/ssltunnel.js

@@ -95,8 +95,9 @@ var TunnelService = {
 
     // method to check if the box has certificates
     hasCertificates: function() {
-        return fs.existsSync(path.join('ssl', 'certificate.pem')) &&
-            fs.existsSync(path.join('ssl', 'privatekey.pem'));
+        const sslDir = config.get('ssltunnel.directory');
+        return fs.existsSync(path.join(sslDir, 'certificate.pem')) &&
+            fs.existsSync(path.join(sslDir, 'privatekey.pem'));
     },
 
     // method to check if the box has a registered tunnel

src/test/run-tests.sh

@@ -2,7 +2,8 @@
 
 SCRIPTDIR="$(dirname ""$0"")"
 
-if [ ! -f "ssl/certificate.pem" ]; then
+MOZIOT_HOME="${MOZIOT_HOME:=${HOME}/mozilla-iot}"
+if [ ! -f "${MOZIOT_HOME}/data/ssl/certificate.pem" ]; then
   ${SCRIPTDIR}/../../tools/make-self-signed-cert.sh
 fi
 

src/tunnel_setup.js

@@ -33,8 +33,9 @@ var TunnelSetup = {
             }
 
             // then we check if we have certificates installed
-            if ((fs.existsSync(path.join('ssl', 'certificate.pem'))
-                && fs.existsSync(path.join('ssl', 'privatekey.pem')))
+            const sslDir = config.get('ssltunnel.directory');
+            if ((fs.existsSync(path.join(sslDir, 'certificate.pem'))
+                && fs.existsSync(path.join(sslDir, 'privatekey.pem')))
                 || notunnel) {
                 // if certs are installed,
                 // then we don't need to do anything and return

tools/config-editor.py

@@ -9,7 +9,11 @@
 import tempfile
 
 
-_DEFAULT_DATABASE = '/home/pi/mozilla-iot/gateway/db.sqlite3'
+_MOZIOT_HOME = os.getenv('MOZIOT_HOME')
+if not _MOZIOT_HOME:
+    _MOZIOT_HOME = os.path.join(os.path.expanduser('~'), 'mozilla-iot')
+
+_DEFAULT_DATABASE = os.path.join(_MOZIOT_HOME, 'data', 'db.sqlite3')
 _DEFAULT_EDITOR = 'nano'
 
 

tools/deploy-certificates.sh

@@ -1,15 +1,16 @@
 #!/bin/bash
 
-gateway_dir="/home/pi/mozilla-iot/gateway"
+MOZIOT_HOME="${MOZIOT_HOME:=/home/pi/mozilla-iot}"
+SSL_DIR="${MOZIOT_HOME}/data/ssl"
 
 # Make sure the certificate and private key files are never world readable.
 umask 077
 
 # Copy in the new certs.
-mkdir -p "${gateway_dir}/ssl"
-cp "${RENEWED_LINEAGE}/chain.pem" "${gateway_dir}/ssl/chain.pem"
-cp "${RENEWED_LINEAGE}/cert.pem" "${gateway_dir}/ssl/certificate.pem"
-cp "${RENEWED_LINEAGE}/privkey.pem" "${gateway_dir}/ssl/privatekey.pem"
+[ ! -d "${SSL_DIR}" ] && mkdir -p "${SSL_DIR}"
+cp "${RENEWED_LINEAGE}/chain.pem" "${SSL_DIR}/chain.pem"
+cp "${RENEWED_LINEAGE}/cert.pem" "${SSL_DIR}/certificate.pem"
+cp "${RENEWED_LINEAGE}/privkey.pem" "${SSL_DIR}/privatekey.pem"
 
 # Restart the gateway.
 systemctl restart mozilla-iot-gateway.service

tools/make-self-signed-cert.sh

@@ -2,7 +2,9 @@
 
 set -x
 
-mkdir -p ssl
-openssl genrsa -out ssl/privatekey.pem 2048
-openssl req -new -sha256 -key ssl/privatekey.pem -out ssl/csr.pem -subj '/CN=www.toizom.com/O=MozillaIoT Gateway/C=US'
-openssl x509 -req -in ssl/csr.pem -signkey ssl/privatekey.pem -out ssl/certificate.pem
+MOZIOT_HOME="${MOZIOT_HOME:=${HOME}/mozilla-iot}"
+SSL_DIR="${MOZIOT_HOME}/data/ssl"
+[ ! -d "${SSL_DIR}" ] && mkdir -p "${SSL_DIR}"
+openssl genrsa -out "${SSL_DIR}/privatekey.pem" 2048
+openssl req -new -sha256 -key "${SSL_DIR}/privatekey.pem" -out "${SSL_DIR}/csr.pem" -subj '/CN=www.toizom.com/O=MozillaIoT Gateway/C=US'
+openssl x509 -req -in "${SSL_DIR}/csr.pem" -signkey "${SSL_DIR}/privatekey.pem" -out "${SSL_DIR}/certificate.pem"