From 73f8aa370e00457934a38ea766398c6e4ecc56cb Mon Sep 17 00:00:00 2001 From: Mike Stegeman Date: Mon, 12 Feb 2018 14:32:37 -0700 Subject: [PATCH] Relocate persistent data files. These need to be moved out of the gateway tree to better handle upgrades and for data persistence when using Docker. --- .gitignore | 3 +- README.md | 12 ++-- config/default.js | 16 +++-- config/test.js | 11 ---- package.json | 1 + run-app.sh | 4 +- src/app.js | 81 +++++++++++++++++++++++--- src/controllers/settings_controller.js | 7 ++- src/controllers/uploads_controller.js | 2 +- src/ssltunnel.js | 5 +- src/test/run-tests.sh | 3 +- src/tunnel_setup.js | 5 +- ssl/.gitkeep | 0 static/uploads/README.md | 1 - tools/config-editor.py | 6 +- tools/deploy-certificates.sh | 11 ++-- tools/make-self-signed-cert.sh | 10 ++-- tools/manual-renew-certs.sh | 34 +++++------ tools/renew-certificates.sh | 12 ++-- 19 files changed, 149 insertions(+), 75 deletions(-) delete mode 100644 ssl/.gitkeep delete mode 100644 static/uploads/README.md diff --git a/.gitignore b/.gitignore index 24f4c81f0c..9cbf174abc 100644 --- a/.gitignore +++ b/.gitignore @@ -5,9 +5,10 @@ .nyc_output/ /build OZW_Log.txt +config/local.js db.sqlite3 node_modules/ static/js/lib/stm_web.min.js -static/uploads/floorplan.svg +static/uploads zwcfg_0x*.xml zwscene.xml diff --git a/README.md b/README.md index 659182c23c..b077b208e2 100644 --- a/README.md +++ b/README.md @@ -118,13 +118,15 @@ $ yarn Add SSL certificate: - If you don't plan on using Mozilla's provided tunneling service to set up a `*.mozilla-iot.org` domain, you can use your own SSL certificate. The HTTPS server looks for `privatekey.pem` and `certificate.pem`. You can use a real certificate or generate a self-signed one by following the steps below. + If you don't plan on using Mozilla's provided tunneling service to set up a `*.mozilla-iot.org` domain, you can use your own SSL certificate. The HTTPS server looks for `privatekey.pem` and `certificate.pem` in the directory specified by `ssltunnel.directory` in your config. You can use a real certificate or generate a self-signed one by following the steps below. ``` - $ mkdir -p ssl - $ openssl genrsa -out ssl/privatekey.pem 2048 - $ openssl req -new -sha256 -key ssl/privatekey.pem -out ssl/csr.pem - $ openssl x509 -req -in ssl/csr.pem -signkey ssl/privatekey.pem -out ssl/certificate.pem + $ MOZIOT_HOME="${MOZIOT_HOME:=${HOME}/mozilla-iot}" + $ SSL_DIR="${MOZIOT_HOME}/data/ssl" + $ [ ! -d "${SSL_DIR}" ] && mkdir -p "${SSL_DIR}" + $ openssl genrsa -out "${SSL_DIR}/privatekey.pem" 2048 + $ openssl req -new -sha256 -key "${SSL_DIR}/privatekey.pem" -out "${SSL_DIR}/csr.pem" + $ openssl x509 -req -in "${SSL_DIR}/csr.pem" -signkey "${SSL_DIR}/privatekey.pem" -out "${SSL_DIR}/certificate.pem" ``` Start the web server: diff --git a/config/default.js b/config/default.js index 80263d6617..a31b109223 100644 --- a/config/default.js +++ b/config/default.js @@ -8,13 +8,18 @@ 'use strict'; +const os = require('os'); +const home = os.homedir(); + module.exports = { // Expose CLI cli: true, + userConfigDir: `${home}/mozilla-iot/data/config`, + ports: { https: 4443, - http: 8080 + http: 8080, }, // Whether the gateway is behind port forwarding and should use simplified // port-free urls @@ -25,7 +30,7 @@ module.exports = { listUrl: 'https://raw.githubusercontent.com/mozilla-iot/addon-list/master/list.json', }, database: { - filename: './db.sqlite3', + filename: `${home}/mozilla-iot/data/config/db.sqlite3`, removeBeforeOpen: false, }, ipc: { @@ -44,7 +49,7 @@ module.exports = { }, }, uploads: { - directory: '../static/uploads/' // Directory to store uploads in + directory: `${home}/mozilla-iot/data/static/uploads/`, // Directory to store uploads in }, authentication: { defaultUser: null, @@ -55,7 +60,8 @@ module.exports = { domain: 'mozilla-iot.org', pagekite_cmd: './pagekite.py', port: 443, - certemail: 'certificate@mozilla-iot.org' + certemail: 'certificate@mozilla-iot.org', + directory: `${home}/mozilla-iot/data/ssl`, }, - bcryptRounds: 2 + bcryptRounds: 2, }; diff --git a/config/test.js b/config/test.js index 322e2abf57..ac4f0b85f7 100644 --- a/config/test.js +++ b/config/test.js @@ -26,15 +26,4 @@ module.exports = { ipc: { protocol: 'inproc', }, - settings: { - defaults: { - }, - }, - uploads: { - directory: '../../static/uploads/' // Directory to store uploads in - }, - authentication: { - defaultUser: null, - }, - bcryptRounds: 2 }; diff --git a/package.json b/package.json index 5f5291768f..aadc9bb793 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "greenlock": "^2.1.15", "jsonwebtoken": "^8.1.0", "le-challenge-dns": "https://github.com/mozilla-iot/le-challenge-dns", + "mkdirp": "^0.5.1", "mustache-express": "^1.2.5", "nanomsg": "^3.3.0", "nocache": "^2.0.0", diff --git a/run-app.sh b/run-app.sh index 5066271a37..8fe67b1761 100755 --- a/run-app.sh +++ b/run-app.sh @@ -1,5 +1,7 @@ #!/bin/bash +MOZIOT_HOME="${MOZIOT_HOME:=${HOME}/mozilla-iot}" + run_app() { export NVM_DIR="$HOME/.nvm" [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm @@ -16,4 +18,4 @@ run_app() { npm start } -run_app > ${HOME}/mozilla-iot/gateway/run-app.log 2>&1 +run_app > "${MOZIOT_HOME}/data/log/run-app.log" 2>&1 diff --git a/src/app.js b/src/app.js index e9f9c32c64..4c1fa41aa4 100644 --- a/src/app.js +++ b/src/app.js @@ -19,6 +19,7 @@ const GetOpt = require('node-getopt'); const config = require('config'); const path = require('path'); const mustacheExpress = require('mustache-express'); +const mkdirp = require('mkdirp'); const addonManager = require('./addon-manager'); const db = require('./db'); const Router = require('./router'); @@ -34,6 +35,19 @@ require('./log-timestamps'); // This is then used in other places (like src/addons/plugin/ipc.js) require('./app-instance'); +// Relocate the database before opening it, if necessary. +// This has been moved out of the gateway directory for Docker data persistence +// purposes. +const dbPath = config.get('database.filename'); +const dbDir = path.dirname(dbPath); +if (!fs.existsSync(dbPath)) { + mkdirp.sync(dbDir); +} + +if (fs.existsSync('db.sqlite3')) { + fs.renameSync('db.sqlite3', dbPath); +} + // Open the database db.open(); @@ -57,20 +71,69 @@ if (fs.existsSync('notunnel')) { } // Move certificates, if necessary. If this throws an error, let it bubble up. -if (!fs.existsSync('ssl')) { - fs.mkdirSync('ssl'); +// These have been moved out of the gateway directory for Docker data +// persistence purposes. +const sslDir = config.get('ssltunnel.directory'); +if (!fs.existsSync(sslDir)) { + mkdirp.sync(sslDir); } if (fs.existsSync('privatekey.pem')) { - fs.renameSync('privatekey.pem', path.join('ssl', 'privatekey.pem')); + fs.renameSync('privatekey.pem', path.join(sslDir, 'privatekey.pem')); +} else if (fs.existsSync('ssl/privatekey.pem')) { + fs.renameSync('ssl/privatekey.pem', path.join(sslDir, 'privatekey.pem')); } if (fs.existsSync('certificate.pem')) { - fs.renameSync('certificate.pem', path.join('ssl', 'certificate.pem')); + fs.renamesync('certificate.pem', path.join(sslDir, 'certificate.pem')); +} else if (fs.existsSync('ssl/certificate.pem')) { + fs.renameSync('ssl/certificate.pem', path.join(sslDir, 'certificate.pem')); } if (fs.existsSync('chain.pem')) { - fs.renameSync('chain.pem', path.join('ssl', 'chain.pem')); + fs.renameSync('chain.pem', path.join(sslDir, 'chain.pem')); +} else if (fs.existsSync('ssl/chain.pem')) { + fs.renameSync('ssl/chain.pem', path.join(sslDir, 'chain.pem')); +} + +if (fs.existsSync('ssl')) { + fs.rmdirSync('ssl'); +} + +// Move old uploads, if necessary. +// This has been moved out of the gateway directory for Docker data persistence +// purposes. +const uploadDir = config.get('uploads.directory'); +if (!fs.existsSync(uploadDir)) { + mkdirp.sync(uploadDir); +} + +const oldUploadDir = `${__dirname}/../static/uploads`; +if (fs.existsSync(oldUploadDir) && fs.lstatSync(oldUploadDir).isDirectory()) { + const fnames = fs.readdirSync(oldUploadDir); + for (const fname of fnames) { + fs.renameSync(path.join(oldUploadDir, fname), path.join(uploadDir, fname)); + } + + fs.rmdirSync(oldUploadDir); + fs.symlinkSync(uploadDir, oldUploadDir); +} + +// Create a user config if one doesn't exist. +const userConfig = config.get('userConfigDir'); +if (!fs.existsSync(userConfig)) { + mkdirp.sync(userConfig); +} + +const userConfigPath = path.join(userConfig, 'local.js'); +if (!fs.existsSync(userConfigPath)) { + fs.writeFileSync( + userConfigPath, '\'use strict\';\n\nmodule.exports = {\n};'); +} + +const localConfigPath = `${__dirname}/../config/local.js`; +if (!fs.existsSync(localConfigPath)) { + fs.symlinkSync(userConfigPath, localConfigPath); } let httpServer = http.createServer(); @@ -86,11 +149,11 @@ function createHttpsServer() { // HTTPS server configuration const options = { - key: fs.readFileSync(path.join('ssl', 'privatekey.pem')), - cert: fs.readFileSync(path.join('ssl', 'certificate.pem')) + key: fs.readFileSync(path.join(sslDir, 'privatekey.pem')), + cert: fs.readFileSync(path.join(sslDir, 'certificate.pem')) }; - if (fs.existsSync(path.join('ssl', 'chain.pem'))) { - options.ca = fs.readFileSync(path.join('ssl', 'chain.pem')); + if (fs.existsSync(path.join(sslDir, 'chain.pem'))) { + options.ca = fs.readFileSync(path.join(sslDir, 'chain.pem')); } return https.createServer(options); } diff --git a/src/controllers/settings_controller.js b/src/controllers/settings_controller.js index 6b0308b521..bde1e6d286 100644 --- a/src/controllers/settings_controller.js +++ b/src/controllers/settings_controller.js @@ -180,9 +180,10 @@ SettingsController.post('/subscribe', async (request, response) => { console.log('success', results); // ok. we got the certificates. let's save them - fs.writeFileSync(path.join('ssl', 'certificate.pem'), results.cert); - fs.writeFileSync(path.join('ssl', 'privatekey.pem'), results.privkey); - fs.writeFileSync(path.join('ssl', 'chain.pem'), results.chain); + const sslDir = config.get('ssltunnel.directory'); + fs.writeFileSync(path.join(sslDir, 'certificate.pem'), results.cert); + fs.writeFileSync(path.join(sslDir, 'privatekey.pem'), results.privkey); + fs.writeFileSync(path.join(sslDir, 'chain.pem'), results.chain); // now we associate user's emails with the subdomain, unless it was // reclaimed. diff --git a/src/controllers/uploads_controller.js b/src/controllers/uploads_controller.js index bbc09da4f6..e33a35b84f 100644 --- a/src/controllers/uploads_controller.js +++ b/src/controllers/uploads_controller.js @@ -16,7 +16,7 @@ const path = require('path'); const config = require('config'); const Constants = require('../constants'); -const UPLOADS_PATH = path.join(__dirname, config.get('uploads.directory')); +const UPLOADS_PATH = config.get('uploads.directory'); const FLOORPLAN_PATH = path.join(UPLOADS_PATH, 'floorplan.svg'); const FALLBACK_FLOORPLAN_PATH = path.join(Constants.STATIC_PATH, 'images', diff --git a/src/ssltunnel.js b/src/ssltunnel.js index 6b81eeb3f2..e1ea2f08f7 100644 --- a/src/ssltunnel.js +++ b/src/ssltunnel.js @@ -95,8 +95,9 @@ var TunnelService = { // method to check if the box has certificates hasCertificates: function() { - return fs.existsSync(path.join('ssl', 'certificate.pem')) && - fs.existsSync(path.join('ssl', 'privatekey.pem')); + const sslDir = config.get('ssltunnel.directory'); + return fs.existsSync(path.join(sslDir, 'certificate.pem')) && + fs.existsSync(path.join(sslDir, 'privatekey.pem')); }, // method to check if the box has a registered tunnel diff --git a/src/test/run-tests.sh b/src/test/run-tests.sh index b0504ae688..775a93d2ab 100755 --- a/src/test/run-tests.sh +++ b/src/test/run-tests.sh @@ -2,7 +2,8 @@ SCRIPTDIR="$(dirname ""$0"")" -if [ ! -f "ssl/certificate.pem" ]; then +MOZIOT_HOME="${MOZIOT_HOME:=${HOME}/mozilla-iot}" +if [ ! -f "${MOZIOT_HOME}/data/ssl/certificate.pem" ]; then ${SCRIPTDIR}/../../tools/make-self-signed-cert.sh fi diff --git a/src/tunnel_setup.js b/src/tunnel_setup.js index 1efd7a3d80..b1dc47c55b 100644 --- a/src/tunnel_setup.js +++ b/src/tunnel_setup.js @@ -33,8 +33,9 @@ var TunnelSetup = { } // then we check if we have certificates installed - if ((fs.existsSync(path.join('ssl', 'certificate.pem')) - && fs.existsSync(path.join('ssl', 'privatekey.pem'))) + const sslDir = config.get('ssltunnel.directory'); + if ((fs.existsSync(path.join(sslDir, 'certificate.pem')) + && fs.existsSync(path.join(sslDir, 'privatekey.pem'))) || notunnel) { // if certs are installed, // then we don't need to do anything and return diff --git a/ssl/.gitkeep b/ssl/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/static/uploads/README.md b/static/uploads/README.md deleted file mode 100644 index 6e5b8ab524..0000000000 --- a/static/uploads/README.md +++ /dev/null @@ -1 +0,0 @@ -This folder is used for file uploads. diff --git a/tools/config-editor.py b/tools/config-editor.py index c133946907..7a62029466 100755 --- a/tools/config-editor.py +++ b/tools/config-editor.py @@ -9,7 +9,11 @@ import tempfile -_DEFAULT_DATABASE = '/home/pi/mozilla-iot/gateway/db.sqlite3' +_MOZIOT_HOME = os.getenv('MOZIOT_HOME') +if not _MOZIOT_HOME: + _MOZIOT_HOME = os.path.join(os.path.expanduser('~'), 'mozilla-iot') + +_DEFAULT_DATABASE = os.path.join(_MOZIOT_HOME, 'data', 'db.sqlite3') _DEFAULT_EDITOR = 'nano' diff --git a/tools/deploy-certificates.sh b/tools/deploy-certificates.sh index 2d4c929765..bcce79b201 100755 --- a/tools/deploy-certificates.sh +++ b/tools/deploy-certificates.sh @@ -1,15 +1,16 @@ #!/bin/bash -gateway_dir="/home/pi/mozilla-iot/gateway" +MOZIOT_HOME="${MOZIOT_HOME:=/home/pi/mozilla-iot}" +SSL_DIR="${MOZIOT_HOME}/data/ssl" # Make sure the certificate and private key files are never world readable. umask 077 # Copy in the new certs. -mkdir -p "${gateway_dir}/ssl" -cp "${RENEWED_LINEAGE}/chain.pem" "${gateway_dir}/ssl/chain.pem" -cp "${RENEWED_LINEAGE}/cert.pem" "${gateway_dir}/ssl/certificate.pem" -cp "${RENEWED_LINEAGE}/privkey.pem" "${gateway_dir}/ssl/privatekey.pem" +[ ! -d "${SSL_DIR}" ] && mkdir -p "${SSL_DIR}" +cp "${RENEWED_LINEAGE}/chain.pem" "${SSL_DIR}/chain.pem" +cp "${RENEWED_LINEAGE}/cert.pem" "${SSL_DIR}/certificate.pem" +cp "${RENEWED_LINEAGE}/privkey.pem" "${SSL_DIR}/privatekey.pem" # Restart the gateway. systemctl restart mozilla-iot-gateway.service diff --git a/tools/make-self-signed-cert.sh b/tools/make-self-signed-cert.sh index 405e526fdd..3298b45b65 100755 --- a/tools/make-self-signed-cert.sh +++ b/tools/make-self-signed-cert.sh @@ -2,7 +2,9 @@ set -x -mkdir -p ssl -openssl genrsa -out ssl/privatekey.pem 2048 -openssl req -new -sha256 -key ssl/privatekey.pem -out ssl/csr.pem -subj '/CN=www.toizom.com/O=MozillaIoT Gateway/C=US' -openssl x509 -req -in ssl/csr.pem -signkey ssl/privatekey.pem -out ssl/certificate.pem +MOZIOT_HOME="${MOZIOT_HOME:=${HOME}/mozilla-iot}" +SSL_DIR="${MOZIOT_HOME}/data/ssl" +[ ! -d "${SSL_DIR}" ] && mkdir -p "${SSL_DIR}" +openssl genrsa -out "${SSL_DIR}/privatekey.pem" 2048 +openssl req -new -sha256 -key "${SSL_DIR}/privatekey.pem" -out "${SSL_DIR}/csr.pem" -subj '/CN=www.toizom.com/O=MozillaIoT Gateway/C=US' +openssl x509 -req -in "${SSL_DIR}/csr.pem" -signkey "${SSL_DIR}/privatekey.pem" -out "${SSL_DIR}/certificate.pem" diff --git a/tools/manual-renew-certs.sh b/tools/manual-renew-certs.sh index 90d7e5b22c..2a75698c03 100755 --- a/tools/manual-renew-certs.sh +++ b/tools/manual-renew-certs.sh @@ -1,7 +1,7 @@ #!/bin/bash script_dir=$(readlink -f $(dirname "$0")) -moziot_dir="/home/pi/mozilla-iot" +MOZIOT_HOME="${MOZIOT_HOME:=/home/pi/mozilla-iot}" moziot_email="certificate@mozilla-iot.org" pagekite_pidfile="/tmp/_pagekite.pid" temp_dir="$(mktemp -d)" @@ -32,14 +32,14 @@ if [ -z "$email" ]; then abort "Usage:\n\t$prog EMAIL" fi -if [ ! -f "${moziot_dir}/gateway/tunneltoken" ]; then - abort "Could not read ${moziot_dir}/gateway/tunneltoken" +if [ ! -f "${MOZIOT_HOME}/gateway/tunneltoken" ]; then + abort "Could not read ${MOZIOT_HOME}/gateway/tunneltoken" fi -domain="$(grep -oP '"name":".*?"' "${moziot_dir}/gateway/tunneltoken" | \ +domain="$(grep -oP '"name":".*?"' "${MOZIOT_HOME}/gateway/tunneltoken" | \ cut -d: -f2 | \ sed 's/"//g').mozilla-iot.org" -token="$(grep -oP '"token":".*?"' "${moziot_dir}/gateway/tunneltoken" | \ +token="$(grep -oP '"token":".*?"' "${MOZIOT_HOME}/gateway/tunneltoken" | \ cut -d: -f2 | \ sed 's/"//g')" @@ -88,7 +88,7 @@ server_pid=$! cd - >/dev/null 2>&1 echo "Starting PageKite." -"${moziot_dir}/gateway/pagekite.py" \ +"${MOZIOT_HOME}/gateway/pagekite.py" \ --clean \ --frontend="${domain}:443" \ --service_on="https:${domain}:localhost:4443:${token}" \ @@ -102,14 +102,14 @@ certbot certonly \ --preferred-challenges=http \ -d "${domain}" \ --force-renewal \ - --config-dir "${moziot_dir}/etc" \ - --work-dir "${moziot_dir}/var/lib" \ - --logs-dir "${moziot_dir}/var/log" \ + --config-dir "${MOZIOT_HOME}/etc" \ + --work-dir "${MOZIOT_HOME}/var/lib" \ + --logs-dir "${MOZIOT_HOME}/var/log" \ --non-interactive \ --agree-tos \ --email "${moziot_email}" || abort "Failed to verify and renew." -chown -R pi:pi "${moziot_dir}/etc" "${moziot_dir}/var" +chown -R pi:pi "${MOZIOT_HOME}/etc" "${MOZIOT_HOME}/var" echo "Stopping temporary web server." kill -15 "${server_pid}" >/dev/null 2>&1 @@ -120,13 +120,13 @@ kill -15 $(<"${pagekite_pidfile}") >/dev/null 2>&1 rm -f "${pagekite_pidfile}" echo "Copying in new certificates." -mkdir -p "${moziot_dir}/gateway/ssl" -cp "${moziot_dir}/etc/live/${domain}/cert.pem" \ - "${moziot_dir}/gateway/ssl/certificate.pem" -cp "${moziot_dir}/etc/live/${domain}/privkey.pem" \ - "${moziot_dir}/gateway/ssl/privatekey.pem" -cp "${moziot_dir}/etc/live/${domain}/chain.pem" \ - "${moziot_dir}/gateway/ssl/chain.pem" +mkdir -p "${MOZIOT_HOME}/data/ssl" +cp "${MOZIOT_HOME}/etc/live/${domain}/cert.pem" \ + "${MOZIOT_HOME}/data/ssl/certificate.pem" +cp "${MOZIOT_HOME}/etc/live/${domain}/privkey.pem" \ + "${MOZIOT_HOME}/data/ssl/privatekey.pem" +cp "${MOZIOT_HOME}/etc/live/${domain}/chain.pem" \ + "${MOZIOT_HOME}/data/ssl/chain.pem" echo "Registering domain with server." curl "http://api.mozilla-iot.org/setemail" \ diff --git a/tools/renew-certificates.sh b/tools/renew-certificates.sh index 884fcd8d6a..6e78cf7bbd 100755 --- a/tools/renew-certificates.sh +++ b/tools/renew-certificates.sh @@ -1,14 +1,14 @@ #!/bin/bash -moziot_dir="/home/pi/mozilla-iot" +MOZIOT_HOME="${MOZIOT_HOME:=/home/pi/mozilla-iot}" # NOTE: --renew-hook is deprecated and has been replaced by --deploy-hook. # However, the version of certbot in raspbian stretch doesn't have # --deploy-hook support, so we're sticking with the old option for now. certbot renew \ - --config-dir "$moziot_dir/etc" \ - --logs-dir "$moziot_dir/var/log" \ - --work-dir "$moziot_dir/var/lib" \ - --renew-hook "$moziot_dir/gateway/tools/deploy-certificates.sh" + --config-dir "${MOZIOT_HOME}/etc" \ + --logs-dir "${MOZIOT_HOME}/var/log" \ + --work-dir "${MOZIOT_HOME}/var/lib" \ + --renew-hook "${MOZIOT_HOME}/gateway/tools/deploy-certificates.sh" -chown -R pi:pi "$moziot_dir/etc" "$moziot_dir/var" +chown -R pi:pi "${MOZIOT_HOME}/etc" "${MOZIOT_HOME}/var"