Merge bce53e3 into 03cd194

Jenkinsfile

@@ -0,0 +1,64 @@
+node('node') {
+  try {
+
+    stage 'Checkout'
+
+      checkout scm
+
+    stage 'Test'
+
+      sh """#!/bin/bash -e
+        source '/var/lib/jenkins/.nvm/nvm.sh'
+        node -v
+        git clean -fdx
+        npm install
+        npm test
+      """
+
+    stage 'Build Docker'
+
+      sh "git rev-parse --short HEAD > .git/commit-id"
+      def commit_id = readFile('.git/commit-id').trim()
+      sh "./dockerfiles/build/build-landlord.sh ${env.BUILD_ID} ${commit_id} latest"
+      sh "./dockerfiles/build/build-renter.sh ${env.BUILD_ID} ${commit_id} latest"
+      sh "./dockerfiles/build/push.sh storjlabs/landlord:${env.BUILD_ID} storjlabs/landlord:${commit_id} storjlabs/landlord:latest"
+      sh "./dockerfiles/build/push.sh storjlabs/renter:${env.BUILD_ID} storjlabs/renter:${commit_id} storjlabs/renter:latest"
+
+    stage 'Deploy'
+
+      echo 'Push to Repo'
+      sh "./dockerfiles/deploy/deploy.staging.sh bridge-landlord storjlabs/landlord:${env.BUILD_ID}"
+      sh "./dockerfiles/deploy/deploy.staging.sh bridge-renter storjlabs/renter:${env.BUILD_ID}"
+
+    stage 'Cleanup'
+
+      echo 'prune and cleanup'
+      sh """#!/bin/bash -e
+        source '/var/lib/jenkins/.nvm/nvm.sh'
+        rm node_modules -rf
+      """
+
+      /*
+      mail body: 'project build successful',
+        from: 'build@storj.io',
+        replyTo: 'build@storj.io',
+        subject: 'project build successful',
+        to: "${env.CHANGE_AUTHOR_EMAIL}"
+      */
+
+  }
+
+  catch (err) {
+    currentBuild.result = "FAILURE"
+
+    /*
+    mail body: "project build error is here: ${env.BUILD_URL}" ,
+      from: 'build@storj.io',
+      replyTo: 'build@storj.io',
+      subject: 'project build failed',
+      to: "${env.CHANGE_AUTHOR_EMAIL}"
+
+      throw err
+    */
+  }
+}

dockerfiles/build/build-landlord.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+for TAG in "$@"; do
+  TAG_PARAMS+=" -t storjlabs/landlord:${TAG}"
+done
+
+echo "Building with TAG_PARAMS: ${TAG_PARAMS}"
+
+docker build ${TAG_PARAMS} -f ./dockerfiles/landlord.dockerfile .
+result=$?
+
+if [[ $result != 0 ]]; then
+  echo "Error building docker image"
+  exit 1
+fi
+
+echo "Success..."
+exit 0

dockerfiles/build/build-renter.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+for TAG in "$@"; do
+  TAG_PARAMS+=" -t storjlabs/renter:${TAG}"
+done
+
+echo "Building with TAG_PARAMS: ${TAG_PARAMS}"
+
+docker build ${TAG_PARAMS} -f ./dockerfiles/renter.dockerfile .
+result=$?
+
+if [[ $result != 0 ]]; then
+  echo "Error building docker image"
+  exit 1
+fi
+
+echo "Success..."
+exit 0

dockerfiles/build/push.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+for TAG in "$@"; do
+  docker push $TAG
+  result=$?
+done
+
+if [[ $result != 0 ]]; then
+  echo "Error pushing docker image"
+  exit 1
+fi
+
+echo "Success..."
+exit 0

dockerfiles/deploy/deploy.staging.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+PROJECT_NAME=$1
+CONTAINER=$2
+
+kubectl config set-cluster storj-nonprod
+kubectl --namespace storj-staging patch deployment $PROJECT_NAME -p"{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"$PROJECT_NAME\",\"image\":\"$CONTAINER\"}]}}}}"

dockerfiles/landlord.dockerfile

@@ -0,0 +1,35 @@
+# Use the latest Node v6 (LTS) release
+FROM node:6
+
+# Install troubleshooting utils
+RUN apt-get update && apt-get install vim telnet -y
+
+# We use dumb-init since Node.js is pretty terrible at running as PID 1
+RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64 \
+ && chmod +x /usr/local/bin/dumb-init
+
+# wait.sh forces our app to wait for other containers to come up before starting
+RUN wget -O /bin/wait.sh https://raw.githubusercontent.com/Storj/storj-sdk/master/scripts/wait.sh
+
+# We will run our application from /usr/src/app to be a good linux citizen
+RUN mkdir /storj && mkdir /storj/complex
+WORKDIR /storj/complex
+
+# Cache node_modules
+ADD ./package.json ./
+
+# Thanks to the above line, npm install only re-runs if package.json changes
+RUN npm install
+
+# Finally add in all of our source files
+ADD . .
+
+# setup.sh allows us to prime complex's configuration with the envrionement the container starts with, i.e. the IP address that gets assigned to it, allowing us to dynamically generate the configuration file at startup
+# Removed for production use but may need this to generate a config file
+#ADD setup.sh /bin/setup.sh
+
+# Our container needs dumb-init to handle PID-1 responsibilities from the linux kernel, wait.sh to make sure the services complex depends on are up before starting, and setup.sh to generate the configuration file for starting storj-complex
+ENTRYPOINT ["dumb-init", "--", "/bin/bash", "/bin/wait.sh", "./dockerfiles/scripts/setup-landlord.sh"]
+
+# By default, run storj-complex at startup
+CMD ["./bin/storj-complex.js -c /etc/storj/landlord.conf"]

dockerfiles/renter.dockerfile

@@ -0,0 +1,35 @@
+# Use the latest Node v6 (LTS) release
+FROM node:6
+
+# Install troubleshooting utils
+RUN apt-get update && apt-get install vim telnet -y
+
+# We use dumb-init since Node.js is pretty terrible at running as PID 1
+RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64 \
+ && chmod +x /usr/local/bin/dumb-init
+
+# wait.sh forces our app to wait for other containers to come up before starting
+RUN wget -O /bin/wait.sh https://raw.githubusercontent.com/Storj/storj-sdk/master/scripts/wait.sh
+
+# We will run our application from /usr/src/app to be a good linux citizen
+RUN mkdir /storj && mkdir /storj/complex
+WORKDIR /storj/complex
+
+# Cache node_modules
+ADD ./package.json ./
+
+# Thanks to the above line, npm install only re-runs if package.json changes
+RUN npm install
+
+# Finally add in all of our source files
+ADD . .
+
+# setup.sh allows us to prime complex's configuration with the envrionement the container starts with, i.e. the IP address that gets assigned to it, allowing us to dynamically generate the configuration file at startup
+# Removed for production use but may need this to generate a config file
+#ADD setup.sh /bin/setup.sh
+
+# Our container needs dumb-init to handle PID-1 responsibilities from the linux kernel, wait.sh to make sure the services complex depends on are up before starting, and setup.sh to generate the configuration file for starting storj-complex
+ENTRYPOINT ["dumb-init", "--", "/bin/bash", "/bin/wait.sh", "./dockerfiles/scripts/setup-renter.sh"]
+
+# By default, run storj-complex at startup
+CMD ["./bin/storj-complex.js -c /etc/storj/renter.conf"]

dockerfiles/scripts/setup-landlord.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+mkdir /etc/storj
+
+./dockerfiles/scripts/template_parse.sh ./dockerfiles/templates/landlord.conf.tmpl /etc/storj/landlord.conf storjlandlord
+
+exec $@

dockerfiles/scripts/setup-renter.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+mkdir /etc/storj
+mkdir /etc/storj/keys
+
+# Create the Network Private Extended Key file
+echo ${network_private_extended_key_string} > /etc/storj/keys/complex.key
+echo ${migration_private_key_string} > /etc/storj/keys/complexMigration.key
+
+# Create a random index for the renter
+export storjrenter_opts__networkIndex=${RANDOM}
+
+./dockerfiles/scripts/template_parse.sh ./dockerfiles/templates/renter.conf.tmpl /etc/storj/renter.conf storjrenter
+
+exec $@

dockerfiles/scripts/template_parse.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+TEMPLATE_IN_FILE=$1
+TEMPLATE_OUT_FILE=$2
+ENV_VAR_PREFIX=$3
+PREFIX_REGEX="${ENV_VAR_PREFIX}_(.*)="
+KEY_REGEX="(${ENV_VAR_PREFIX}_.*)="
+
+echo "Template in: $TEMPLATE_IN_FILE  Template out: $TEMPLATE_OUT_FILE  Var prefix: $ENV_VAR_PREFIX"
+
+# Copy template file to desired destination
+cp ${TEMPLATE_IN_FILE} ${TEMPLATE_OUT_FILE}
+
+# Get all environment variables with the specified prefix and
+# iterate through the env vars we found and replace any instances
+# of them in the config file with their values
+for ENV_ENTRY in $(env); do
+  # echo "Checking entry ${ENV_ENTRY}"
+
+  if [[ $ENV_ENTRY =~ $PREFIX_REGEX ]]; then
+    echo "Key ${ENV_ENTRY} is a match"
+
+    # Get the key minus the prefix
+    ENV_SUB_KEY="${BASH_REMATCH[1]}"
+
+    # Get the key including the prefix
+    [[ $ENV_ENTRY =~ $KEY_REGEX ]]
+    ENV_KEY=${BASH_REMATCH[1]}
+
+    # Use the key to get the value
+    ENV_VAL="${!ENV_KEY}"
+
+    echo "ENV_KEY is: ${ENV_KEY}"
+    echo "SUB_KEY is: ${ENV_SUB_KEY} and ENV_VAL is: ${ENV_VAL}"
+
+    # Sanatize the replacement value for sed
+    SAN_ENV_VAL=$(echo $ENV_VAL | sed -e 's/[\/&]/\\&/g')
+
+    echo "SAN_ENV_VAL is: ${SAN_ENV_VAL}"
+
+    # Replace all instances of the subkey with the value
+    sed -i "s/{{ ${ENV_SUB_KEY} }}/${SAN_ENV_VAL}/g" ${TEMPLATE_OUT_FILE}
+  fi
+done
+
+# Clean any remaining template variables out of the template and try to guess
+# if it should be a blank string, numeric value, or false
+sed -i 's/"{{ .* }}"/""/g' ${TEMPLATE_OUT_FILE}
+sed -i "s/{{ .* }}/false/g" ${TEMPLATE_OUT_FILE}

dockerfiles/templates/landlord.conf.tmpl

@@ -0,0 +1,29 @@
+{
+  "type": "Landlord",
+  "opts": {
+    "logLevel": {{ opts__logLevel }},
+    "amqpUrl": "{{ opts__amqpUrl }}",
+    "amqpOpts": {
+    },
+    "serverPort": {{ opts__serverPort }},
+    "serverOpts": {
+      "certificate": null,
+      "key": null,
+      "authorization": {
+        "username": "{{ opts__serverOpts__authorization__username }}",
+        "password": "{{ opts__serverOpts__authorization__password }}"
+      }
+    },
+    "mongoOpts": {
+      "mongos": {
+        "checkServerIdentity": {{ opts__mongoOpts__mongos__checkServerIdentity }},
+        "ssl": {{ opts__mongoOpts__mongos__ssl }},
+        "sslValidate": {{ opts__mongoOpts__mongos__sslValidate }}
+      },
+      "user": "{{ opts__mongoOpts__user }}",
+      "pass": "{{ opts__mongoOpts__pass }}"
+    },
+    "mongoUrl": "{{ opts__mongoUrl }}"
+  }
+}
+

dockerfiles/templates/renter.conf.tmpl

@@ -0,0 +1,42 @@
+{
+  "type": "Renter",
+  "opts": {
+    "mongoUrl": "{{ opts__mongoUrl }}",
+    "mongoOpts": {
+      "ssl": {{ opts__mongoOpts__ssl }},
+      "user": "{{ opts__mongoOpts__user }}",
+      "pass": "{{ opts__mongoOpts__pass }}",
+      "mongos": {
+        "checkServerIdentity": {{ opts__mongoOpts__mongos__checkServerIdentity }},
+        "ssl": {{ opts__mongoOpts__mongos__ssl }},
+        "sslValidate": {{ opts__mongoOpts__mongos__sslValidate }}
+      }
+    },
+    "logLevel": {{ opts__logLevel }},
+    "amqpUrl": "{{ opts__amqpUrl }}",
+    "amqpOpts": {
+    },
+    "migrationPrivateKey": "/etc/storj/keys/complexMigration.key",
+    "networkPrivateExtendedKey": "/etc/storj/keys/complex.key",
+    "networkOpts": {
+      "rpcPort": {{ opts__networkOpts__rpcPort }},
+      "rpcAddress": "{{ opts__networkOpts__rpcAddress }}",
+      "doNotTraverseNat": {{ opts__networkOpts__doNotTraverseNat }},
+      "tunnelServerPort": {{ opts__networkOpts__tunnelServerPort }},
+      "tunnelGatewayRange": {
+        "min": {{ opts__networkOpts__tunnelGatewayRange__min }},
+        "max": {{ opts__networkOpts__tunnelGatewayRange__max }}
+      },
+      "maxTunnels": {{ opts__networkOpts__maxTunnels }},
+      "seedList": [
+      ],
+      "bridgeUri": "{{ opts__networkOpts__bridgeUri",
+      "maxConnections": {{ opts__networkOpts__maxConnections }}
+    },
+    "totalRenters": {{ opts__totalRenters }},
+    "renterOverlap": {{ opts__renterOverlap }},
+    "migrationPrivateKeyString": "{{ opts__migrationPrivateKeyString }}",
+    "networkPrivateExtendedKeyString": "{{ opts__networkPrivateExtendedKeyString }}",
+    "networkIndex": {{ opts__networkIndex }}
+  }
+}

lib/config.js

@@ -24,11 +24,11 @@ function BaseConfig(config) {
   this._ = {};
 
   assert.ok(config, 'No config supplied');
-  assert(typeof config.logLevel === 'number', 'Invalid logLevel');
+  assert(typeof Number(config.logLevel) === 'number', 'Invalid logLevel');
   assert(typeof config.amqpUrl === 'string', 'Invalid amqpUrl');
   assert(typeof config.amqpOpts === 'object', 'Invalid amqpOpts');
 
-  this._.logLevel = config.logLevel;
+  this._.logLevel = Number(config.logLevel);
   this._.amqpUrl = config.amqpUrl;
   this._.amqpOpts = config.amqpOpts;
 }

package.json

@@ -12,7 +12,9 @@
     "coverage": "STORJ_ALLOW_LOOPBACK=1 NODE_ENV=test ./node_modules/.bin/istanbul cover ./node_modules/.bin/_mocha -- --recursive",
     "test": "npm run testsuite && npm run linter",
     "make-docs": "./node_modules/.bin/jsdoc index.js lib -r -R README.md -u ./doc -c .jsdoc.json --verbose -d ./jsdoc",
-    "publish-docs": "gh-pages -d jsdoc --repo git@github.com:Storj/complex.git"
+    "publish-docs": "gh-pages -d jsdoc --repo git@github.com:Storj/complex.git",
+    "start-renter": "./dockerfiles/scripts/setup-renter.sh ./bin/storj-complex.js -c /etc/storj/renter.conf",
+    "start-landlord": "./dockerfiles/scripts/setup-landlord.sh ./bin/storj-complex.js -c /etc/storj/landlord.conf"
   },
   "repository": {
     "type": "git",