Merge pull request mozilla#219 from Pomax/bug898463

noxmox as S3 emulation

README.md

@@ -24,28 +24,10 @@ git clone git@github.com:[yourname]/thimble.webmaker.org.git --recursive
 3a) you may need the XCode console tools (on OSX) or the VC++ express + windows SKD 7.1 stack (on Windows) in order for node-gyp to compile some npm dependencies
 3b) go into the thimble.webmaker.org dir and run ```npm install```
 
-4) as an optional step, when you don't want to test with a live AWS-S3 instance, you
-can set up fake-s3 to handle the S3 publication:
+4) set up the environment variables (see next section).
 
-```
-gem install fakes3
-mkdir fakes3
-```
-
-**NOTE:** this requires ruby. If you do not have this installed, visit http://ruby-lang.org
-
-5) set up the environment variables (see next section).
-
-You are now ready to run the app, by first starting up the sanitizer in the
-htmlsanitizer.org directory, by running ```python app.py```
-
-In addition, launch the fake S3 service in a terminal with:
-
-```
-fakes3 -r ./fakes3 -h localhost -p 6060
-```
+You can now run Thimble from the thimble.webmaker.org directory using
 
-You can then run Thimble from the thimble.webmaker.org directory using
 ```node app```
 
 Finally, there is a special variable that enables an additional route
@@ -67,26 +49,6 @@ cp env.dist .env
 
 and the Thimble code will pick up on it when run through node.
 
-**NOTE:** If you are using fakes3, you will need to make sure that
-[bucket].localhost points to localhost, which most likely requires
-you to add the following rule to your hosts file:
-
-```
-127.0.0.1 bucketName.localhost
-```
-
-If your bucket name is "test", then this rule should point to
-test.localhost, if your bucket name is "potato", point to potato.localhost,
-etc.
-
-Note that this file is used on all conventional operation systems, but
-lives in different places:
-
-* on *n*x, it is located at ```<systemroot>/etc/hosts```
-* on OSX, it is located at ```<systemroot>/private/etc/hosts```
-* on Windows, it is located at ```<systemroot>\system32\drivers\etc\hosts```
-
-
 Development additionals
 -----------------------
 
@@ -127,7 +89,6 @@ commands to make sure things work. This is mostly making sure that all the
 variables that are found in the `env.dist` file also exist in your heroku
 environment:
 
-
 ```
 > heroku config:set NODE_ENV="development"
 > heroku config:set HOSTNAME="htt:// ...heroku instance..."
@@ -152,26 +113,16 @@ Also, for Amazon S3, the following values are quite important:
 > heroku config:set S3_SECRET="your private S3 secret string"
 ```
 
-Unless you're using FakeS3, in which case you can scroll up to the FakeS3
-section, which explains the environment variables when using fake publishing.
-Although FakeS3 also gets reset when your app gets restarted, so if you
-want persistent data, it is strongly recommended to use a real S3 bucket.
+Note that when deploying to heroku, there will be no S3 emulation available.
 
 A note on credentials
 ---------------------
 
-The login credentials  in the `LOGINAPI` variable map to the `ALLOWED_USERS`
-variable used by the login instance that you rely on. This login regulates
-who can ask the login service for user information. It is not the list of
-"which persona user is allowed to access the login service".
+The login credentials  in the `LOGINAPI` variable map to the `ALLOWED_USERS` variable used by the login instance that you rely on. This login regulates who can ask the login service for user information. It is not the list of "which persona user is allowed to access the login service".
 
-The makeapi credentials map to the `ALLOWED_USERS` variable for the
-MakeAPI instance, and regulate who can query and push to the makeAPI.
-Again, this is not related to persona logins in any way.
+The makeapi credentials map to the `ALLOWED_USERS` variable for the MakeAPI instance, and regulate who can query and push to the makeAPI. Again, this is not related to persona logins in any way.
 
-Also note that the `SESSION_SECRET` environment variable is the secret
-that Thimble uses for setting its own local cookie, and can be any
-string you like (except an empty string).
+Also note that the `SESSION_SECRET` environment variable is the secret that Thimble uses for setting its own local cookie, and can be any string you like (except an empty string).
 
 New Relic
 ---------

app.js

@@ -29,6 +29,7 @@ var appName = "thimble",
     app = express(),
     env = new habitat(),
     node_env = env.get('NODE_ENV'),
+    emulate_s3 = env.get('S3_EMULATION') || !env.get('S3_KEY'),
     WWW_ROOT = path.resolve(__dirname, 'public'),
     /**
       We're using two databases here: the first is our normal database, the second is
@@ -269,3 +270,9 @@ require('webmaker-loginapi')(app, {
 app.listen(env.get("PORT"), function(){
   console.log('Express server listening on ' + env.get("HOSTNAME"));
 });
+
+// If we're in running in emulated S3 mode, run a mini
+// server for serving up the "s3" published content.
+if (emulate_s3) {
+  require("mox-server").runServer(env);
+}

env.dist

@@ -57,11 +57,13 @@ export DB_NAME=
 #export LEGACY_DB_NAME="LegacyThimble"
 
 # AWS-S3 information
-export S3_BUCKET="bucket.name.here"
-export S3_DOMAIN=
-export S3_PORT=
-export S3_KEY="your.s3.key.here"
-export S3_SECRET="your.s3.secret.here"
+export S3_BUCKET="local.publication.bucket"
+export S3_KEY=
+export S3_SECRET=
+
+# This value is only used for localhost testing with emulated S3
+export S3_DOMAIN="http://localhost:12319"
+export S3_EMULATION=true
 
 # Optional google analytics settings - if used, they will inject
 # the google analytics JS snippet with your information filled in.

lib/middleware.js

@@ -1,7 +1,9 @@
 module.exports = function middlewareConstructor(env) {
 
   var metrics = require('./metrics')(env),
-      utils = require("./utils");
+      utils = require("./utils"),
+      devmode = env.get("NODE_ENV") === "development",
+      knox = devmode ? require("noxmox").mox : require("knox");
 
   return {
     /**
@@ -232,7 +234,6 @@ module.exports = function middlewareConstructor(env) {
 
     generateUrls: function(appName, s3Url, domain) {
       var url = require("url"),
-          knox = require("knox"),
           s3 = knox.createClient(s3Url);
 
       return function(req, res, next) {
@@ -362,8 +363,7 @@ module.exports = function middlewareConstructor(env) {
       //       this line prevents knox from forming url-validation-failing S3 URLs.
       if(!options.port) { delete options.port; }
 
-      var knox = require("knox"),
-          s3 = knox.createClient(options);
+      var s3 = knox.createClient(options);
 
       return function(req, res, next) {
         var userId = req.session.username,

package.json

@@ -13,7 +13,9 @@
     "makeapi-client": "https://github.com/mozilla/makeapi-client/tarball/v0.5.6",
     "less-middleware": "0.1.12",
     "newrelic": "0.9.20",
+    "mox-server": "https://github.com/Pomax/mox-server/archive/master.tar.gz",
     "node-statsd": "0.0.7",
+    "noxmox": "0.2.3",
     "nunjucks": "0.1.10",
     "webmaker-i18n": "https://github.com/mozilla/node-webmaker-i18n/archive/v0.1.1.tar.gz",
     "request": "2.21.0",