Skip to content
Browse files

initial commit

  • Loading branch information...
0 parents commit 7311957ce007f3af922c837f4f40b30515cd1af9 @stomita committed May 23, 2012
17 .gitignore
@@ -0,0 +1,17 @@
+lib-cov
+*.seed
+*.log
+*.csv
+*.dat
+*.out
+*.pid
+*.gz
+
+pids
+logs
+results
+
+node_modules
+npm-debug.log
+
+.env
1 nodejs/Procfile
@@ -0,0 +1 @@
+web: node app.js
13 nodejs/README.md
@@ -0,0 +1,13 @@
+Heroku Screenshot Server: Node.js
+=======================
+
+Deploying to Heroku
+-----
+
+ $ heroku create --stack cedar
+ $ heroku config:add AWS_ACCESS_KEY_ID=<your aws access key id>
+ $ heroku config:add AWS_SECRET_ACCESS_KEY=<your aws secret access key>
+ $ heroku config:add UPLOAD_BUCKET_NAME=<aws s3 bucket name to store screenshots>
+ $ git push heroku master
+
+
2 nodejs/app.js
@@ -0,0 +1,2 @@
+require("coffee-script");
+require("./lib/app.coffee");
90 nodejs/lib/app.coffee
@@ -0,0 +1,90 @@
+###
+app.coffee
+###
+
+events = require "events"
+express = require "express"
+crypto = require "crypto"
+socketio = require "socket.io"
+s3upload = require "./s3upload"
+WorkerQueue = require "./queue"
+
+app = module.exports = express.createServer()
+
+###
+Express server configure
+###
+app.configure ->
+ app.set "views", __dirname + "/../views"
+ app.set "view engine", "ejs"
+ app.use express.bodyParser()
+ app.use express.methodOverride()
+ app.use app.router
+ app.use express.static(__dirname + "/../public")
+
+app.configure "development", ->
+ app.use express.errorHandler(
+ dumpExceptions: true
+ showStack: true
+ )
+
+app.configure "production", ->
+ app.use express.errorHandler()
+
+
+###
+SHA1 Hash Utility Function
+###
+sha1 = (str) ->
+ crypto.createHash('sha1').update(str).digest('hex')
+
+
+###
+Worker Queue
+###
+queue = new WorkerQueue()
+
+
+###
+Socket IO Channels
+###
+io = socketio.listen(app)
+
+channels =
+ request:
+ io.of("/request")
+ .on "connection", (socket) ->
+ console.log "<requester> connect"
+ socket.on "render", (url) ->
+ console.log "<requester> render #{url}"
+ hash = sha1(url)
+ queue.enqueue
+ url: url
+ hash: hash
+ form : s3upload.createForm(hash)
+ socket.on "disconnect", ->
+ console.log "<requester> disconnect"
+
+ render:
+ io.of("/render")
+ .on "connection", (socket) ->
+ console.log "<renderer> connect"
+ renderer = new events.EventEmitter()
+ renderer.on "dispatch", (req) -> socket.emit "render", req
+ queue.wait(renderer)
+ socket.on "complete", (response) ->
+ console.log "<renderer> notify #{response.imageUrl}"
+ channels.request.emit "image", response.imageUrl
+ queue.wait(renderer)
+ socket.on "fail", ->
+ queue.wait(renderer)
+ socket.on "disconnect", ->
+ console.log "<renderer> disconnect"
+ queue.remove(renderer)
+
+
+###
+Start server
+###
+app.listen process.env.PORT ? 3000
+console.log "Express server listening on port %d in %s mode", app.address().port, app.settings.env
35 nodejs/lib/queue.coffee
@@ -0,0 +1,35 @@
+events = require "events"
+###
+ Worker Queue
+###
+class WorkerQueue extends events.EventEmitter
+ maxWorkers: 100
+ maxRequests: 50
+
+ constructor: ->
+ @_requests = []
+ @_workers = []
+
+ wait: (worker) ->
+ request = @_requests.shift()
+ if request
+ worker.emit "dispatch", request
+ else if @maxWorkers > @_workers.length
+ @_workers.push(worker)
+ else
+ @emit "error", message: "Workers Limit Exceeded"
+
+ remove: (worker) ->
+ @_workers = (w for w in @_workers when w isnt worker)
+
+ enqueue: (request) ->
+ worker = @_workers.shift()
+ if worker
+ worker.emit "dispatch", request
+ else if @maxRequests > @_requests.length
+ @_requests.push(request)
+ else
+ @emit "error", message: "Request Limit Exceeded"
+
+
+module.exports = WorkerQueue
62 nodejs/lib/s3upload.coffee
@@ -0,0 +1,62 @@
+crypto = require "crypto"
+
+config =
+ aws:
+ accessKeyId: process.env.AWS_ACCESS_KEY_ID
+ secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
+ upload:
+ bucketName: process.env.UPLOAD_BUCKET_NAME
+ path: 'files/'
+ expiration: 5
+
+toISO8601 = (d) ->
+ pad2 = (n) -> (if n < 10 then '0' else '') + n
+ pad3 = (n) -> (if n < 10 then '00' else if n < 100 then '0' else '') + n
+ [
+ d.getUTCFullYear()
+ '-'
+ pad2(d.getUTCMonth() + 1)
+ '-'
+ pad2(d.getUTCDate())
+ 'T'
+ pad2(d.getUTCHours())
+ ':'
+ pad2(d.getUTCMinutes())
+ ':'
+ pad2(d.getUTCSeconds())
+ '.'
+ pad3(d.getUTCMilliseconds())
+ 'Z'
+ ].join('')
+
+
+module.exports =
+ createForm: (filename) ->
+ filePath = config.upload.path + filename + '.jpg'
+ policy =
+ expiration : toISO8601(new Date(Date.now() + 60000 * config.upload.expiration))
+ conditions : [
+ { bucket: config.upload.bucketName }
+ [ "starts-with", "$key", config.upload.path ]
+ { acl: "public-read" }
+ { success_action_status : "201" }
+ [ "starts-with", "$Content-Type", "image/" ]
+ [ "content-length-range", 0, 524288 ]
+ ]
+ policyB64 = new Buffer(JSON.stringify(policy)).toString('base64')
+ signature = crypto.createHmac('sha1', config.aws.secretAccessKey)
+ .update(policyB64)
+ .digest('base64')
+ {
+ action : "http://#{config.upload.bucketName}.s3.amazonaws.com/"
+ fields :
+ AWSAccessKeyId: config.aws.accessKeyId
+ key: filePath
+ acl: "public-read"
+ success_action_status: "201"
+ "Content-Type": "image/jpeg"
+ policy: policyB64
+ signature: signature
+ }
+
+
11 nodejs/package.json
@@ -0,0 +1,11 @@
+{
+ "name": "application-name"
+ , "version": "0.0.1"
+ , "private": true
+ , "dependencies": {
+ "express": "2.5.8"
+ , "ejs": ">= 0.0.1"
+ , "coffee-script": "1.2.x"
+ , "socket.io": "0.9.x"
+ }
+}
23 nodejs/public/js/render.js
@@ -0,0 +1,23 @@
+var socket;
+function init() {
+ socket = io.connect("/render");
+ socket.on("render", function(params) {
+ console.log("render:"+JSON.stringify(params));
+ });
+}
+
+function notifyComplete(pageUrl, imageUrl) {
+ socket.emit("complete", {
+ url: pageUrl,
+ imageUrl: imageUrl
+ });
+}
+
+function notifyFailure(pageUrl) {
+ socket.emit("failure", {
+ url: pageUrl
+ });
+}
+
+
+$(init);
20 nodejs/public/js/request.js
@@ -0,0 +1,20 @@
+var socket;
+function init() {
+ socket = io.connect("/request");
+ $('form').submit(function(e) {
+ e.preventDefault();
+ e.stopPropagation();
+ var url = $('#url').val();
+ socket.emit("render", url);
+ return false;
+ });
+ socket.on("image", function(imageUrl) {
+ $('<img>').attr('src', imageUrl).prependTo($('#images'));
+ if ($('img').size() > 10) {
+ $('img:last-child').remove();
+ }
+ });
+}
+
+$(init);
+
12 nodejs/public/render.html
@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html>
+ <head>
+ <script type="text/javascript" src="//ajax.googleapis.com/ajax/libs/jquery/1.7.2/jquery.min.js"></script>
+ <script type="text/javascript" src="/socket.io/socket.io.js"></script>
+ <script type="text/javascript" src="/js/render.js"></script>
+ </head>
+ <body>
+ <form>
+ </form>
+ </body>
+</html>
15 nodejs/public/request.html
@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html>
+ <head>
+ <script type="text/javascript" src="//ajax.googleapis.com/ajax/libs/jquery/1.7.2/jquery.min.js"></script>
+ <script type="text/javascript" src="/socket.io/socket.io.js"></script>
+ <script type="text/javascript" src="/js/request.js"></script>
+ </head>
+ <body>
+ <form>
+ URL to render: <input type="text" id="url">
+ <input type="submit" value="Submit">
+ </form>
+ <div id="images"></div>
+ </body>
+</html>
1 phantomjs/Procfile
@@ -0,0 +1 @@
+renderer: phantomjs screenshot.coffee $PUSH_SERVER_URL
12 phantomjs/README.md
@@ -0,0 +1,12 @@
+Heroku Screenshot Server: PhantomJS
+=======================
+
+Deploying to Heroku
+-----
+
+ $ heroku create --stack cedar --buildpack http://github.com/stomita/heroku-buildpack-phantomjs.git
+ $ heroku config:add PUSH_SERVER_URL=http://<app name of nodejs screenshot server>.herokuapp.com/render.html
+ $ git push heroku master
+ $ heroku ps:scale renderer=<num of phantomjs screenshot renderer>
+
+
146 phantomjs/screenshot.coffee
@@ -0,0 +1,146 @@
+fs = require('fs')
+sys = require('system')
+webpage = require('webpage')
+
+if sys.args.length < 2
+ console.log "Usage: phantomjs screenshot.coffee <push-server-url> [screen-width] [screen-height] [image-width] [image-height] [wait]"
+ return
+
+pushServerUrl = sys.args[1]
+screenSize =
+ width : sys.args[2] || 1024
+ height : sys.args[3] || 768
+imageSize =
+ width : sys.args[4] || 400
+ height : sys.args[5] || 300
+
+renderingWait = Number(sys.args[6] || 1000)
+
+###
+ Loading page
+###
+loadPage = (url, callback) ->
+ page = webpage.create()
+ page.viewportSize = screenSize
+ page.clipRect = { top: 0, left: 0, width: screenSize.width, height: screenSize.height }
+ page.onAlert = (msg) ->
+ console.log msg
+ page.onError = (msg, trace) ->
+ console.log msg
+ trace.forEach (item) -> console.log " ", item.file, ":", item.line
+ page.open url, (status) ->
+ callback (if status is "success" then page else null)
+ page
+
+
+###
+ Render and upload page image
+###
+renderPage = (url, filename, callback) ->
+ console.log "rendering #{url} to #{filename} ..."
+ loadPage url, (page) ->
+ return callback(null) unless page
+ setTimeout ->
+ page.evaluate -> document.documentElement.style.backgroundColor = '#fff'
+ page.render(filename)
+ callback(filename)
+ , 1000
+
+###
+ Resize image size
+###
+resizeImageFile = (srcFile, dstFile, imageSize, callback) ->
+ console.log "resizing #{srcFile} to #{dstFile}..."
+ page = webpage.create()
+ page.viewportSize = imageSize
+ page.clipRect = { left: 0, right: 0, width: imageSize.width, height: imageSize.height }
+ html = "<html><body style=\"margin:0;padding:0\">"
+ html += "<img src=\"file://#{srcFile}\" width=\"#{imageSize.width}\" height=\"#{imageSize.height}\">"
+ html += "</body></html>"
+ page.content = html
+ page.onLoadFinished = ->
+ page.render(dstFile)
+ callback(dstFile)
+
+###
+ Upload file using form
+###
+uploadFile = (file, form, callback) ->
+ console.log "uploading file #{file}..."
+ page = webpage.create()
+ html = "<html><body>"
+ html += "<form action=\"#{form.action}\" method=\"post\" enctype=\"multipart/form-data\">"
+ for n, v of form.fields
+ html += "<input type=\"hidden\" name=\"#{n}\" value=\"#{v}\" >"
+ html += "<input type=\"file\" name=\"file\" >"
+ html += "</form></body></html>"
+ page.content = html
+ page.uploadFile("input[name=file]", file)
+ page.evaluate -> document.forms[0].submit()
+ page.onLoadFinished = (status) ->
+ url = page.evaluate( -> location.href )
+ if url is form.action
+ page.onLoadFinished = null
+ console.log "uploading done."
+ loc = page.content.match(/<Location>(http[^<]+)<\/Location>/)
+ if loc
+ console.log "image location: #{loc[1]}"
+ callback loc[1]
+ else
+ callback null
+ page.release()
+
+###
+ Connecting to socket.IO push server
+###
+connect = (callback) ->
+ loadPage pushServerUrl, (page) ->
+ return conn(null) unless page
+ console.log "connected to #{pushServerUrl}"
+ conn = new Connection(page)
+ callback(conn)
+
+###
+ SocketIO server connection
+###
+class Connection
+ constructor: (@page) ->
+ page.onConsoleMessage = (msg) =>
+ console.log msg
+ return unless msg.indexOf "render:" is 0
+ try
+ request = JSON.parse(msg.substring(7))
+ @onRenderRequest?(request)
+ catch e
+
+ onRenderRequest: null
+
+ notify: (message) ->
+ if message is "complete"
+ args = Array.prototype.slice.call(arguments, 1)
+ @page.evaluate("function(){ notifyComplete('#{args.join("','")}'); }")
+ else
+ @page.evaluate("function(){ notifyFailure('#{args.join("','")}'); }")
+
+
+###
+ init
+###
+connect (conn) ->
+ return console.log("connection failure.") unless conn
+ conn.onRenderRequest = (request) ->
+ filename = Math.random().toString(36).substring(2)
+ captureFile = "/tmp/#{filename}.jpg"
+ imageFile = "/tmp/#{filename}_#{imageSize.width}x#{imageSize.height}.jpg"
+ renderPage request.url, captureFile, (captureFile) ->
+ console.log "captureFile #{captureFile}"
+ resizeImageFile captureFile, imageFile, imageSize, (imageFile) ->
+ uploadFile imageFile, request.form, (imageUrl) ->
+ if imageUrl
+ conn.notify("complete", request.url, imageUrl)
+ else
+ conn.notify("failure", request.url)
+ fs.remove(captureFile)
+ fs.remove(imageFile)
+
+

0 comments on commit 7311957

Please sign in to comment.
Something went wrong with that request. Please try again.