Browse files

upload-progress and streaming uploads

  • Loading branch information...
1 parent d86d757 commit 29c4a113d5e71a356c159a068b519898a334d7aa Martin Schuhfuss committed Sep 25, 2012
Showing with 146 additions and 180 deletions.
  1. +89 −0 lib/middleware/eyefiMultipart.js
  2. +0 −104 lib/middleware/multipart.js
  3. +2 −2 lib/middleware/soapParser.js
  4. +2 −44 lib/routes/upload.js
  5. +2 −2 lib/server.js
  6. +31 −23 package.json
  7. +20 −5 standalone.js
View
89 lib/middleware/eyefiMultipart.js
@@ -0,0 +1,89 @@
+// based on Connect.multipart-middleware, extended to handle tar-uploads
+var formidable = require('formidable'),
+ _limit = require('express').limit,
+ tar = require('tar'),
+ path = require('path'),
+ fs = require('fs'),
+
+ qs = require('qs');
+
+function mime(req) {
+ var str = req.headers['content-type'] || '';
+ return str.split(';')[0];
+};
+
+exports = module.exports = function (config) {
+ var logger = config.logger,
+ uploadPath = config.uploadPath;
+
+ return function multipart(req, res, next) {
+ if (req._body) return next();
+
+ req.body = req.body || {};
+
+ // ignore GET, check Content-Type
+ if ('GET' == req.method || 'HEAD' == req.method) return next();
+ if ('multipart/form-data' != mime(req)) return next();
+
+ // flag as parsed
+ req._body = true;
+
+ // parse
+ var form = new formidable.IncomingForm, fields = {}, done;
+
+ // file-uploads from eye-fi cards are always tar-files, so we may save a bit of
+ // disk-roundtrips by parsing them right away…
+ form.onPart = function(part) {
+ if(!part.filename || part.mime !== 'application/x-tar') {
+ return form.handlePart(part);
+ }
+
+ part.pipe(tar.Parse())
+ .on('error', next)
+ .on('entry', function __onTarEntry(entry) {
+ var outfile = path.join(uploadPath, entry.props.path);
+ entry.pipe(fs.createWriteStream(outfile));
+ entry.on('end', function __onTarEntryEnd() {
+ logger.log('file extraction complete', outfile);
+
+ req.app.emit('imageReceived', { filename: outfile });
+ });
+ });
+ };
+
+ form.on('progress', function(bytesReceived, bytesExpected) {
+ req.app.emit('uploadProgress', { received: bytesReceived, expected: bytesExpected });
+ });
+
+ form.on('field', function(name, val) {
+ if (Array.isArray(fields[name])) {
+ fields[name].push(val);
+ } else if (fields[name]) {
+ fields[name] = [fields[name], val];
+ } else {
+ fields[name] = val;
+ }
+ });
+
+ form.on('error', function(err) {
+ next(err);
+ done = true;
+ });
+
+ form.on('end', function() {
+ // next() already called by the error-event
+ if (done) return;
+
+ try {
+ req.body = qs.parse(fields);
+
+ console.log(req.body);
+ next();
+ } catch (err) {
+ next(err);
+ }
+ });
+
+ form.parse(req);
+ }
+};
View
104 lib/middleware/multipart.js
@@ -1,104 +0,0 @@
-/*!
- * adapted
- *
- * Connect - multipart
- * Copyright(c) 2010 Sencha Inc.
- * Copyright(c) 2011 TJ Holowaychuk
- * MIT Licensed
- */
-
-var formidable = require('formidable'),
- _limit = require('express').limit,
-
- qs = require('qs');
-
-/**
- * noop middleware.
- */
-
-function noop(req, res, next) { next(); }
-
-function mime(req) {
- var str = req.headers['content-type'] || '';
- return str.split(';')[0];
-};
-
-exports = module.exports = function (options) {
- options = options || {};
-
- var limit = options.limit
- ? _limit(options.limit)
- : noop;
-
- return function multipart(req, res, next) {
- if (req._body) return next();
-
- req.body = req.body || {};
- req.files = req.files || {};
-
- // ignore GET
- if ('GET' == req.method || 'HEAD' == req.method) return next();
-
- // check Content-Type
- if ('multipart/form-data' != mime(req)) return next();
-
- // flag as parsed
- req._body = true;
-
- // parse
- limit(req, res, function (err) {
- if (err) return next(err);
-
- var form = new formidable.IncomingForm
- , data = {}
- , files = {}
- , done;
-
- Object.keys(options).forEach(function (key) {
- form[key] = options[key];
- });
-
- function ondata(name, val, data) {
- if (Array.isArray(data[name])) {
- data[name].push(val);
- } else if (data[name]) {
- data[name] = [data[name], val];
- } else {
- data[name] = val;
- }
- }
-
- form.on('progress', function(bytesReceived, bytesExpected) {
- req.app.emit('uploadProgress', { received: bytesReceived, expected: bytesExpected });
- });
-
- form.on('field', function (name, val) {
- ondata(name, val, data);
- });
-
- form.on('file', function (name, val) {
- ondata(name, val, files);
- });
-
- form.on('error', function (err) {
- next(err);
- done = true;
- });
-
- form.on('end', function () {
- // next() already called by the error-event
- if (done) return;
-
- try {
- req.body = qs.parse(data);
- req.files = qs.parse(files);
- next();
- } catch (err) {
- next(err);
- }
- });
-
- form.parse(req);
- });
- }
-};
View
4 lib/middleware/soapParser.js
@@ -25,9 +25,9 @@ module.exports = function(config) {
req.soap = {};
req.soap.action = req.headers.soapaction.slice(1,-1); // soap-action comes quoted
- // in case of file-uploads, the request was already handled by the bodyParser,
+ // in case of file-uploads, the request was already handled by the multipart-middleware,
// otherwise the buffer is still readable and we need to collect the data first…
- if(!req.readable) {
+ if(req._body) {
logger.debug('parsing xml-string', req.body.SOAPENVELOPE);
parser.parseString(req.body.SOAPENVELOPE, xmlParsingComplete);
View
46 lib/routes/upload.js
@@ -1,50 +1,8 @@
"use strict";
-var tar = require('tar'),
- fs = require('fs'),
- path = require('path');
-
module.exports = function(config) {
- var logger = config.logger;
-
+ // uploads are handled by the middleware, so we're done here...
return function __uploadHandler(req, res) {
- if(!req.soap || req.soap.err) {
- logger.error('error in request-data.');
-
- return res.end();
- }
-
- if(!req.files.FILENAME) {
- logger.error('missing uploaded file.');
-
- return res.end();
- }
-
- var reqData = req.soap.data["SOAP-ENV:Body"]["ns1:UploadPhoto"],
- uploadPath = config.uploadPath;
-
- logger.debug('got upload-request', reqData);
-
- fs.createReadStream(req.files.FILENAME.path)
- .pipe(tar.Parse())
- .on("error", function (err) {
- logger.error("an error occurred while extracting uploaded data: " + err);
-
- res.end();
- })
- .on("entry", function (entry) {
- var outfile = path.join(uploadPath, entry.props.path);
- entry.pipe(fs.createWriteStream(outfile));
- entry.on("end", function () {
- logger.log('file extraction complete', outfile);
-
- req.app.emit('imageReceived', { filename: outfile });
- });
- })
- .on("end", function () {
- logger.debug('sending upload-response');
-
- res.render('uploadSuccess');
- });
+ res.render('uploadSuccess');
};
};
View
4 lib/server.js
@@ -2,7 +2,7 @@ var express = require('express'),
routes = require('./routes'),
soapParser = require('./middleware/soapParser'),
logger = require('./middleware/logger'),
- multipart = require('./middleware/multipart.js');
+ multipart = require('./middleware/eyefiMultipart');
exports = module.exports = createServer;
@@ -31,7 +31,7 @@ function createServer(config) {
app.set('view engine', 'xml');
app.set('view options', { layout: false });
- app.use(multipart());
+ app.use(multipart(config));
// the cards do not set the soapaction-header when uploading files.
app.use(function(req,res,next) {
View
54 package.json
@@ -1,25 +1,33 @@
{
- "name": "eyefi",
- "version": "0.1.1",
- "description": "integrate image-uploads from eye-fi cards into your application",
- "main": "index.js",
- "repository": {
- "type": "git",
- "url": "http://github.com/usefulthink/node-eyefi.git"
- },
- "homepage": "http://github.com/usefulthink/node-eyefi",
- "keywords": [ "eye-fi", "eyefi", "image-server", "photography" ],
- "author": {
- "name": "usefulthink",
- "email": "m.schuhfuss@gmail.com",
- "url": "http://github.com/usefulthink"
- },
- "license": "WTFPL",
- "dependencies": {
- "express": ">=3.0",
- "ejs": ">=0.8",
- "xml2js": ">=0.2",
- "tar": ">=0.1",
- "devnull": "*"
- }
+ "name": "eyefi",
+ "version": "0.1.1",
+ "description": "integrate image-uploads from eye-fi cards into your application",
+ "main": "index.js",
+ "repository": {
+ "type": "git",
+ "url": "http://github.com/usefulthink/node-eyefi.git"
+ },
+ "homepage": "http://github.com/usefulthink/node-eyefi",
+ "keywords": [
+ "eye-fi",
+ "eyefi",
+ "image-server",
+ "photography"
+ ],
+ "author": {
+ "name": "usefulthink",
+ "email": "m.schuhfuss@gmail.com",
+ "url": "http://github.com/usefulthink"
+ },
+ "license": "WTFPL",
+ "dependencies": {
+ "express": ">=3.0",
+ "ejs": ">=0.8",
+ "xml2js": ">=0.2",
+ "tar": ">=0.1",
+ "devnull": "*",
+ "formidable": "~1.0.11",
+ "qs": "~0.5.1",
+ "bytes": "~0.1.0"
+ }
}
View
25 standalone.js
@@ -3,18 +3,33 @@
var fs = require('fs'),
path = require('path'),
eyefi = require('./lib/server'),
+ bytes = require('bytes'),
Logger = require('devnull'),
config = JSON.parse(fs.readFileSync(path.join(__dirname, 'config.json')));
-config.logger = new Logger({
- level: 6
+var logger = config.logger = new Logger({
+ level: 9
});
var eyefiServer = eyefi(config).start();
-config.logger.notice("eyefi-server started and listening");
+logger.notice("eyefi-server started and listening");
-eyefiServer.on('imageReceived', function(imgData) {
- config.logger.notice('received an image: ' + imgData.filename);
+eyefiServer.on('imageReceived', function __logImageReceived(imgData) {
+ logger.notice('received an image: ' + imgData.filename);
+});
+
+var progressLoggingPaused = false;
+eyefiServer.on('uploadProgress', function(progressData) {
+ var br=progressData.received || 0,
+ be=progressData.expected || 0,
+ pct=(100*br/be).toFixed(2);
+
+ if(!progressLoggingPaused || br==be) {
+ logger.metric('progress: ' + bytes(br) + ' of ' + bytes(be) + '(' + pct + '%)');
+
+ progressLoggingPaused = true;
+ setTimeout(function() { progressLoggingPaused=false; }, 100);
+ }
});

0 comments on commit 29c4a11

Please sign in to comment.