Permalink
Browse files

initial release

  • Loading branch information...
0 parents commit 94050d4e6ee7def608ad5dc08be061f55e4412cc @rootslab rootslab committed Mar 17, 2011
Showing with 1,036 additions and 0 deletions.
  1. +2 −0 .npmignore
  2. +5 −0 History.md
  3. +165 −0 Readme.md
  4. +1 −0 examples/Readme.md
  5. +133 −0 examples/testapp.js
  6. +2 −0 index.js
  7. +31 −0 lib/extensions.js
  8. +437 −0 lib/formaline.js
  9. +87 −0 lib/quickSearch.js
  10. +10 −0 package.json
  11. +162 −0 parser-benchmarks/QS-benchmark-test.js
  12. +1 −0 parser-benchmarks/Readme.md
2 .npmignore
@@ -0,0 +1,2 @@
+
+
5 History.md
@@ -0,0 +1,5 @@
+
+0.1.0 / 2011-16-03
+==================
+
+ * Initial release
165 Readme.md
@@ -0,0 +1,165 @@
+
+# formaline
+
+ formaline is a new (node.js) module for handling simple form posts and fast parsing of file uploads,
+ it is ready (I think) for integration with connect.js
+
+## Installation
+
+with npm:
+ $ npm install formaline
+
+with git:
+ $ git clone git://github.com/rootslab/formaline.git
+
+## Features
+
+ in progress..
+
+ - It is possible to create instances via configuration object
+ - Useful configuration parameters (like maxBytes..)
+ - Fluid exceptions handling
+ - Many events for total control of parsing flow
+ - Very Fast and Simple Parser (see parser-benchmarks)
+ - It is possible to preserve or auto-remove incomplete files upload, due to exceeding of max bytes limit
+ - It is possible to easily integrate with connect.js
+ - It is not perfect, who is ?
+
+ etc..
+
+ (the parsing data rate depends on many factor, it is possible to reach 700 MB/s and more with a *real* Buffer totally loaded in RAM and a basic ~60 bytes boundary string (like Firefox uses), is not the same thing with a real chunked data upload, as others libraries announce,)
+
+
+## Usage
+
+*module usage:*
+
+ var formaline = require('formaline');
+
+ var config = {
+
+ /*
+ temporary upload directory for file uploads
+ for every upload request a subdirectory is created that contains receiving files
+ default is /tmp/ -->
+ */
+
+ tmpUploadDir: '/var/www/upload/',
+
+ /*
+ default is false, or integer chunk factor,
+ every k chunk emits 'dataprogress' event start from first chunk ( 1+(0*k) 1+(1*k),1+(2*k),1+(3*k) ),
+ minimum multply factor value is 2 -->
+ */
+
+ emitDataProgress: !true, //3,10,100 (every k chunks)
+
+ /*
+ max bytes allowed, this is the max bytes writed to disk before stop to write
+ this is also true for serialzed fields not only for files upload -->
+ */
+
+ maxBytes: 3949000, //bytes ex.: 1024*1024*1024 , 512
+
+ /*
+ default false, bypass headers value, continue to write to disk
+ until maxBytes bytes are writed.
+ if true -> stop receiving data, when headers Content-Length exceeds maxBytes
+ */
+
+ blockOnReqHeaderContentLength: !true,
+
+ /*
+ remove file not completed due to maxBytes,
+ if true formaline emit fileremoved event,
+ otherwise return a path array of incomplete files when 'end' event is emitted
+ */
+
+ removeIncompleteFiles : true,
+
+ /*
+ enable various logging levels
+ it is possible to switch on/off one or more levels at the same time
+ debug: 'off' turn off logging
+ */
+
+ logging: 'debug:on,1:on,2:on,3:off' //string ex.: 'debug:on,1:off,2:on,3:off'
+
+ /*
+ listeners
+ */
+
+ listeners: {
+ 'warning': function(msg){
+ ...
+ },
+ 'headersexception': function ( isUpload, errmsg, res, next ) {
+ ...
+ next();
+ },
+ 'exception': function ( isUpload, errmsg, res, next ) {
+ ...
+ next();
+ },
+ 'filepathexception': function ( path, errmsg, res, next ) {
+ ...
+ next();
+ },
+ 'field': function ( fname, fvalue ) {
+ ...
+ },
+ 'filereceived': function ( filename, filedir, ctype, filesize ) {
+ ...
+ },
+ 'fileremoved': function ( filename, filedir ) {
+ ...
+ },
+ 'dataprogress': function ( bytesReceived, chunksReceived ) {
+ ...
+ },
+ 'end': function ( incompleteFiles, response, next ) {
+ response.writeHead(200, {'content-type': 'text/plain'});
+ response.end();
+ //next();
+ }
+ }
+ };
+
+ /*
+ create an instance of formaline with configuration object and parse req
+ */
+
+ new formaline( config ).parse( req, res, next );
+
+ *See Also :*
+ - examples directory for seeing formaline in action.
+ - parser-benchmarks directory for testing algorithm parsing speed (wow).
+
+## API
+
+ in progress..
+
+## License
+
+(The MIT License)
+
+Copyright (c) 2011 Guglielmo Ferri <44gatti@gmail.com>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
1 examples/Readme.md
@@ -0,0 +1 @@
+##TODO
133 examples/testapp.js
@@ -0,0 +1,133 @@
+
+
+var http = require('http'),
+ formaline = require('../lib/formaline').formaline,
+ connect = require('connect'),
+ server;
+
+var getHtmlForm = function(req, res,next) {
+ if (req.url === '/test/') {
+ res.writeHead(200, {'content-type': 'text/html'});
+ res.end('<b>Multiple File Upload:</b><br/><br/>\
+ <form action="/test/upload" enctype="multipart/form-data" method="post">\
+ <input type="text" name="title"><br>\
+ <input type="file" name="upload" multiple="multiple"><br>\
+ <input type="submit" value="Upload">\
+ </form><br/>\
+ <b>Simple Post:</b><br/><br/>\
+ <form action="/test/post" method="post">\
+ <input type="text" name="demofield1"><br>\
+ <input type="text" name="demofield2"><br>\
+ <input type="submit" value="Submit">\
+ </form>'
+ );
+ } else {
+ next();
+ }
+};
+var log = console.log,
+ receivedFiles,
+ removedFiles,
+ receivedFields;
+
+var handleFormRequest = function(req,res,next){
+ receivedFiles = {};
+ removedFiles = {};
+ receivedFields = {};
+ if ( (req.url === '/test/upload') || (req.url === '/test/post') ){
+ var config = {
+ //default is /tmp/ -->
+ //tmpUploadDir: '/var/www/demo/upload/',
+ // default is false, or integer chunk factor,
+ // every n chunk emit event 1+(0*n) 1+(1*n),1+(2*n),1+(3*n),
+ // minimum factor value is 2 -->
+ emitDataProgress: !true,//false,true,3,10,100
+ // max bytes allowed, this is the max bytes writed to disk before stop to write
+ // this is also true for serialzed fields not only for files upload -->
+ maxBytes: 3949000,//bytes ex.: 1024*1024*1024, 512
+ //default false, bypass headers value, continue to write to disk
+ //until maxBytes bytes are writed.
+ //if true -> stop receiving data, when headers content length exceeds maxBytes
+ blockOnReqHeaderContentLength: !true,
+ //remove file not completed due to maxBytes,
+ //if true formaline emit fileremoved event,
+ //otherwise return a path array of incomplete files
+ removeIncompleteFiles : true,
+ //listeners
+ listeners: {
+ 'warning': function(msg){
+ log('\n warning -->',msg);
+ },
+ 'headersexception': function(isUpload,errmsg,res,next){
+ log('\n headersexception -->',errmsg);
+ next();
+ },
+ 'exception': function(isUpload,errmsg,res,next){
+ log('\n exception --> ',errmsg);
+ next();
+ },
+ 'filepathexception': function(path,errmsg,res,next){//there is a file upload
+ log('\n filepathexception -->',path,'msg:',errmsg+'\n');
+ next();
+ },
+ 'field': function(fname,fvalue){
+ receivedFields[fname] = fvalue;
+ log('\n field--> ',fname,fvalue);
+ },
+ 'filereceived': function(filename,filedir,ctype,filesize) {
+ receivedFiles[filename] = { filedir: filedir, ctype: ctype, filesize: filesize };
+ log('\n filereceived --> name: '+filename+', path: '+filedir+', content type: '+ctype+', bytes: '+filesize+'\n');
+ },
+ 'fileremoved': function(filename,filedir) {
+ log('\n fileremoved --> name: '+filename+', path: '+filedir+'\n');
+ removedFiles[filename] = { filedir: filedir };
+ log('all files removed: '+removedFiles);
+ },
+ 'dataprogress': function(bytesReceived, chunksReceived) {
+ log('\n dataprogress --> bytes:', bytesReceived,'chunks:', chunksReceived);
+ },
+ 'end': function(incompleteFiles,response,next) {
+ log('\n-> Post Done');
+ response.writeHead(200, {'content-type': 'text/plain'});
+ response.write('-> all data received!\n');
+ response.write('\n-> upload root dir: '+config.tmpUploadDir+' \n');
+ response.write('-> max allowed bytes: '+config.maxBytes+' \n');
+ response.write('-> removeIncompleteFiles: '+config.removeIncompleteFiles+'\n');
+ response.write('-> emitDataProgress: '+config.emitDataProgress+'\n');
+ response.write('-> blockOnReqHeaderContentLength: '+config.blockOnReqHeaderContentLength+'\n');
+ response.write('\n-> fields received: '+JSON.stringify(receivedFields)+'\n');
+ response.write('-> files received: '+JSON.stringify(receivedFiles)+'\n');
+ if(config.removeIncompleteFiles ){
+ response.write('-> files removed : '+JSON.stringify(removedFiles)+'\n');
+ }else{
+ if( incompleteFiles.length !== 0 ){
+ response.write('-> incomplete files (not removed) : '+incompleteFiles+'\n');
+ }
+ }
+ receivedFiles = null;
+ removedFiles = null;
+ receivedFields = null;
+ response.end();
+ //next();//test
+ }
+ },
+ //enable various logging levels
+ //it is possible to switch on/off one or more levels at the same time
+ //debug: 'off' turn off logging
+ logging: 'debug:on,1:on,2:on,3:off'
+ };
+
+ new formaline(config).parse(req,res,next);
+
+ } else {
+ res.writeHead(404, {'content-type': 'text/plain'});
+ res.end('404');
+ }
+
+};
+
+server = connect( getHtmlForm , handleFormRequest, function(){console.log('\nSuccessfully call next() function');} );
+
+server.listen(3000);
+
+console.log('listening on http://localhost:3000/');
2 index.js
@@ -0,0 +1,2 @@
+
+module.exports = require('./lib/formaline').formaline;
31 lib/extensions.js
@@ -0,0 +1,31 @@
+
+//based on ExtCore createDelegate
+
+Function.prototype.createDelegate = function(obj, args, appendArgs){
+ var method = this;
+ return function() {
+ var callArgs = args || arguments;
+ if (appendArgs === true){
+ callArgs = Array.prototype.slice.call(arguments, 0);
+ callArgs = callArgs.concat(args);
+ }else if ( typeof appendArgs === 'number'){
+ callArgs = Array.prototype.slice.call(arguments, 0); // copy arguments first
+ var applyArgs = [appendArgs, 0].concat(args); // create method call params
+ Array.prototype.splice.apply(callArgs, applyArgs); // splice them in
+ }
+ return method.apply(obj || this, callArgs);
+ };
+};
+
+apply = function(obj, config){
+ if(obj && config && typeof config == 'object'){
+ for(var param in config){
+ if(typeof obj[param] !== 'undefined'){//apply only if property already exists in constructor
+ obj[param] = config[param];
+ }
+ }
+ }
+ return obj;
+};
+
+emptyFn = function(){};
437 lib/formaline.js
@@ -0,0 +1,437 @@
+
+/*!
+ * formaline
+ * Copyright(c) 2011 Guglielmo Ferri <44gatti@gmail.com>
+ * MIT Licensed
+ */
+
+/**
+ * Library version.
+ */
+
+exports.version = '0.1.0';
+
+var fs = require('fs'),
+ emitter = require('events').EventEmitter,
+ querystring = require('querystring'),
+ path = require('path'),
+ ext = require('./extensions'),
+ parser = require('./quickSearch');
+
+var setDebuggingLevel = function(dstring){
+ var p, dlevels = querystring.parse("debug:off,1:on,2:on,3:on", ',',':');//debug:'on' print always : 0 level and level not
+ if(dstring){
+ try{
+ p = querystring.parse( dstring, ',', ':' );
+ dlevels = p;
+ }catch(err){
+ console.log( 'formaline.setDebuggingLevel(): config string parse error ->', err.message );
+ }
+ }
+ return function(){
+ var args = Array.prototype.slice.call(arguments),//convert to array
+ level = args[0];
+ if( dlevels.debug === 'off' ){ return; }
+     if( typeof level === 'number' ){
+ if( ( level === 0 ) || ( dlevels[level] === 'on' )){
+ return console.log.apply(this,args.slice(1,args.length));
+ }
+ }else{
+ return console.log.apply(this,args);
+ }
+ };
+};
+
+
+var formaline = function (config){
+ emitter.call(this,[]);
+ this.req = null;
+ this.boundString = '';
+ this.boundBuffer = null;
+ this.qsBuffer = '';
+ this.chunked = false;
+ this.chunksReceived = 0;
+ this.currentChunksReceived = 0;//only for checking last chunk in dataProgress
+ this.bytesReceived = 0;
+ this.fileStream = null;
+ this.fileSize = 0;
+ this.totalMillis = 0;
+ this.tmpUploadDir = '/tmp/';
+ this.emitDataProgress = null;
+ this.maxBytes = 1024*1024*1024; //bytes
+ this.bytesWritedToDisk = 0;
+ this.blockOnReqHeaderContentLength = false;//block receiving data when ContentLength exceeds maxBytes ?
+ this.completedFiles = [];
+ this.incompleteFiles = [];
+ this.removeIncompleteFiles = !false;
+ this.listeners = {};
+ this.logging = null;
+ if(config && (typeof config === 'object')){
+ var me = this;
+ apply( this, config );
+ (function(){
+ var e, l = me.listeners;
+ for (e in l) {
+ if (typeof l[e] === 'function') {
+ me.on(e,l[e]);
+ }//else{
+ //me.on(p,emptyFn);
+ //}
+ }
+ })();
+ }
+ this.logger = setDebuggingLevel(this.logging);
+};
+
+formaline.prototype.__proto__ = emitter.prototype;
+
+fproto = formaline.prototype;
+
+fproto.parse = function( req, res, next ){
+ this.next = ( next && (typeof next === 'function')) ? next : emptyFn;//try to add connect layer compatibility
+
+ if( req && ( typeof req === 'object' ) && ( req.body === undefined ) && ( req.method === 'POST' || req.method === 'PUT' ) &&
+ ( ~req.headers['content-type'].indexOf('multipart/form-data' ) || ~req.headers['content-type'].indexOf('urlencoded' ) ) ){
+ this.req = req;
+ this.res = res;
+
+ //this.req.setEncoding('utf8');
+
+ try{ //check upload dir Existence
+ this.tmpUploadDir = this.tmpUploadDir+parseInt(new Date().getTime()*Math.random()*32,10)+'/';
+ fs.mkdirSync(this.tmpUploadDir,'0755');
+ //fs.mkdirSync(this.tmpUploadDir+'completed/','0755');
+ }catch(oerr){
+ this.logger(0,'\nformaline.parse(req,res) mkDir Warning-->', this.tmpUploadDir, ' msg: '+oerr.message);
+ this.tmpUploadDir = '/tmp/'+parseInt(new Date().getTime()*Math.random()*10*32,10)+'/';
+ this.emit('warning',oerr.message);
+ try{
+ fs.mkdirSync(this.tmpUploadDir,'0755');
+ }catch(ierr){
+ this.logger(0,'\nformaline.parse(req,res) default mkDir Error-->',this.tmpUploadDir,' msg:',ierr.message);
+ this.emit('filepathexception', this.tmpUploadDir, ierr.message, this.res, this.next);
+ return;
+ }
+ }
+
+ this.dataProgress = (function(){
+ var dProgress = this.emitDataProgress;
+ if( dProgress === true ){
+ return function(isEnd){
+ this.emit('dataprogress',this.bytesReceived,this.chunksReceived);//every chunks
+ };
+ }else if( typeof dProgress === 'number'){
+ dProgress = parseInt(dProgress,10);
+ if( dProgress < 2 ){ dProgress = 2; }
+ return function(isEnd){
+ if( ( ( this.chunksReceived % dProgress ) === 1 ) || isEnd ){// mod 1 is for first chunk
+ this.emit('dataprogress',this.bytesReceived,this.chunksReceived);//every dProgress chunks
+ }
+ };
+ }else{
+ return emptyFn;
+ }
+ }).createDelegate(this,[],true)();
+
+ var hs = req.headers,
+ clength = hs['content-length'],
+ ctype = hs['content-type'],
+ bytes2Receive = 0;
+
+
+ if( clength ){
+ try{
+ bytes2Receive = parseInt(clength,10);
+ if( bytes2Receive > this.maxBytes ){//check Max Upload Size in bytes
+ if(this.blockOnReqHeaderContentLength === true){
+ this.emit('headersexception',(ctype.indexOf('multipart/form-data' )!== -1) ? true : false ,'formaline.parse(req,res) req.headers[content-length] exceeds max allowable: '+bytes2Receive+' > '+this.maxBytes, this.res, this.next);
+ return;
+ }
+ //warning
+ this.logger(1,'\nformaline.parse(req,res) req.headers[content-length] --> Content Length Warning, bytes to receive:',bytes2Receive,'max allowed:',this.maxBytes);
+ this.emit('warning', 'Content Length Warning, bytes to receive: '+bytes2Receive+', allowed: '+this.maxBytes);
+ }else{
+ this.logger(0,'\nformaline.parse(req,res) req.headers[content-length] --> ', bytes2Receive);
+ }
+ }catch(err){
+ //error
+ this.emit('headersexception',(ctype.indexOf('multipart/form-data' )!== -1) ? true : false ,'formaline.parse(req,res) req.headers[content-length] parse error -> value is: ',clength, this.res, this.next);
+ this.logger(0,'\nformaline.parse(req,res) req.headers[content-length] --> Parse Length Exception', err);
+ return;
+ }
+ }else{
+ //error
+ this.emit('headersexception',(ctype.indexOf('multipart/form-data' )!== -1) ? true : false ,'formaline.parse(req,res) req.headers[content-length] not defined', this.res, this.next);
+ this.logger(0,'\nformaline.parse(req,res) req.headers[content-length] --> Parse Length Error');
+ return;
+ }
+
+ if( ctype ){
+ this.logger(0,'\nformaline.parse(req,res) req.headers[content-type] --> ', ctype);
+ if( ~ctype.indexOf('multipart/form-data') ){
+
+ this.boundString = ctype.match(/boundary=([^;]+)/mi)[1];
+ this.boundBuffer = new Buffer( '--' + this.boundString );
+
+ this.logger(1,'\nboundary pattern:'+this.boundString);
+ this.logger(1,'\nboundary length:'+this.boundString.length);
+
+ req.addListener('data',(function(chunk){
+ this.req.pause();//Pause req event (data,end..)
+
+ var bb = this.boundBuffer,
+ bblength = bb.length,
+ chunkLength = chunk.length,
+ stime = new Date(),
+ results = parser.quickSearch(bb,chunk),
+ etime = new Date(),
+ resultsLength = results.length;
+
+ this.totalMillis += ( etime - stime );
+ this.bytesReceived += chunk.length;
+ this.chunksReceived++;
+
+ this.dataProgress();//emit DataProgress
+
+ this.logger(3,'\nreceiving data-->');
+ this.logger(3,'chunk size:',chunk.length,'\n--> quickSearch results:\n',results);
+
+ if( this.bytesReceived <= this.maxBytes ){// is size allowed?
+ if( this.chunked && this.fileStream ){// file data is chunked? && fileStream exists?
+ if( resultsLength === 0 ){//chunk is only data payload
+ //write to file?
+ this.fileStream.write(chunk);
+ this.fileSize += chunk.length;
+ this.bytesWritedToDisk += chunk.length;
+ this.logger(3,'<--this chunk contains only data..\n');
+ }else{ //chunk contains other boundaries, the first result is the end of previous data chunk
+ var rstart = results[0].start;
+ var fileData = new Buffer( rstart - 2 ); //last two chars are CRLF
+ if( ( this.bytesWritedToDisk + fileData.length < this.maxBytes ) ){
+ try{
+ chunk.copy(fileData,0, 0, rstart - 2 );
+ this.fileStream.write(fileData);
+ }catch(err){
+ this.emit(true,'exception','formaline.parse(req,res) exception, copying buffer data file : '+this.fileStream.path+' --> '+err.message,this.res, this.res, this.next);
+ this.logger(0,'\nformaline.parse(req,res) exception -->', err);
+ return;
+ }
+ this.fileSize += fileData.length;
+ this.bytesWritedToDisk += fileData.length;
+ }
+ this.fileStream.end();
+ this.completedFiles.push(this.fileStream.path);
+ this.emit('filereceived',path.basename(this.fileStream.path),path.dirname(this.fileStream.path),this.fileStream.ctype,this.fileSize);
+ this.logger(3,'<-- this chunk contains data and fields..\n');
+ }
+ }
+ }else{
+ if( this.fileStream && (this.incompleteFiles.indexOf(this.fileStream.path) < 0) ){
+ this.incompleteFiles.push(this.fileStream.path);
+ this.emit('warning','formaline.parse(req,res) maxUploadsize exceeded, file incomplete -->'+path.basename(this.fileStream.path));
+
+ }
+ }
+
+ for( var i = 0; i < resultsLength; i++ ){
+ var result = results[i];
+ var rfinish = result.finish;
+ var rstart = result.start;
+ var fileData = null;
+
+ var heads = new Buffer( ( rfinish ) - ( rstart + bblength + 2) );//only the headers
+ chunk.copy(heads, 0, rstart + bblength + 2, rfinish );
+ var headers = heads.toString();
+ var fieldName = headers.match(/name="([^\"]+)"/mi);
+
+ if(fieldName){
+ var fileName = headers.match(/filename="([^;]+)"/mi);
+ var contentType = headers.match(/Content-Type:([^;]+)/mi);
+
+ if(fileName){//file field
+ this.logger(3,' ->field: '+fieldName[1]+', file: '+fileName[1]+', content-type: '+contentType[1]);
+ var filepath = this.tmpUploadDir + fileName[1];
+ if( this.completedFiles.indexOf(filepath) < 0 ){ //file with the same name already exists
+ this.fileStream = new fs.WriteStream( filepath );//,{ flags: 'w', encoding: null, mode: 0666 });
+ }else{
+ filepath = this.tmpUploadDir +(new Date().getTime())+'_'+fileName[1];
+ this.fileStream = new fs.WriteStream( filepath )
+ }
+ this.fileStream.ctype = contentType[1];//hack TODO
+ this.fileSize = 0;//reset fileSize
+ //chunkLength - "--", if matched boundary is not at the end of chunk, the last result field is chunked -->
+ if( i === resultsLength - 1 ) {//last result
+ if( rfinish < chunkLength - 2 ){ //there is no boundary at the end of chunk, it is data
+ this.logger(3,' -->', results[i], '<-- last field is chunked');
+ this.chunked = true;
+ //var fileData = new Buffer( chunkLength - ( rfinish + 4 ) );
+ if(this.fileStream){
+ if( this.bytesReceived <= this.maxBytes ){
+ try{
+ var fileData = new Buffer( chunkLength - ( rfinish + 4 ) );
+ chunk.copy(fileData, 0, rfinish + 4 ,chunkLength );
+ this.fileStream.write(fileData);
+ }catch(err){
+ this.emit(true,'exception','formaline.parse(req,res) exception, copying buffer datafile : '+this.fileStream.path+' --> '+err.message, this.res, this.next);
+ this.logger(0,'\nformaline.parse(req,res) exception -->', err);
+ return;
+ }
+ this.fileSize += fileData.length;
+ this.bytesWritedToDisk += fileData.length;
+ }else{
+ if( this.incompleteFiles.indexOf(this.fileStream.path) < 0 ){
+ this.incompleteFiles.push(this.fileStream.path);
+ this.emit('warning','formaline.parse(req,res) maxUploadsize exceeded, file incomplete -->'+path.basename(this.fileStream.path));
+ }
+ }
+ }
+ }
+ }else{
+ if(this.fileStream){
+ var fileData = new Buffer( results[i+1].start - 2 - ( results[i].finish + 4 ) );//+ CRLFCRLF
+ if( this.bytesWritedToDisk + fileData.length < this.maxBytes ){
+ try{
+ chunk.copy(fileData, 0, results[i].finish + 4 , results[i+1].start - 2 );
+ this.fileStream.write(fileData);
+ }catch(err){
+ this.emit(true,'exception','formaline.parse(req,res) exception, copying buffer datafile : '+this.fileStream.path+' --> '+err.message, this.res, this.next);
+ this.logger(0,'\nformaline.parse(req,res) exception -->', err);
+ return;
+ }
+ this.fileSize += fileData.length;
+ this.bytesWritedToDisk += fileData.length;
+ }else{
+ if( (this.incompleteFiles.indexOf(this.fileStream.path) < 0) ){
+ this.incompleteFiles.push(this.fileStream.path);
+ this.emit('warning','formaline.parse(req,res) maxUploadsize exceeded, file incomplete -->'+path.basename(this.fileStream.path));
+ }
+ }
+ //emit fileSize = 0 when file size exceeds maxUploadSize, or empty file
+ if( ( this.fileSize > 0 ) && (this.incompleteFiles.indexOf(this.fileStream.path) < 0 ) ){
+ this.completedFiles.push(this.fileStream.path);
+ this.emit('filereceived',path.basename(this.fileStream.path),path.dirname(this.fileStream.path),contentType[1],this.fileSize);
+ }
+ this.fileStream.end();
+ }
+ this.logger(3,'\n ->created file stream -->',this.fileStream.path,'\n');
+ }
+ }else{//normal field
+ if( i < resultsLength - 1 ){
+ try{
+ var fileData = new Buffer( results[i+1].start - 2 - ( results[i].finish + 4 ) );//+ CRLFCRLF
+ chunk.copy(fileData, 0, results[i].finish + 4 , results[i+1 ].start - 2 );
+ }catch(err){
+ this.emit(true,'exception','formaline.parse(req,res) exception, field: '+fieldName[1]+' --> '+err.message, this.res, this.next);
+ this.logger(0,'\nformaline.parse(req,res) exception -->', err);
+ }
+ this.logger(3,' ->field:',fieldName[1]+',',(contentType) ? 'content-type: '+contentType[1]+',' : 'no type,','data: *',fileData.toString(),'*');//is many times null for field
+ this.emit('field',fieldName[1],fileData.toString());
+ }
+ }
+ }
+ }//end for
+
+ req.resume();
+
+ }).createDelegate(this,[],true));
+
+ req.addListener('end',(function(nbytes){
+
+ this.logger(2,'\n[]------------PARSER STATS-------------[]\n chunks received:',this.chunksReceived);
+ this.logger(2, ' parsed:', (this.bytesReceived/(1024*1024)).toFixed(4), 'MB in', ( this.totalMillis / 1000 ), 'secs' );
+ this.logger(2, ' average data rate:', (( this.bytesReceived/(1024*1024) ) / ( this.totalMillis / 1000 )).toFixed(1), 'MB/sec' );
+ this.logger(2, ' average chunk size:', ((this.bytesReceived/1024)/this.chunksReceived ).toFixed(3), 'KBytes' );
+ this.logger(2, ' average chunk rate:', (( this.chunksReceived ) / ( this.totalMillis / 1000 )).toFixed(1), 'chunk/sec' );
+ this.logger(2,'[]-------------------------------------[]\n');
+ this.chunksReceived = 0;
+ this.bytesReceived = 0;
+ this.bytesWritedToDisk = 0;
+
+ this.fileStream = null;
+ this.boundString = null;
+ this.boundBuffer = null;
+ this.tmpUploadDir = '';
+ this.maxBytes = 0;
+
+ this.totalMillis = 0;
+ this.chunked = false;
+ this.fileSize = 0;
+ if(!this.removeIncompleteFiles === true){
+ this.emit('end', this.incompleteFiles, this.res, this.next);
+ }else{
+ if( this.incompleteFiles.length === 0 ){
+ this.emit('end', [], this.res, this.next);//incomplete files are already removed, previously it emits exception and fileremoved events
+ } else {
+ for(var i = 0, ufile = this.incompleteFiles, len = ufile.length, currfile = ufile[0]; i < len; i++, currfile = ufile[i] ){
+ fs.unlink(currfile,(function(err,cfile,i,len){
+ if(err){
+ this.logger(0,'\nformaline.parse(req,res) exception -->', err);
+ this.emit('warning','formaline.parse(req,res) exception, file unlink error: '+cfile+' --> '+err.message );
+ }else{
+ this.logger(0,'\nformaline.parse(req,res) incomplete file removed -->', cfile);
+ this.emit('fileremoved',path.basename(cfile),path.dirname(cfile));
+ }
+ if( i === len - 1){
+ this.emit('end', [], this.res, this.next);//incomplete files are already removed, previously it emits exception and fileremoved events
+ }
+ }).createDelegate(this,[currfile,i,len],true));
+ }
+ }
+ //this.emit('end', [], this.res, this.next);//incomplete files are already removed, previously it emits exception and fileremoved events
+ }
+ this.incompleteFiles = [];
+ this.completedFiles = [];
+ }).createDelegate(this,[clength],true));
+
+ }else if(~ctype.indexOf('urlencoded')){//serialized form
+ //check if size of data exceeds maxBytes
+ if( bytes2Receive > this.maxBytes ){
+ if(this.blockOnReqHeaderContentLength === true ){
+ this.emit('headersexception', false,'formaline.parse(req,res) req.headers[content-length] exceeds max allowable: '+bytes2Receive+' > '+this.maxBytes, this.res, this.next);
+ return;
+ } else {
+ //warning
+ this.logger(1,'\nformaline.parse(req,res) req.headers[content-length] --> Content Length Warning, bytes to receive:',bytes2Receive,'max allowed:',this.maxBytes);
+ this.emit('warning', 'formaline.parse(req,res), Content Length Warning, bytes to receive: '+bytes2Receive+', allowed: '+this.maxBytes);
+ }
+ }
+
+ req.addListener('data', (function(chunk){
+ this.qsBuffer += chunk.toString('utf8');
+ }).createDelegate(this,[],true));
+
+ req.addListener('end', (function(){
+ var fields = querystring.parse( this.qsBuffer, '&', '=' );
+ for( var f in fields ){
+ this.emit('field',f,fields[f]);
+ }
+ this.emit('end',[],this.res, this.next);
+ }).createDelegate( this, [], true ));
+
+ }else{
+ //error
+ this.emit('headersexception',false ,'formaline.parse(req,res) req.headers[content-type] --> '+ctype+' handler is not defined', this.res, this.next);
+ this.logger(0,'\nformaline.parse(req,res) req.headers[content-type] -->',ctype,' handler is not defined ');
+ return;
+ }
+ }else{
+ //error
+ this.emit('headersexception',false ,'formaline.parse(req,res) req.headers[content-type] is not defined', this.res, this.next);
+ this.logger(0,'\nformaline.parse(req,res) req.headers[content-type] --> Parse Type Error');
+ return;
+ }
+ }else{
+ //error
+ this.emit('headersexception',false ,'formaline.parse(req,res) req.headers are not defined', this.res, this.next);
+ this.logger(0,'\nformaline.parse(req,res) req.headers[..] --> no headers Error');
+ return;
+ }
+
+};//end parse
+
+exports.formaline = formaline;
+exports.parse = formaline;
+
+
+
+
87 lib/quickSearch.js
@@ -0,0 +1,87 @@
+
+var lookupTable = function(p){
+ var b = new Buffer(255),
+ m = p.length;
+ called = 0,
+ cmatches = 0;
+ for( var i = 0; i <= 255 ; b[i] = 0x00, i++ );
+ for( var j = 0; j < m ; b[p[j]] = m - j, j++ );
+ return {
+ containsChar: function(c){
+ return (b[(new Buffer(c))[0]] & 1);
+ },
+ getShift: function(ncode){
+ return ( b[ncode] || m + 1);// if 0 return max shift, max m = 254
+ },
+ contains: function(ncode){
+ return ( b[ncode] & 1);
+ },
+ getRawBuffer: function(){
+ return b;
+ }
+ };
+};
+
+
+var crlfcrlfMatch = function(t,l){ //stop at first match of CRLFCRLF, obviously.. l is start index
+ var p = new Buffer([13,10,13,10]), //4 bytes CRLFCRLF
+ m = p.length,
+ n = t.length,
+ s = l || 0,
+ ok = 1,
+ lkt = lookupTable(p),
+ lkb = lkt.getRawBuffer();
+ for( var j=s, z=0, x=p[0], pos=j, y=t[pos], i=m, c=t[i]; j<=n-m; i=j+m, c=t[i], z=0, ok = 1, pos=j, x=p[0], y=t[pos] ){
+ //console.log(z,pos,x,y,x===y); //matching sequence good for testing
+ for(; z<m; z++, pos++, x=p[z], y=t[pos]){
+ //console.log(z,pos,x,'-'+y+'-',x===y); //matching sequence good for testing
+ if( x === y ){ continue; }
+ else{ ok = 0; break; }
+ }
+ if( ok ){
+ //console.log('matched CRLFCRLF at:', j );
+ //console.log(' stripe:',t[j-5],t[j-4],t[j-3],t[j-2],t[j-1],'-',t[j],'-',t[j+1],t[j+2],t[j+3],t[j+4],t[j+5],t[j+6]);//,' j:',j);
+ break;
+ }
+ //console.log('CRLFCRLF --> j:',j,'c:',c,'jump:',lkb[c] || m + 1);
+ j += lkb[c] || m + 1;
+ }//end for
+ return j ; // j is the first char index of the crlfcrlf sequence
+};
+
+exports.quickSearch = function(p,t){
+ var n = t.length,
+ m = p.length,
+ pmatches = 0,
+ lkt = lookupTable(p),
+ lkb = lkt.getRawBuffer();
+ var result = {};
+ var arr = new Array();
+ //Every Boundary has two "--" than normal --(--Axc43434) end with --(--Axc43434)--
+ for( var j=0, ok=1, z=0, x=p[0], pos=0, y=t[pos], i=m, c=t[i]; j<=n-m; i=j+m, c=t[i], z=0, pmatches+=ok, ok=1, pos=j, x=p[0], y=t[pos]){
+ for(; z<m; z++, pos++, x=p[z], y=t[pos]){
+ //console.log(' z:',z,'pos:',pos,'x:',x,'y:',y,x===y); //matching sequence good for testing
+ if( x === y ){
+ //console.log(z,pos,x,y,x===y); //matching sequence good for testing
+ continue;
+ }
+ else{ ok = 0; break; }
+ }
+ if( ok ){
+ //console.log('matched BOUNDARY at:',j);
+ //console.log(' stripe:',t[j-5],t[j-4],t[j-3],t[j-2],t[j-1],'-',t[j],'-',t[j+1],t[j+2],t[j+3],t[j+4],t[j+5],t[j+6]);//,' j:',j);
+ var hs = crlfcrlfMatch( t, j + m + 2 );// +2 is for CRLF boundary\r\n
+ result = {
+ start: j ,
+ finish: hs || m
+ };
+ arr.push(result);
+ }
+ //console.log('QUICK --> j:',j,'c:',c,'jump:',lkb[c] || m + 1);
+ j += lkb[c] || m + 1;
+ }//end for
+ //console.log('QS-->',result,arr);
+ return arr;
+};
+
+
10 package.json
@@ -0,0 +1,10 @@
+{
+ "name": "formaline"
+ , "version": "0.1.0"
+ , "description": "fast node.js module for handling simple form posts and file uploads,ready for connect.js integration"
+ , "keywords": []
+ , "author": "Guglielmo Ferri <44gatti@gmail.com>"
+ , "dependencies": {}
+ , "main": "index"
+ , "engines": { "node": "0.4.x" }
+}
162 parser-benchmarks/QS-benchmark-test.js
@@ -0,0 +1,162 @@
+var log = console.log;
+
+var emitter = require('events').EventEmitter;
+
+var buildBuffer = function(p,MBsize,gapFactor){
+ var s = new Date(),
+ len = p.length,
+ gap = Math.pow(len,(gapFactor && gapFactor>1 ) ? gapFactor : 3),//power od len
+ //gap = parseInt(Math.log(len)*Math.log(len)*Math.log(len)*Math.log(len),10),
+ mb = 1024*1024,
+ size = MBsize || 700.1, // megabytes
+ tSize = parseInt( size * mb, 10 ),
+ logp = Math.log(len), //log b
+ logt = Math.log(tSize),//log a
+ logr = logt / logp,
+ maxLenPower = parseInt(logr,10); // maxLenPower = max power of len after which the 2nd pattern writed is out of buffer bound
+ log('\n ->\tMin Gap Factor: 2\n ->\tMax Gap Factor:',maxLenPower,'\n ->\tCurrent Gap Factor:', (gapFactor) ? gapfactor : 3 );
+
+ for( var i = 0, c = 1, t = new Buffer( tSize ); i + len < tSize; i += len ){
+ if( (i % (gap) ) === 0 ){
+ t.write(p.toString()+'\r\nContent-Disposition: form-data\r\nLorem Ipsum et Dolor sit amet, Quisquisce\r\n\r\n',i);
+ }else{
+ t[i] = i % 255;
+ }
+
+ }
+ log('\n ->\tpattern:',p.toString());
+ log(' ->\tmax pattern length: 254 bytes');
+ log(' ->\tpattern length:',len,'bytes');
+ log(' ->\tpattern gap:',gap,'bytes (distance in bytes of boundary occurrences)');
+ log(' ->\tplength / pgap:',len/gap,'\n');
+
+ var mtime = new Date() - s;
+ log(' ->\tbuffer size in MB:',t.length/1024/1024);
+ log(' ->\tbuffer creation time:',mtime/1000,'secs\n');
+ return t;
+};
+
+var lookupTable = function(p){
+ var b = new Buffer(255),
+ m = p.length;
+ called = 0,
+ cmatches = 0;
+ for( var i = 0; i <= 255 ; b[i] = 0x00, i++ );
+ for( var j = 0; j < m ; b[p[j]] = m - j, j++ );
+ return {
+ containsChar: function(c){
+ return (b[(new Buffer(c))[0]] & 1);
+ },
+ getShift: function(ncode){
+ return ( b[ncode] || m + 1);// if 0 return max shift, max m = 254
+ },
+ contains: function(ncode){
+ return ( b[ncode] & 1);
+ },
+ getRawBuffer: function(){
+ return b;
+ }
+ };
+};
+
+
+var quickSearch = function(p,t,cback){
+ var n = t.length,
+ m = p.length,
+ pmatches = 0,
+ cyc = 0,//outer cycles
+ ycy = 0,//inner cycles
+ end = null,
+ stats = {},
+ beg = new Date(),
+ lkt = lookupTable(p),
+ lkb = lkt.getRawBuffer();
+ var result = {};
+ var arr = new Array();
+
+ for( var j=0, ok=1, z=0, x=p[0], y=t[0], pos=0, i=m, c=t[i]; j<=n-m; i=j+m, c=t[i], z=0, pmatches+=ok, ok=1, pos=j, x=p[0], y=t[pos], cyc++ ){
+ for(; z<m; z++, pos++, x=p[z], y=t[pos], ycy++ ){
+ if( x === y ){ continue; }
+ else{ ok = 0; break; }
+ }
+ if( ok ){
+
+ var hs = crlfcrlfMatch( t, j + m + 2 );// +2 is for CRLF boundary\r\n
+ result = {
+ start: j ,
+ finish: hs || m
+ };
+ arr.push(result);
+
+ }
+ j += lkb[c] || m + 1;
+ }//end for
+ end = new Date();
+ stats = {
+ millis : end - beg,
+ tbytes : n,
+ pbytes: m,
+ matches: pmatches,
+ steps: cyc,
+ cycles: ycy,
+ results: arr
+ };
+ if(cback && ( typeof cback === 'function') ){ cback(stats); }
+};
+
+var crlfcrlfMatch = function(t,l){ //stop at first match obviously.. l is start index
+ var p = new Buffer([13,10,13,10]), //4 bytes CRLFCRLF
+ m = p.length,
+ n = t.length,
+ s = l || 0,
+ ok = 1,
+ cyc = 0,//outer cycles
+ ycy = 0,//inner cycles
+ lkt = lookupTable(p),
+ lkb = lkt.getRawBuffer();
+ for( var j=s, z=0, x=p[0], pos=j, y=t[pos], i=m, c=t[i]; j<=n-m; i=j+m, c=t[i], z=0, ok = 1, pos=j, x=p[0], y=t[pos], cyc++ ){
+ //log(z,pos,x,y,x===y); //matching sequence good for testing
+ for(; z<m; z++, pos++, x=p[z], y=t[pos], ycy++ ){
+ //log(z,pos,x,'-'+y+'-',x===y); //matching sequence good for testing
+ if( x === y ){ continue; }
+ else{ ok = 0; break; }
+ }
+ if( ok ){
+ //log('matched CRLFCRLF at:', j );
+ //log(' stripe:',t[j-5],t[j-4],t[j-3],t[j-2],t[j-1],'-',t[j],'-',t[j+1],t[j+2],t[j+3],t[j+4],t[j+5],t[j+6]);//,' j:',j);
+ break;
+ }
+ //log('CRLFCRLF --> j:',j,'c:',c,'jump:',lkb[c] || m + 1);
+ j += lkb[c] || m + 1;
+ }//end for
+
+ return j ;
+};
+
+
+
+var printStats = function( stats ){
+ //log( 'results:', stats.results );
+ log( ' ->\ttotal matches:', stats.matches );
+ log( ' ->\tstep cycles: '+stats.steps+'\n ->\tinner cycles: '+stats.cycles+'\n ->\ttext bytes: '+stats.tbytes+'\n ->\tpattern bytes: '+stats.pbytes );
+ log( ' ->\tmatching time:', ( stats.millis )/1000,'secs' );
+ log( ' ->\tparsing speed:',(stats.tbytes/stats.millis) / 1024, 'MB/s\n' );
+};
+
+
+var bsize,
+ gapfactor,
+ pattern = '---------------------------2046863043300497616870820724\r\n';
+
+process.argv.forEach(function (val, index, array) {
+ (index===2) ? (bsize = parseInt(val,10)) : null;
+ (index===3) ? (gapfactor = parseInt(val,10)) : null;
+ (index===4) ? (pattern = ((val.length > 1) && (val.length < 255) ) ? ('--'+val+'\r\n') : pattern ) : null;
+});
+
+
+var p = new Buffer( pattern ),// max 254 chars due to single byte use
+ t = buildBuffer( p, bsize, gapfactor );
+
+quickSearch( p, t, printStats );
+
1 parser-benchmarks/Readme.md
@@ -0,0 +1 @@
+##TODO

0 comments on commit 94050d4

Please sign in to comment.