Formidable doesn't seem's to close open files uploaded from web workers #255

Open
Elanouette opened this Issue Sep 8, 2013 · 0 comments

Projects

None yet

1 participant

@Elanouette

Hi,

I have an issue with Node-Formidable but I don’t know if it’s a bug or if it’s related to the way I’m trying to use it. I would need some help to find it out.

I’m using web workers to chunk and upload large file. As FormData() isn’t available in web workers, I have to send the chunk directly inside the xhr command.

Node-Formidable receive and write the chunk to disk but it seem’s to leave the files stream open because I’m hitting the node process ulimit and it crashes throwing the EMFILE error…

On my system I have the error at around 230 chunks as my ulimit is 256. This doesn’t append when I using FormData directly from the .html file but I cannot using it that way because it create others issue related to high CPU usage by the browser after the upload is complete and web workers solve that.

You can reproduce by using this code and a file large enough to generate more chunks that your uLimit :

app.js :

var http = require('http');
var formidable = require('formidable');
var fs = require('fs');
var path = require('path');
var i = 0;

http.createServer(function(req, res){
    if(req.method.toLowerCase() == 'get'){
        if(req.url != '/favicon.ico'){
            res.writeHead(200, {'Content-Type' : resolveType(req)});

            var url = __dirname;

            if(req.url == '/'){
                url += '/views/index.html';
            }
            else {
                url += '/public' + req.url;
            }

            var readStream = fs.createReadStream(url).pipe(res).on('error', function(o_O){
                console.log(o_O);
            });
        }
        else{
            res.end();
        }
    }

    if(req.method.toLowerCase() == 'post' && req.url == '/upload'){     
        var form = new formidable.IncomingForm({
            uploadDir: './tmp/'
        });

        form.parse(req, function(err, fields, files){
            if (err){
                console.error(err.message);
                return;
            }

            if (files.file){
                console.log('Upload complete - Chunk '+ i +' file : '+ files.file.path);
                i += 1;
            }       

            res.writeHead(200, {'content-type': 'text/plain'});
            res.end('upload complete : chunk #'+ fields.chunk);
        });
    }
}).listen(3000, function(){
    console.log("Express server listening on port 3000");
});

var resolveType = function(req){
    var extension = path.extname(req.url);

    if(extension == 'js'){
        return 'text/javascript';
    }
    else{
        return 'text/html';
    }
};

/views/index.html :

<html xmlns="http://www.w3.org/1999/xhtml">
    <head>
        <meta http-equiv="content-type" content="text/html;charset=ISO-8859-1">
    </head>

    <body>
        <div>
            <input type="file" id="afile" class="button">
        </div>

        <script type="text/javascript">
            var worker = new Worker('/worker.js');

            worker.onmessage = function(e) {
                console.log(e.data);
            };

            worker.onerror = function(e) {
                console.log(JSON.stringify(e));
            };

            document.querySelector('input[type="file"]').addEventListener('change', function(e) {
                worker.postMessage(this.files);
            }, false);      
        </script>   
    </body>
</html>

/public/worker.js :

var uploaders=[];

self.onmessage = function(e){

    var blob=e.data[0];
    var slice = blob.slice || blob.webkitSlice || blob.mozSlice;

    const BYTES_PER_CHUNK= 131072; //128kb chunk so a 40MB files should trigger the EMFILE error for a uLimit at 256.
    const SIZE=blob.size;

    var start=0;
    var end=BYTES_PER_CHUNK;

    while(start<SIZE){
        upload(slice.call(blob, start,end));

        start=end;
        end=start+BYTES_PER_CHUNK;
    }
}

function upload(blobOrFile){
    var last = 0;
    var newChunk = true;

    var xhr=new XMLHttpRequest();

    xhr.open('POST', '/upload', true);
    xhr.setRequestHeader("Content-Type", "application/octet-stream");

    xhr.onload=function(e){
        uploaders.pop();

        if(!uploaders.length){
            self.postMessage('UPLOAD DONE!!!');
            self.close()
        }
    };

    uploaders.push(xhr);
    xhr.send(blobOrFile);
}

What are my options to have node formidable to close these files after the upload ?

Thank you,

Eric

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment