diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d344ba6 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +config.json diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..bed14e2 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "blob_fs/node-mongodb-native"] + path = blob_fs/node-mongodb-native + url = https://github.com/christkv/node-mongodb-native.git +[submodule "blob_s3/sax-js"] + path = blob_s3/sax-js + url = https://github.com/isaacs/sax-js.git diff --git a/README.md b/README.md new file mode 100644 index 0000000..cbfdcc1 --- /dev/null +++ b/README.md @@ -0,0 +1,151 @@ +# vblob-node + + Node blob gateway service. + +## Features + + - RESTful web service + - Plugin model (currently support local fs, s3) + - streaming in/out blobs + - basic blob operations: create/delete/get/copy blobs; create/list/delete buckets; query with prefix/delimiter/marker/max-keys; virtual folders + - user defined meta data + - S3 compatibility + - virtualized buckets (automatically mapping buckets to different backends) + +## Authors + + - Sonic Wang (wangs@vmware.com) + +## Dependency + +### Common Modules + + - Express, web framework + - Winston, logging module + +### Local FS driver: + + - node-mongodb-native: https://github.com/christkv/node-mongodb-native.git (use latest source code, no npm install) + - A mongo db service with a provisioned user account + +#### mongo setup + - start mongo: `./mongod -f mongodb.config` (in bin folder - adjust to correct config file location - config file points to db location) + - note port on startup (must match option.mds.port in config.json) + - start mongo console: `./mongo` in bin folder + - change database, e.g.: `use test` (must match option.mds.db in fs driver part of config.json) + - setup user account, e.g.: `db.addUser('user1', 'password1')` (must match option.mds.user and option.mds.pwd) + + +### Amazon S3 driver: + - sax-js xml parser: https://github.com/isaacs/sax-js.git + - Amazon S3 storage account (a valid id/key pair) + +## Configuration + + config.json + +The above file contains a stringtified JSON object describing supported driver details. + + { + "drivers":[ + {"fs-sonic" : { + "type" : "fs", + "option" : {"root" : "/home/sonicwcl/Workspace/data2", + "mds" : { + "host" : "127.0.0.1", + "port" : "28100", + "db" : "test2", + "user" : "sonic1", + "pwd" : "password1" + } + } + } + }, + {"s3-sonic" : { + "type" : "s3", + "option" : { + "key" : "dummy", + "secret" : "dummy" + } + } + } + ], + "port" : "8080", + "default" : "fs", + "logfile" : "/tmp/log" + } + +Each driver must specify its type. Currently `fs` and `s3` are supported. The `option` value contains the neccessary information a driver needs. For `fs`, the value contains root directory for storing blobs; host/port/db/user/password for mongodb. For `s3`, the values contains a pair of s3 key and secret. `default` means the default driver type. When there are naming conflicts, the driver appears first in the array will be chosen. `logfile` specifies the path to log file. + +## Usage + + node server.js [-f path_to_config_file] + +Currently supported drivers are `fs` and `s3`. If no default driver is specified in configure file, the first one appears in configuration file will be the default one. Otherwise, the specified one is the default driver. The default driver is currently used to determine in which backend a new bucket should be created. + +After the gateway service starts, gateway will periodically detect buckets in all the backends and create a mapping in memory. A request to a particular bucket will be correctly routed to corresponding backend. Currently any client could send http RESTful requests to the server. + +### Listing buckets + + curl http://localhost:3000/ --verbose + +### Listing a bucket + + curl http://localhost:3000/container1/ --verbose + +One could add a query to the URL. Currently four criteria are supported: prefix; delimiter; marker; max-keys. E.g.: + + curl "http://localhost:3000/container1/?prefix=A/&delimiter=/" --verbose + +The above query will also list virtual folders in result as well. + +### Create a bucket + + curl http://localhost:3000/container1 --request PUT --verbose + +### Delete a bucket + + curl http://localhost:3000/container1 --request DELETE --verbose + +### Uploading a file + + curl http://localhost:3000/container1/file1.txt --request PUT -T file1.txt --verbose + +Currently user-defined meta data is supported. All user meta keys start with prefix `x-amz-meta-`. E.g.: + + curl http://localhost:3000/container1/file1.txt --requst PUT -T file1.txt -H "x-amz-meta-comment:hello_world" + +### Copying a file + + curl http://localhost:3000/container1/file1.txt --request PUT -H "x-amz-copy-source:/container2/file2.txt" + +The above request will direct gateway to copy file2.txt in container2 to file1.txt in container1. Currently only intra-driver copy is supported. This means both container1 and container2 must be within the same driver(backend). This operation will copy meta data as well. All user-defined meta data in file2.txt will be copied to file1.txt. + +This operation will return code `200`. In addition, the response body includes a JSON format object. It has two fields: `LastModified`, and `ETag`. + +### Deleting a file + + curl http://localhost:3000/container1/file1.txt --request DELETE --verbose + +### Reading a file + + curl http://localhost:3000/container1/file1.txt --verbose + +Currently additional header `range` is supported for single range read as well. Thus user can issue something like this: + + curl http://localhost:3000/container1/file1.txt -H "range:bytes=123-892" --verbose + +## S3 compatibility + +There is strong demand for an S3 compatibility. Thus we implement front end APIs to be S3 compatible. This means urls, headers and request bodies are all S3 compatible. At the same time, responses will be S3 compatible as well. This means response headers and bodies are all S3 compatible. + +In order not to make this project a pure S3 emulator effort, we will restrict the compatibility to a subset of all S3 features. + +Currently all responses (except blob streams) are of JSON format. This is an intermediate representation. We convert S3 XML format to a JSON equivalent one. We use this as a reference model for other drivers (currently local fs). That is, every driver returns responses in a JSON format that is able to be converted to S3 XML without losing any information. + +## Server Tuning + +When gateway is handling a great amount of concurrent requests, it may open too many file descriptors. It is suggested to increase the file descriptor limit. E.g.: in linux one may type + + ulimit -n 16384 + diff --git a/blob_fs/blob_fs.js b/blob_fs/blob_fs.js new file mode 100644 index 0000000..aebcd1f --- /dev/null +++ b/blob_fs/blob_fs.js @@ -0,0 +1,940 @@ +/* + Author: wangs@vmware.com + Require additional library node-mongodb-native: https://github.com/christkv/node-mongodb-native.git + Do not use NPM to install that lib, version's too old; use the latest source code + Set the root dir of the blob, e.g.: var fb = new FS_blob("/mnt/sdb1/tmp"); + A mongo db service is needed for storing meta data. + start a mongod process, and create a user id/pwd + Need winston module for logging +*/ +var winston = require('winston'); +var fs = require('fs'); +var Path = require('path'); +var crypto = require('crypto'); +var util = require('util'); +var events = require("events"); +var mongo_path = "./node-mongodb-native/lib/mongodb"; +var Db = require( mongo_path ).Db; +var Connection = require( mongo_path ).Connection; +var Server = require( mongo_path ).Server; +var BSON = require( mongo_path ).BSONNative; +var PREFIX_LENGTH = 2; //how many chars we use for hash prefixes +var MAX_LIST_LENGTH = 1000; //max number of objects to list +var base64_char_table = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +function hex_val(ch) +{ + if (48 <= ch && ch <= 57) { return ch - 48; } + return ch - 97 + 10; +} + +function hex2base64(hex_str) +{ + hex_str = hex_str.toLowerCase(); + var result = ""; + var va = new Array(8); + var ca = new Array(8); + for (var idx = 0; idx < hex_str.length; ) + { + for (var idx2 = 0; idx2 < 6; idx2++) + { + if (idx+idx2 < hex_str.length) { + va[idx2] = hex_str.charCodeAt(idx2+idx); + va[idx2] = hex_val(va[idx2]); + } else { va[idx2] = 0; } + } + ca[0] = base64_char_table.charAt((va[0] << 2) + (va[1] >> 2)); + ca[1] = base64_char_table.charAt(((va[1]&0x03)<<4)+va[2]); + ca[2] = base64_char_table.charAt((va[3] << 2) + (va[4] >> 2)); + ca[3] = base64_char_table.charAt(((va[4]&0x03)<<4)+va[5]); + if (idx + 5 < hex_str.length) { + //normal case + result += (ca[0]+ca[1]+ca[2]+ca[3]); + } else if (idx + 3 < hex_str.length) { + //padding 1 + result += (ca[0]+ca[1]+ca[2]+"="); + } else { + //padding 2 + result += (ca[0]+ca[1]+"=="); + } + idx += 6; + } + return result; +} + +function common_header() +{ + var header = {}; + header.Connection = "close"; + header["content-type"] = "application/json"; + var dates = new Date(); + header.date = dates.toString(); + header.Server = "FS"; + header["x-amz-request-id"] = "1D2E3A4D5B6E7E8F9"; //No actual request id for now + header["x-amz-id-2"] = "3F+E1E4B1D5A9E2DB6E5E3F5D8E9"; //no actual request id 2 + return header; +} + +function error_msg(statusCode,code,msg,resp) +{ + resp.resp_code = statusCode; + resp.resp_header = common_header(); + resp.resp_body = {"Error":{ + "Code": code, + "Message" : ( msg && msg.toString )? msg.toString() : "" + }}; + //no additional info for now +} + +function FS_blob(root_path,mds_cred,callback) //fow now no encryption for fs +{ + this.root_path = root_path; + var host = mds_cred.host; + var port = mds_cred.port; + var this1 = this; + this.MDS = null; + var db = new Db(mds_cred.db, new Server(host, port, {}), {native_parser:true}); + db.open(function(err, db) { + if (err) { + winston.log('error',(new Date())+' - Please make sure mongodb host and port are correct!'); + throw err; + } + db.authenticate(mds_cred.user,mds_cred.pwd,function(err,res) { + if (!err) { winston.log('info',(new Date())+" - connectd to mongo");} else { + winston.log('error',(new Date())+' - Please use correct credentials!'); + throw err; + } + this1.MDS = db; + if (callback) { callback(this1); } + }); + }); +} + +FS_blob.prototype.create_container = function(container,resp) +{ + var c_path = this.root_path + "/" + container; + if (Path.existsSync(c_path) === false) + { + winston.log('debug',(new Date())+" - path "+c_path+" does not exist! Let's create one"); + fs.mkdirSync(c_path,"0777"); + } else + { + winston.log('debug',(new Date())+" - path "+c_path+" exists!"); + } + resp.resp_code = 200; + var header = common_header(); + delete header["content-type"]; + header["content-length"] = 0; + header.location = '/'+container; + resp.resp_header = header; + resp.resp_end(); +}; + +FS_blob.prototype.create_container_meta = function(container,resp,fb) +{ + var dTime = new Date(); + fb.MDS.collection(container, {safe:true},function(err,coll) { + if (coll) {winston.log('debug',"container "+container+" exists!"); + resp.resp_code = 200; + var header = common_header(); + delete header["content-type"]; + header["content-length"] = 0; + header.location = '/' + container; + resp.resp_header = header; + resp.resp_end(); + return; + } + fb.MDS.createCollection(container, function (err,col) { + if (err) { winston.log('error',(new Date())+" - container creation error! "+err); + error_msg(500,"InternalError",err,resp); resp.resp_end(); + return; + } + col.insert({ + "vblob_container_name" : container, + "vblob_create_time" : dTime.toString(), + "vblob_update_time" : dTime.toString() + }, {safe:true}, function (err, item) { + if (!err) { + winston.log('debug'+(new Date())+" - Inserted item "+util.inspect(item)+" to db"); + col.ensureIndex("vblob_file_name", function(err,resp) { } ); + col.ensureIndex("vblob_container_name", function(err,resp) {} ); //for quickly locating collection info + fb.create_container(container,resp); + } + else { + winston.log('error',(new Date())+" - Insertion failed for container "+container); + error_msg(500,"InternalError",err,resp); resp.resp_end(); + fb.MDS.dropCollection(container, function(err,result) {} ); + } + }); + }); + }); +}; + +FS_blob.prototype.delete_container_meta = function(container,resp) +{ + winston.log('debug',(new Date())+" - deleting "+container); + this.MDS.dropCollection(container,function (err,result) { + if (err) { + winston.log("error",(new Date())+" - deleting container "+container+" err! "+err); + error_msg(500,"InternalError",err,resp); resp.resp_end(); return; + } + else { winston.log("debug",(new Date())+" - deleted container "+container); } + var header = common_header(); + delete header["content-type"]; + resp.resp_code = 204; resp.resp_header = header; + resp.resp_end(); + }); +}; + +//delete a container; fail if it's not empty +FS_blob.prototype.delete_container = function(container,resp,fb) +{ + var c_path = this.root_path + "/" + container; + if (Path.existsSync(c_path) === false) + { + error_msg(404,"NoSuchBucket","No such bucket on disk",resp); resp.resp_end(); return; + } + fs.rmdir(c_path,function(err) { + if (err) { error_msg(409,"BucketNotEmpty","The bucket you tried to delete is not empty.",resp); resp.resp_end(); return; } + fb.delete_container_meta(container,resp); + }); +}; +/* + complete name: /container/filename + physical path: /container/prefix/filename + prefix calculaton: prefix of PREFIX_LENGTH chars of md5 digest of filename + TODO: store uploaded file to a temporary place, and rename it afterwards +*/ + +//supress warning: no making functions inside loops +function remove_uploaded_file(fdir_path,esp_name) //folder and file name +{ + fs.unlink(fdir_path+"/"+esp_name,function(err) { + winston.log('error',(new Date())+" - Error in deleting upload file: " + err); + fs.rmdir(fdir_path,function(err2) {}); + }); +} + +FS_blob.prototype.create_file = function (container,filename,data,resp,fb) +{ + if (resp === undefined) { resp = null; } + var c_path = this.root_path + "/" + container; + if (!Path.existsSync(c_path)) { + winston.log("error",(new Date())+" - no such container"); + error_msg(404,"NoSuchBucket","No such bucket on disk",resp); resp.resp_end(); return; + } + var file_path = c_path + "/" + filename; //complete representation: /container/filename + var md5_name = crypto.createHash('md5'); + //md5_name.update(file_path); + md5_name.update(filename); //de-couple from root and container paths + var name_digest = md5_name.digest('hex'); + winston.log('debug',(new Date())+" - create: the md5 hash for string "+filename+" is "+name_digest); + var name_dig_pre = name_digest.substr(0,PREFIX_LENGTH); + var fdir_path = c_path + "/" + name_dig_pre; //actual dir for the file /container/hashprefix/ + if (!Path.existsSync(fdir_path)) { //create such folder + fs.mkdirSync(fdir_path,"0777"); + } + var esp_name = filename.replace(/\//g,"$_$"); + var stream = fs.createWriteStream(fdir_path+"/"+esp_name); + var md5_etag = crypto.createHash('md5'); + var md5_base64 = null; + var file_size = 0; + stream.on("error", function (err) { + winston.log('error',(new Date())+" - write stream " + filename+err); + if (resp !== null) { + error_msg(500,"InternalError",err,resp); resp.resp_end(); + } + data.destroy(); + stream.destroy(); + fs.unlink(fdir_path+"/"+esp_name,function(err) { + fs.rmdir(fdir_path,function(err) { } ); + }); + return; + }); + data.on("error", function (err) { + if (resp !== null) { + error_msg(500,"InternalError",err,resp); resp.resp_end(); + } + winston.log('error',(new Date())+' - input stream '+filename+err); + data.destroy(); + stream.destroy(); + fs.unlink(fdir_path+"/"+esp_name,function(err) { + fs.rmdir(fdir_path,function(err) {}); + }); + return; + }); + data.on("data",function (chunk) { + md5_etag.update(chunk); + file_size += chunk.length; + stream.write(chunk); + }); + data.on("end", function () { + winston.log('debug',(new Date())+' - upload ends'); + data.upload_end = true; + stream.end(); + stream.destroySoon(); + //md5_etag = md5_etag.digest('hex'); + //fb.create_file_meta(container,filename,{"vblob_file_etag":md5_etag,"vblob_file_size":file_size},res,fb); + }); + stream.on("close", function() { + winston.log('debug',(new Date())+" - close write stream "+filename); + md5_etag = md5_etag.digest('hex'); + var opts = {"vblob_file_etag":md5_etag,"vblob_file_size":file_size}; + var keys = Object.keys(data.headers); + for (var idx = 0; idx < keys.length; idx++) { + var obj_key = keys[idx]; + if (obj_key.match(/^x-amz-meta-/i)) { + var sub_key = obj_key.substr(11); + sub_key = "vblob_meta_" + sub_key; + opts[sub_key] = data.headers[obj_key]; + } else if (obj_key.match(/^content-md5$/i)) { + //check if content-md5 matches + md5_base64 = hex2base64(md5_etag); + if (md5_base64 !== data.headers[obj_key]) // does not match + { + if (resp !== null) { + error_msg(400,"InvalidDigest","The Content-MD5 you specified was invalid.",resp); resp.resp_end(); + } + winston.log('error',(new Date())+' - '+filename+' md5 not match: uploaded: '+ md5_base64 + ' specified: ' + data.headers[obj_key]); + data.destroy(); + remove_uploaded_file(fdir_path,esp_name); + return; + } + } else if (obj_key.match(/^content-type$/i)) { + opts[obj_key.toLowerCase()] = data.headers[obj_key]; + } + } + if (!data.connection && data.headers.vblob_create_time) + { opts.vblob_create_time = data.headers.vblob_create_time; } + fb.create_file_meta(container,filename,opts,resp,fb,!data.connection); + }); + if (data.connection) // copy stream does not have connection + { + data.connection.on('close',function() { + winston.log('debug',(new Date())+' - client disconnect'); + if (data.upload_end === true) { return; } + winston.log('warn',(new Date())+' - interrupted upload: ' + filename); + data.destroy(); + stream.destroy(); + fs.unlink(fdir_path+"/"+esp_name, function() { + fs.rmdir(c_path,function(err) {}); + }); + return; + }); + } +}; + +FS_blob.prototype.create_file_meta = function (container, filename, opt,resp,fb,is_copy) +{ + if (opt === undefined) { opt = null; } + if (resp === undefined) { resp = null; } + var doc = {}; + if (opt !== null) { + for (var key in opt) + { doc[key] = opt[key]; } + } + var dDate = new Date(); + if (!doc.vblob_create_time) //special consideration for copy + { doc.vblob_create_time = dDate.toString(); } + doc.vblob_update_time = dDate.toString(); + doc.vblob_file_name = filename; + fb.MDS.collection(container, {safe:true}, function (err,coll) { + if (err || !coll) { + winston.log('error',(new Date())+" - In creating file "+filename+" meta in container "+container+" "+err); + if (resp !== null) { + error_msg(404,"NoSuchBucket",err,resp); + resp.resp_end(); + } + fb.delete_file(container,filename,null); + return; + } + coll.findOne({"vblob_file_name":filename}, function (err, obj) { + if (err) { + if (resp !== null) { error_msg(500,"InternalError",err,resp); resp.resp_end();} + fb.delete_file_meta(container,filename,null,fb); + return; + } + if (!obj) { + //new object + coll.insert(doc, function(err,docs) { + if (err) { + winston.log('error',(new Date())+" - In creating file "+filename+" meta in container "+container+" "+err); + if (resp !== null) { error_msg(500,"InternalError",err,resp); resp.resp_end(); } + fb.delete_file_meta(container,filename,null,fb); + return; + } else { + winston.log('debug',(new Date())+" - Created meta for file "+filename+" in container "+container); + var header = common_header(); + header.ETag = opt.vblob_file_etag; + resp.resp_code = 200; + winston.log('debug',(new Date())+' - is_copy: ' + is_copy); + if (is_copy) { + resp.resp_body = ('{"CopyObjectResult":')+('{"LastModified":"'+doc.vblob_update_time+'",') + ('"ETag":"'+opt.vblob_file_etag+'"}}'); + header["content-length"] = resp.resp_body.length; + resp.resp_header = header; + } else { + delete header["content-type"]; + header["content-length"] = 0; + resp.resp_header = header; + } + resp.resp_end(); + } + }); + } else { + //update + delete doc.vblob_create_time; + var u_doc = {}; + //we need to get rid of all user-defined meta (coz this is an overwrite) + if (true) { + var keys = Object.keys(obj); + for (var idx = 0; idx < keys.length; idx++) { + var obj_key = keys[idx]; + if (!obj_key.match(/^vblob_meta_/i)) { continue; } + if (doc[obj_key]) { continue; } //don't delete the one to be inserted! + u_doc[obj_key] = 1; + } + } + coll.update({"_id":obj._id},{$set:doc, $unset:u_doc}, function (err, cnt) { + if (err) { + winston.log('error',(new Date())+" - In creating file "+filename+" meta in container "+container+" "+err); + if (resp !== null) { error_msg(500,"InternalError",err,resp); resp.resp_end(); } + fb.delete_file_meta(container,filename,null,fb); + return; + } else { + winston.log('debug',(new Date())+" - Created meta for file "+filename+" in container "+container); + var header = common_header(); + header.ETag = opt.vblob_file_etag; + resp.resp_code = 200; + winston.log('debug',(new Date())+' - is_copy: ' + is_copy); + if (is_copy) { + resp.resp_body = ('{"CopyObjectResult":')+('{"LastModified":"'+doc.vblob_update_time+'",') + ('"ETag":"'+opt.vblob_file_etag+'"}}'); + header["content-length"] = resp.resp_body.length; + resp.resp_header = header; + } else { + delete header["content-type"]; + header["content-length"] = 0; + resp.resp_header = header; + } + resp.resp_end(); + } + }); + } + }); + }); +}; + +FS_blob.prototype.delete_file_meta = function (container, filename, resp, fb) +{ + if (resp === undefined) { resp = null; } + fb.MDS.collection(container, {safe:true}, function (err,coll) { + if (err || !coll) { winston.log('error',(new Date())+" - In deleting file "+filename+" meta in container "+container+" "+err); if (resp!== null) { error_msg(404,"NoSuchBucket",err,resp); resp.resp_end();} return; } + coll.remove({vblob_file_name:filename}, function(err,docs) { + if (err) { + winston.log('error',(new Date())+" - In deleting file "+filename+" meta in container "+container+" "+err); + if (resp !== null) { + error_msg(404,"NoSuchFile",err,resp); + resp.resp_end(); + } + return; + } + else { winston.log('debug',(new Date())+" - Deleted meta for file "+filename+" in container "+container); } + fb.delete_file(container,filename,resp); + }); + }); +}; + +FS_blob.prototype.delete_file = function (container, filename, resp) +{ + if (resp === undefined) { resp = null; } + var c_path = this.root_path + "/" + container; + var file_path = c_path + "/" + filename; //complete representation: /container/filename + var md5_name = crypto.createHash('md5'); + //md5_name.update(file_path); + md5_name.update(filename); //de-couple from root and container paths + var name_digest = md5_name.digest('hex'); + winston.log('debug',(new Date())+" - delete: the md5 hash for "+filename+" is "+name_digest); + var name_dig_pre = name_digest.substr(0,PREFIX_LENGTH); + var fdir_path = c_path + "/" + name_dig_pre; //actual dir for the file /container/hashprefix/ + if (!Path.existsSync(fdir_path)) { //check such folder + if (resp !== null) { + //var dDate = new Date(); res.header("Date",dDate.toString()); res.send(404); + error_msg(404,"NoSuchFile","File does not exists on Disk",resp); + resp.resp_end(); + return; + } + return; + } + var esp_name = filename.replace(/\//g,"$_$"); + fs.unlink(fdir_path+"/"+esp_name, function (err) { + if (err) { winston.log('error',+(new Date())+" - Deleting file "+err); if(resp !== null) {error_msg(500,"InternalError",err,resp); resp.resp_end(); resp = null; } } + fs.rmdir(fdir_path, function(err) { + if (resp !== null) { + //resp.writeHeader(204,common_header()); resp.end(); + resp.resp_code = 204; + var header = common_header(); + delete header["content-type"]; + resp.resp_header = header; + resp.resp_end(); + } + }); + }); +}; + +FS_blob.prototype.copy_file = function (dest_c,dest_f,container,filename,requ,resp,fb) +{ + var c_path = this.root_path + "/" + container; + if (!Path.existsSync(c_path)) { + error_msg(404,"NoSuchBucket","No such container",resp);resp.resp_end();return; + } + var file_path = c_path + "/" + filename; //complete representation: /container/filename + var md5_name = crypto.createHash('md5'); + //md5_name.update(file_path); + md5_name.update(filename); //de-couple from root and container paths + var name_digest = md5_name.digest('hex'); + winston.log('debug',(new Date()) + " - copy: the md5 hash for "+filename+" is "+name_digest); + var name_dig_pre = name_digest.substr(0,PREFIX_LENGTH); + var fdir_path = c_path + "/" + name_dig_pre; //actual dir for the file /container/hashprefix/ + if (!Path.existsSync(fdir_path)) { //check such folder + error_msg(404,"NoSuchFile","No such file",resp);resp.resp_end();return; + } + var esp_name = filename.replace(/\//g,"$_$"); + var file_size = fs.statSync(fdir_path+"/"+esp_name).size; + if (file_size === undefined) { + error_msg(404,"NoSuchFile","No such file",resp);resp.resp_end(); return; + } + var etag_match=null, etag_none_match=null, date_modified=null, date_unmodified=null; + var meta_dir=null; + if (true){ + var keys = Object.keys(requ.headers); + for (var idx = 0; idx < keys.length; idx++) + { + if (keys[idx].match(/^x-amz-copy-source-if-match$/i)) + { etag_match = requ.headers[keys[idx]]; } + else if (keys[idx].match(/^x-amz-copy-source-if-none-match$/i)) + { etag_none_match = requ.headers[keys[idx]]; } + else if (keys[idx].match(/^x-amz-copy-source-if-unmodified-since$/i)) + { date_unmodified = requ.headers[keys[idx]]; } + else if (keys[idx].match(/^x-amz-copy-source-if-modified-since$/i)) + { date_modified = requ.headers[keys[idx]]; } + else if (keys[idx].match(/^x-amz-metadata-directive$/i)) + { meta_dir = requ.headers[keys[idx]]; } + } + } + if (meta_dir === null) { meta_dir = 'COPY'; } + else { meta_dir = meta_dir.toUpperCase(); } + if ((meta_dir !== 'COPY' && meta_dir !== 'REPLACE') || + (etag_match && date_modified) || + (etag_none_match && date_unmodified) || + (date_modified && date_unmodified) || + (etag_match && etag_none_match) ) { + error_msg(400,"NotImplemented","The headers are not supported",resp); + resp.resp_end(); return; + } + //read meta here + this.MDS.collection(container,{safe:true},function(err,coll) { + if (err||!coll) { error_msg(404,"NoSuchBucket",err,resp); resp.resp_end(); return; } + coll.findOne({"vblob_file_name":filename}, function (err, obj) { + if (err||!obj) { + error_msg(404,"NoSuchFile",err,resp); resp.resp_end(); return; + } + if (file_size !== obj.vblob_file_size) { + error_msg(500,"InternalError","file corrupted",resp); resp.resp_end(); return; + } + //check etag, last modified + var check_modified = true; + var t1,t2; + if (date_modified) { + t1 = new Date(date_modified).valueOf(); + t2 = new Date(obj.vblob_update_time).valueOf(); + check_modified = t2 > t1; + } else if (date_unmodified) { + t1 = new Date(date_unmodified).valueOf(); + t2 = new Date(obj.vblob_update_time).valueOf(); + check_modified = t2 <= t1; + } + if ((etag_match && obj.vblob_file_etag !== etag_match) || + (etag_none_match && obj.vblob_file_etag === etag_none_match) || + check_modified === false) + { + error_msg(412,"PreconditionFailed","At least one of the preconditions you specified did not hold.",resp); resp.resp_end(); return; + } + var keys,keys2; var idx; //supress warning + if (dest_c !== container || dest_f !== filename) { + var st = fs.createReadStream(fdir_path+"/"+esp_name); + st.on('error',function(err) { throw err;}); + requ.headers.vblob_create_time = obj.vblob_create_time; + keys = Object.keys(obj); + if (meta_dir === 'COPY') { + //delete request meta headers, add object's + keys2 = Object.keys(requ.headers); + for (var idx2 = 0; idx2 < keys2.length; idx2++) { + if (keys2[idx2].match(/^x-amz-meta-/i)) + { delete requ.headers[keys2[idx2]]; } + } + for (idx = 0; idx < keys.length; idx++) { + var key = keys[idx]; + if (key.match(/^vblob_meta_/i)) { + var key2 = key.replace(/^vblob_meta_/i,"x-amz-meta-"); + if (requ.headers[key2]) { continue; } + requ.headers[key2] = obj[key]; + } + } + } + st.headers = requ.headers; + fb.create_file(dest_c,dest_f,st,resp,fb); + } else {//copy self to self, only update meta, replace meta only, use only meta in header + if (meta_dir === 'COPY') { + error_msg(400,"NotImplemented","The headers are not supported",resp); + resp.resp_end(); return; + } + var opts = {"vblob_file_etag":obj.vblob_file_etag,"vblob_file_size":file_size}; + keys = Object.keys(obj); + //overwrite here + keys = Object.keys(requ.headers); + for (idx = 0; idx < keys.length; idx++) { + var obj_key = keys[idx]; + if (obj_key.match(/^x-amz-meta-/i)) { + var key3 = obj_key.replace(/^x-amz-meta-/i,"vblob_meta_"); + opts[key3] = requ.headers[obj_key]; + } + } + fb.create_file_meta(dest_c,dest_f,opts,resp,fb,true); + } + }); + }); +}; + +FS_blob.prototype.read_file = function (container, filename, range,requ,resp,verb) +{ + var c_path = this.root_path + "/" + container; + if (!Path.existsSync(c_path)) { + error_msg(404,"NoSuchBucket","No such container",resp);resp.resp_end();return; + } + var file_path = c_path + "/" + filename; //complete representation: /container/filename + var md5_name = crypto.createHash('md5'); + //md5_name.update(file_path); + md5_name.update(filename); //de-couple from root and container paths + var name_digest = md5_name.digest('hex'); + winston.log('debug',(new Date())+" - read: the md5 hash for "+filename+" is "+name_digest); + var name_dig_pre = name_digest.substr(0,PREFIX_LENGTH); + var fdir_path = c_path + "/" + name_dig_pre; //actual dir for the file /container/hashprefix/ + if (!Path.existsSync(fdir_path)) { //check such folder + error_msg(404,"NoSuchFile","No such file",resp);resp.resp_end();return; + } + var esp_name = filename.replace(/\//g,"$_$"); + var file_size = fs.statSync(fdir_path+"/"+esp_name).size; + if (file_size === undefined) { + error_msg(404,"NoSuchFile","No such file",resp);resp.resp_end(); return; + } + var etag_match=null, etag_none_match=null, date_modified=null, date_unmodified=null; + if (true){ + var keys = Object.keys(requ.headers); + for (var idx = 0; idx < keys.length; idx++) + { + if (keys[idx].match(/^if-match$/i)) + { etag_match = requ.headers[keys[idx]]; } + else if (keys[idx].match(/^if-none-match$/i)) + { etag_none_match = requ.headers[keys[idx]]; } + else if (keys[idx].match(/^if-unmodified-since$/i)) + { date_unmodified = requ.headers[keys[idx]]; } + else if (keys[idx].match(/^if-modified-since$/i)) + { date_modified = requ.headers[keys[idx]]; } + } + } + //read meta here + this.MDS.collection(container,{safe:true},function(err,coll) { + if (err||!coll) { error_msg(404,"NoSuchBucket",err,resp); resp.resp_end(); return; } + coll.findOne({"vblob_file_name":filename}, function (err, obj) { + if (err||!obj) { + error_msg(404,"NoSuchFile",err,resp); resp.resp_end(); return; + } + var header = common_header(); + if (file_size !== obj.vblob_file_size) { + error_msg(500,"InternalError","file corrupted",resp); resp.resp_end(); return; + } + var modified_since=true, unmodified_since=true; + var t1,t2; + if (date_modified) { + t1 = new Date(date_modified).valueOf(); + t2 = new Date(obj.vblob_update_time).valueOf(); + modified_since = t2 > t1; + } else if (date_unmodified) { + t1 = new Date(date_unmodified).valueOf(); + t2 = new Date(obj.vblob_update_time).valueOf(); + unmodified_since = t2 <= t1; + } + //412 + if (unmodified_since === false || + etag_match && etag_match !== obj.vblob_file_etag) + { + error_msg(412,"PreconditionFailed","At least one of the preconditions you specified did not hold.",resp); resp.resp_end(); return; + } + //304 + if (modified_since === false || + etag_none_match && etag_none_match === obj.vblob_file_etag) + { + error_msg(304,'NotModified','The object is not modified',resp); + resp.resp_end(); return; + } + header["content-type"] = obj["content-type"] ? obj["content-type"] : "binary/octet-stream"; + header["Content-Length"] = obj.vblob_file_size; + header["Last-Modified"] = obj.vblob_update_time; + header.ETag = obj.vblob_file_etag; + if (true) { + var keys = Object.keys(obj); + for (var idx = 0; idx < keys.length; idx++) { + var obj_key = keys[idx]; + if (obj_key.match(/^vblob_meta_/)) { + var sub_key = obj_key.substr(11); + sub_key = "x-amz-meta-" + sub_key; + header[sub_key] = obj[obj_key]; + } + } + } + header["Accept-Ranges"] = "bytes"; + var st; + if (range !== null && range !== undefined) { + header["Content-Range"] = "bytes "+ (range.start!==undefined?range.start:"")+'-'+(range.end!==undefined?range.end.toString():"") + "/"+obj.vblob_file_size.toString(); + if (range.start === undefined) { range.start = file_size - range.end; delete range.end; } + if (range.end === undefined) { range.end = file_size-1; } + header["Content-Length"] = range.end - range.start + 1; + //resp.writeHeader(206,header); + resp.resp_code = 206; resp.resp_header = header; + if (verb==="get") { + st = fs.createReadStream(fdir_path+"/"+esp_name, range); + //st.pipe(resp); + st.on('data',function(chunk) { + if (resp.client_closed === true) { st.destroy(); resp.resp_end(); return; } + resp.resp_handler(chunk); + }); + st.on('end',function() { resp.resp_end(); }); + } else { resp.resp_end(); } + } else { + resp.resp_code = 200; resp.resp_header = header; + //resp.writeHeader(200,header); + if (verb==="get") { + st = fs.createReadStream(fdir_path+"/"+esp_name); + st.on('error',function(err) { throw err;}); + //st.pipe(resp); + st.on('data',function(chunk) { + if (resp.client_closed === true) { st.destroy(); resp.resp_end(); return; } + resp.resp_handler(chunk); + }); + st.on('end',function() { resp.resp_end(); }); + } else { resp.resp_end(); } + } + }); + }); +}; + +FS_blob.prototype.list_containers = function() +{ + return fs.readdirSync(this.root_path); +}; + +//sequentially issue queries to obtain max-keys number of files/folders +FS_blob.prototype.list_container = function (container, opt, resp) +{ + //TODO: handling marker, prefix, delimiter, virtual folder + var pre_marker = opt.marker; + if (!opt["max-keys"] || parseInt(opt["max-keys"],10) > MAX_LIST_LENGTH) { opt["max-keys"] = MAX_LIST_LENGTH; } + var pre_maxkey= opt["max-keys"]; + this.MDS.collection(container,{safe:true},function(err,coll) { + if (err) { winston.log('error',(new Date())+" - In listing container "+container+" "+err); error_msg(404,"NoSuchBucket",err,resp); resp.resp_end(); return; } + var evt = new events.EventEmitter(); + var res_array = []; //regular file + var res_array2 = []; //folder + evt.on("Next Query", function (opts) { + if (resp.client_closed === true) { winston.log('warn',(new Date())+' - client disconnected'); resp.resp_end(); return; } + winston.log('debug',(new Date())+" - Next Query"); + var cond = {$exists:true}; + if (opts.prefix) { cond.$regex = "^"+opts.prefix; } + if (opts.marker) { cond.$gt = opts.marker; } + var options = {sort:[['vblob_file_name','asc']]}; + if (opts["max-keys"]) { opts["max-keys"] = parseInt(opts["max-keys"],10); options.limit = opts["max-keys"]; } + coll.find({vblob_file_name:cond},{/*'vblob_file_name':true*/},options, function (err, cursor) { + if (err) { winston.log('error',(new Date())+" - Error retrieving data from db "+err); error_msg(500,"InternalError",err,resp); resp.resp_end(); return; } + if (cursor === null || cursor === undefined) {evt.emit("Finish List");} + else if (opts.delimiter) { evt.emit("Next Object", opts, cursor); } + else { + cursor.each( function(err,doc) {//optimization for queries without delimiter + if (resp.client_closed === true) { winston.log('warn',(new Date())+' - client disconnected'); cursor.close(); resp.resp_end(); return; } + if (doc) + { res_array.push({"Key":doc.vblob_file_name, "LastModified":doc.vblob_update_time, "ETag":'"'+doc.vblob_file_etag+'"', "Size":doc.vblob_file_size, "Owner":{}, "StorageClass":"STANDARD"}); + if (opts["max-keys"]) { opts["max-keys"] = opts["max-keys"] - 1; } + } + else { evt.emit("Finish List"); } + }); + } + }); + }); + evt.on("Next Object", function (opts,cursor) { + cursor.nextObject( function (err, doc) { + if (err) { winston.log('error',(new Date())+" - Error retrieving data from db "+err); error_msg(500,"InternalError",err,resp); resp.resp_end(); return; } + if (resp.client_closed === true) { winston.log('warn',(new Date())+' - client disconnected'); cursor.close(); resp.resp_end(); return; } + if (doc === null || doc === undefined) { + if (cursor.totalNumberOfRecords > 0) + { evt.emit("Next Query", opts); } + else { evt.emit("Finish List"); } + } else { + opts.marker = doc.vblob_file_name; + if (opts["max-keys"]) { opts["max-keys"] = opts["max-keys"] - 1; } + var str1; + if (opts.prefix) { str1 = doc.vblob_file_name.substring(opts.prefix.length);} + else { str1 = doc.vblob_file_name; } + var pos; + if (str1 === "") { res_array.push({"Key":doc.vblob_file_name, "LastModified":doc.vblob_update_time, "ETag":'"'+doc.vblob_file_etag+'"', "Size":doc.vblob_file_size, "Owner":{}, "StorageClass":"STANDARD"}); } + else { + //delimiter + //str1 = doc.vblob_file_name; + pos = str1.search(opts.delimiter); + winston.log('debug',(new Date())+" - found delimiter "+opts.delimiter+" at position "+pos); + if (pos === -1) + { + //not found + res_array.push({"Key":doc.vblob_file_name, "LastModified":doc.vblob_update_time, "ETag":'"'+doc.vblob_file_etag+'"', "Size":doc.vblob_file_size, "Owner":{}, "StorageClass":"STANDARD"}); + if (opts["max-keys"] !== undefined && opts["max-keys"] <= 0) {cursor.close(); evt.emit("Finish List");} + else { evt.emit("Next Object", opts,cursor); } + } else + { + var len = 0; + if (opts.prefix) { len = opts.prefix.length; } + var str2 = doc.vblob_file_name.substring(0,len+pos); + opts.marker = str2+String.fromCharCode(doc.vblob_file_name.charCodeAt(len+pos)+1); + winston.log('debug',(new Date())+" - Next Marker "+opts.marker+" and len "+len); + res_array2.push({"Prefix":str2+opts.delimiter}); //another array for folder + cursor.close(); + if (opts["max-keys"] !== undefined && opts["max-keys"] <= 0) { evt.emit("Finish List"); } + else { evt.emit("Next Query", opts); } + } + } + } + }); + }); + evt.on("Finish List", function () { + resp.resp_code = 200; resp.resp_header = common_header(); + var res_json = {}; + res_json.Name = container; + res_json.Prefix = opt.prefix?opt.prefix:{}; + res_json.Marker = pre_marker?pre_marker:{}; + res_json.MaxKeys = ""+pre_maxkey; + if (opt.delimiter) { res_json.Delimiter = opt.delimiter; } + if (opt["max-keys"] <= 0) { res_json.IsTruncated = 'true'; } + else { res_json.IsTruncated = 'false'; } + if (res_array.length > 0) {res_json.Contents = res_array; } //files + if (res_array2.length > 0) { res_json.CommonPrefixes = res_array2;} //folder + resp.resp_body = {"ListBucketResult": res_json}; + resp.resp_end(); + }); + evt.emit("Next Query", opt); + }); +}; + +function render_containers(dirs,resp,fb) +{ + var dates = new Array(dirs.length); + var evt = new events.EventEmitter(); + var counter = dirs.length; + evt.on("Get Date",function (dir_name, idx) { + fb.MDS.collection(dir_name, {safe:true},function(err,col) { + if (err) { + winston.log('error',(new Date())+" - retreiving meta "+err); /*error_msg(500,"InternalError",err,resp); resp.resp_end();*/ + dates[idx] = null; + counter--; if (counter === 0) { evt.emit("Start Render"); } + return; + } //skip this folder + col.findOne({"vblob_container_name" : dir_name}, function (err, item) { + if (err) { error_msg(500,"InternalError",err,resp); resp.resp_end(); return; } + dates[idx] = item.vblob_create_time; + counter--; if (counter === 0) { evt.emit("Start Render"); } + }); + }); + }); + evt.on("Start Render", function () { + resp.resp_code = 200; + resp.resp_header = common_header(); + var output = ""; + output += '{"ListAllMyBucketsResult":'; + output += '{"Buckets":{"Owner":{}';//empty owner information + output += ',"Bucket":['; + for (var i = 0,j=0; i < dirs.length; i++) { + if (j > 0) { output += ','; } + if (dates[i] === null) { continue; } + j = j+1; + output += '{'; + output += ('"Name":"'+dirs[i]+'"'); + output += (',"CreationDate":"'+dates[i]+'"'); + output += ('}'); + } + output += (']}}}'); + resp.resp_body = JSON.parse(output); + resp.resp_end(); + }); + if (dirs.length === 0) { evt.emit("Start Render"); } + for (var i = 0; i < dirs.length; i++) + { evt.emit("Get Date",dirs[i],i); } +} + +//======================================================= +var FS_Driver = module.exports = function(root_path, mds_cred,callback) { + var this1 = this; + var client = new FS_blob(root_path,mds_cred, function(obj) { + this1.client = obj; + if (callback) { callback(this1); } + }); //supress warning +}; + +FS_Driver.prototype.list_buckets = function (requ,resp) { + var dirs = this.client.list_containers(); + render_containers(dirs,resp,this.client); +}; + +FS_Driver.prototype.list_bucket = function(container,option,resp) { + this.client.list_container(container,option,resp); +}; + +FS_Driver.prototype.read_file = function(container,filename,range,verb,resp,requ){ + var range1 = null; + if (range) { + range1 = range; + range1 = range1.substr(6); + var m = range1.match(/^([0-9]*)-([0-9]*)$/); + if (m[1]===m[2]&& m[1]==='') { range1=null; } + else { + range1 = {}; + if (m[1] !== '') { range1.start = parseInt(m[1],10); } + if (m[2] !== '') { range1.end = parseInt(m[2],10); } + } + winston.log('debug',(new Date())+" - Final range: "+util.inspect(range1)); + } + this.client.read_file(container, filename, range1,requ,resp,verb); +}; + +FS_Driver.prototype.create_file = function(container,filename,requ,resp) { + this.client.create_file(container,filename,requ,resp,this.client); +}; + +FS_Driver.prototype.copy_file = function(dest_c, dest_f, src_c,src_f,requ,resp) +{ + this.client.copy_file(dest_c,dest_f,src_c,src_f,requ,resp,this.client); +}; + +FS_Driver.prototype.create_bucket = function(container,resp) { + this.client.create_container_meta(container,resp,this.client); +}; + +FS_Driver.prototype.delete_file = function(container,filename,resp) { + this.client.delete_file_meta(container,filename,resp,this.client); +}; + +FS_Driver.prototype.delete_bucket = function(container,resp) { + this.client.delete_container(container,resp,this.client); +}; + +FS_Driver.prototype.pingDest = function(callback) { + callback(null); +}; + +module.exports.createDriver = function(option,callback) { + return new FS_Driver(option.root, option.mds, callback); +}; diff --git a/blob_fs/node-mongodb-native b/blob_fs/node-mongodb-native new file mode 160000 index 0000000..b978225 --- /dev/null +++ b/blob_fs/node-mongodb-native @@ -0,0 +1 @@ +Subproject commit b978225e6f54538756fb00dce3046b0e192da022 diff --git a/blob_s3/Readme.md b/blob_s3/Readme.md new file mode 100644 index 0000000..5e56c5a --- /dev/null +++ b/blob_s3/Readme.md @@ -0,0 +1,33 @@ + +# knox + + Node Amazon S3 Client. + +## Authors + + - TJ Holowaychuk ([visionmedia](http://github.com/visionmedia)) + +## License + +(The MIT License) + +Copyright (c) 2010 LearnBoost <dev@learnboost.com> + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +'Software'), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/blob_s3/blob_s3.js b/blob_s3/blob_s3.js new file mode 100644 index 0000000..66e41d6 --- /dev/null +++ b/blob_s3/blob_s3.js @@ -0,0 +1,341 @@ +/* + Based on knox project: https://github.com/LearnBoost/knox.git + Additional library: sax-js xml parser: https://github.com/isaacs/sax-js.git (subject to change) + Need amazon s3 id/key pair + Need winston module for logging +*/ +var winston = require('winston'); +var knox = require('./index.js'); +var fs = require('fs'); +var util = require('util'); +var sax = require('./sax-js/lib/sax'); +var url = require('url'); +var mime = require('./lib/knox/mime'); + +function parse_xml(parser,resp) +{ + var parse_stack = []; + var cur_obj = {}; + var char_buf = ""; + var acc = false; + parser.onerror = function (e) { + throw e; + }; + parser.ontext = function (t) { + if (!acc) { return; } + char_buf += t; + }; + parser.onopentag = function (node) { + acc = true; + parse_stack.push(cur_obj); + cur_obj = {}; + }; + parser.onclosetag = function (name) { + if (char_buf !== "" ) { + cur_obj = parse_stack.pop(); + if (cur_obj[name]) { + if (cur_obj[name].push) + { cur_obj[name].push(char_buf); } + else { cur_obj[name] = [cur_obj[name],char_buf]; } + } + else { cur_obj[name] = char_buf; } + } else { + var cur_obj2 = parse_stack.pop(); + if (cur_obj2[name]) { + if (cur_obj2[name].push) + { cur_obj2[name].push(cur_obj); } + else { cur_obj2[name] = [cur_obj2[name],cur_obj]; } + } + else { cur_obj2[name] = cur_obj; } + cur_obj = cur_obj2; + } + char_buf = ""; + acc = false; + }; + parser.onend = function () { + resp.resp_body = cur_obj; + resp.resp_end(); + }; +} + +/* + cl: S3_blob obj + req: request to s3 + resp: response to client + head: if it's HEAD verb +*/ +function general_handler(cl,req,resp,head) +{ + if (head === null || head === undefined) { head = false; } + req.on('response',function(res) { + res.setEncoding('utf8'); + resp.resp_header = res.headers; resp.resp_code = res.statusCode; + var parser = null; + //special handling for redirect + if (res.statusCode === 307) { + var redirect = res.headers.location; + var parsed = url.parse(redirect); + var sp = parsed.hostname.split('.'); + var span = 60; + cl.region_cache[sp[0]] = {"name":(sp[1]+'.'+sp[2]+'.'+sp[3]), "expire":(new Date().valueOf() + span * 1000) }; + } + res.on('data',function(chunk) { + if (parser === null && resp.client_closed !== true) { + parser = sax.parser(true); + parse_xml(parser,resp); + } + if (resp.client_closed === true) { winston.log('warn',(new Date())+' - client connection closed!'); res.destroy(); if (parser) { parser=null; } resp.resp_end();return; } + parser.write(chunk); + }); + res.on('end',function() { if (parser !== null) { parser.close(); } else { resp.resp_end();} }); //parse.end event will trigger response + }).on('error',function(err) { + resp.resp_header = {'Connection':'close'}; + resp.resp_code = 500; + resp.resp_body = '{"Code":"500","Message":"Internal error: '+err+'"}'; + resp.resp_end(); + }); +} + +function S3_blob(credentials) +{ + this.client = knox.createClient({ + key: credentials.key, + secret: credentials.secret + }); + this.region_cache = { }; +} + +//requ: request from client; resp: response to client +S3_blob.prototype.create_container = function(container,requ,resp) +{ + var opt = {}; + if (requ.query.logging !== undefined) { opt.logging = requ.query.logging; } + var req = this.client.put({bucket:container,query:opt},{}); + general_handler(this,req,resp); + requ.on('data', function (chunk) { req.write(chunk); } ); + requ.on('end', function() { req.end(); } ); +}; + +S3_blob.prototype.delete_container = function(container,resp) +{ + var req = this.client.del({bucket:container,endpoint:this.region_cache[container]},{}); + general_handler(this,req,resp); + req.end(); +}; + +S3_blob.prototype.create_file = function(container,filename,header,data,resp) +{ + var self = this; + var parser = null; + var keys = Object.keys(header); + for (var idx = 0; idx < keys.length; idx++) { + if (keys[idx].match(/^x-amz-acl$/i)) { delete header[keys[idx]]; break; } //remove acl, always create private object + } + this.client.putStream2(data,{bucket:container, "filename":filename,endpoint:this.region_cache[container]},header, function(err,res,chunk) { + if (err) {//err,null,null + resp.resp_header = {'Connection':'close'}; + resp.resp_code = 500; + resp.resp_body = '{"Code":"500","Message":"Internal error: '+err+'"}'; + resp.resp_end(); + } else + if (res!==null) {//null,res,null + res.setEncoding('utf8'); + resp.resp_header = res.headers; + resp.resp_code = res.statusCode; + //special handling for redirect + if (res.statusCode === 307) { + var redirect = res.headers.location; + var parsed = url.parse(redirect); + var sp = parsed.hostname.split('.'); + var span = 60; + self.region_cache[sp[0]] = {"name":(sp[1]+'.'+sp[2]+'.'+sp[3]), "expire":(new Date().valueOf() + span * 1000) }; + } + if (res.statusCode >= 300) { + //need to parse + parser = sax.parser(true); + parse_xml(parser,resp); + } + } else if (chunk!==null)//null,null,chunk + { + if (!parser) {throw "Create_File response error!";} //resp.write(chunk);//shouldn't happen! + else { parser.write(chunk);} + } + else { if (parser) { parser.close(); } else { resp.resp_end(); } }//null,null,null + }); +}; + +S3_blob.prototype.copy_file = function(container,filename,header,resp) +{ + var self = this; + var keys = Object.keys(header); + for (var idx = 0; idx < keys.length; idx++) { + if (keys[idx].match(/^x-amz-acl$/i)) { delete header[keys[idx]]; break; } //remove acl, always create private object + } + var req = this.client.put({bucket:container,"filename":filename,endpoint:this.region_cache[container]},header); + general_handler(this,req,resp); + req.end(); +}; + +S3_blob.prototype.delete_file = function(container,filename,resp) +{ + var req = this.client.del({bucket:container,"filename":filename,endpoint:this.region_cache[container]},{}); + general_handler(this,req,resp); + req.end(); +}; + +S3_blob.prototype.list_location = function(container,resp) +{ + var req = this.client.get({bucket:container,endpoint:this.region_cache[container],query:{"location":null}},{}); + general_handler(this,req,resp); + req.end(); +}; + +S3_blob.prototype.list_logging = function(container,resp) +{ + var req = this.client.get({bucket:container,endpoint:this.region_cache[container],query:{"logging":null}},{}); + general_handler(this,req,resp); + req.end(); +}; + +S3_blob.prototype.read_file = function(container, filename, range,verb,resp,requ) +{ + var cl = this; + if (verb === 'head') { requ.query = null; } + var req = this.client[verb]({bucket:container,"filename":filename,endpoint:this.region_cache[container],query:requ.query},requ.headers); + var head = (verb === 'head'); + req.on('response',function(res) { + //res.setEncoding('utf8'); + resp.resp_header = res.headers; resp.resp_code = res.statusCode; + var parser = null; + //special handling for redirect + if (res.statusCode === 307) { + var redirect = res.headers.location; + var parsed = url.parse(redirect); + var sp = parsed.hostname.split('.'); + var span = 60; + cl.region_cache[sp[0]] = {"name":(sp[1]+'.'+sp[2]+'.'+sp[3]), "expire":(new Date().valueOf() + span * 1000) }; + } + if (res.statusCode >= 300) { //only parse when error + res.setEncoding('utf8'); + parser = sax.parser(true); + parse_xml(parser,resp); + } + res.on('data',function(chunk) {//mean it's a GET + if (resp.client_closed === true) { winston.log('warn',(new Date())+' - client connection closed!'); if (parser) { parser=null;} res.destroy(); resp.resp_end();return; } + if (parser) { parser.write(chunk); } //parse error message + else {//streaming out data + resp.resp_handler(chunk); //callback to stream out chunk data + } + }); + res.on('end',function() { if (parser) {parser.close(); } else { resp.resp_end();}}); //time to finish and response back to client + }).on('error',function(err) { + resp.resp_header = {'Connection':'close'}; + resp.resp_code = 500; + resp.resp_body = '{"Code":"500","Message":"Internal error: '+err+'"}'; + resp.resp_end(); + }); + req.end(); +}; + +S3_blob.prototype.list_containers = function(resp) { + var req = this.client.get({},{}); + general_handler(this,req,resp); + req.end(); +}; + +S3_blob.prototype.list_container = function (container,opt,resp) +{ + var self = this; + var req=this.client.get({bucket:container,query:opt,endpoint:this.region_cache[container]}); + general_handler(this,req,resp); + req.end(); +}; + +var S3_Driver = module.exports = function S3_Driver(client) { + this.client = client; +}; + +S3_Driver.prototype.list_buckets = function(requ,resp) +{ + this.client.list_containers(resp); +}; + +/* + queries: marker / prefix / delimiter / max-keys / location / logging +*/ + +S3_Driver.prototype.list_bucket = function(container,option,resp) +{ + var keys = Object.keys(option); + if (keys.length === 1 && keys.indexOf('location') !== -1) { + this.client.list_location(container,resp); + } else if (keys.length === 1 && keys.indexOf('logging') !== -1) { + this.client.list_logging(container,resp); + } + else { + delete option.location; + delete option.logging; + this.client.list_container(container,option,resp); + } +}; + +//response body is a bit stream(if succeed), no need to parse XML +S3_Driver.prototype.read_file = function(container,filename,range,verb,resp,requ) +{ + this.client.read_file(container,filename,range,verb,resp,requ); +}; + +S3_Driver.prototype.create_file = function(container,filename,requ,resp) +{ + this.client.create_file(container,filename,requ.headers,requ,resp); +}; + +S3_Driver.prototype.copy_file = function(dest_c, dest_f, src_c,src_f,requ,resp) +{ + this.client.copy_file(dest_c,dest_f,requ.headers,resp); +}; + +S3_Driver.prototype.delete_file = function(container,filename,resp) +{ + this.client.delete_file(container,filename,resp); +}; + + +S3_Driver.prototype.create_bucket = function(container,resp,requ) +{ + this.client.create_container(container,requ,resp); +}; + +S3_Driver.prototype.delete_bucket = function(container,resp) +{ + this.client.delete_container(container,resp); +}; + +S3_Driver.prototype.pingDest = function(callback) { + var dns = require('dns'); + dns.resolve(this.client.client.endpoint, function(err,addr) { + if (err) { + winston.log('error',(new Date())+' - Cannot resolve s3 domain'); + callback(err); + } else { + var net = require('net'); + var sock = new net.Socket(); + sock.connect(80,addr[0]); + sock.on('connect',function() { callback(); } ); + sock.on('error',function(err) { + winston.log('error',(new Date())+' - Cannot connect to s3'); + callback(err); + }); + } + }); +}; + +module.exports.createDriver = function(option,callback) { + var S3_client = new S3_blob({ + key: option.key, + secret: option.secret} + ); + var dr = new S3_Driver(S3_client); + if (callback) { callback(dr); } + return dr; +}; diff --git a/blob_s3/index.js b/blob_s3/index.js new file mode 100644 index 0000000..33a2a4a --- /dev/null +++ b/blob_s3/index.js @@ -0,0 +1 @@ +module.exports = require('./lib/knox'); diff --git a/blob_s3/lib/knox/auth.js b/blob_s3/lib/knox/auth.js new file mode 100644 index 0000000..23df684 --- /dev/null +++ b/blob_s3/lib/knox/auth.js @@ -0,0 +1,163 @@ +/*! + * knox - auth + * Copyright(c) 2010 LearnBoost + * MIT Licensed + */ + +/** + * Module dependencies. + */ + +var crypto = require('crypto'); +var url = require('url'); + +/** + * Return an "Authorization" header value with the given `options` + * in the form of "AWS :" + * + * @param {Object} options + * @return {String} + * @api private + */ + +exports.authorization = function(options){ + return 'AWS ' + options.key + ':' + exports.sign(options); +}; + +/** + * Simple HMAC-SHA1 Wrapper + * + * @param {Object} options + * @return {String} + * @api private + */ + +exports.hmacSha1 = function(options){ + return crypto.createHmac('sha1', options.secret).update(options.message).digest('base64'); +}; + +/** + * Create a base64 sha1 HMAC for `options`. + * + * @param {Object} options + * @return {String} + * @api private + */ + +exports.sign = function(options){ + options.message = exports.stringToSign(options); + return exports.hmacSha1(options); +}; + +/** + * Create a base64 sha1 HMAC for `options`. + * + * Specifically to be used with S3 presigned URLs + * + * @param {Object} options + * @return {String} + * @api private + */ + +exports.signQuery = function(options){ + options.message = exports.queryStringToSign(options); + return exports.hmacSha1(options); +}; + +/** + * Return a string for sign() with the given `options`. + * + * Spec: + * + * \n + * \n + * \n + * \n + * [headers\n] + * + * + * @param {Object} options + * @return {String} + * @api private + */ + +exports.stringToSign = function(options){ + var headers = options.amazonHeaders || ''; + if (headers) { headers += '\n'; } + return [ + options.verb + , options.md5 + , options.contentType + , options.date.toUTCString() + , headers + options.resource + ].join('\n'); +}; + +/** + * Return a string for sign() with the given `options`, but is meant exclusively + * for S3 presigned URLs + * + * Spec: + * + * \n + * + * + * @param {Object} options + * @return {String} + * @api private + */ + +exports.queryStringToSign = function(options){ + return 'GET\n\n\n' + + options.date + '\n' + + options.resource; +}; + +/** + * Perform the following: + * + * - ignore non-amazon headers + * - lowercase fields + * - sort lexicographically + * - trim whitespace between ":" + * - join with newline + * + * @param {Object} headers + * @return {String} + * @api private + */ + +exports.canonicalizeHeaders = function(headers){ + var buf = [] + , fields = Object.keys(headers).sort(); + for (var i = 0, len = fields.length; i < len; ++i) { + var field = fields[i] + , val = headers[field]; + field = field.toLowerCase(); + if (0 !== field.indexOf('x-amz')) { continue; } + buf.push(field + ':' + val); + } + return buf.join('\n'); +}; + +/** + * Perform the following: + * + * - ignore non sub-resources + * - sort lexicographically + * + * @param {String} resource + * @return {String} + * @api private + */ +exports.canonicalizeResource = function(resource){ + var urlObj = url.parse(resource, true); + var buf = urlObj.pathname; + var qbuf = []; + Object.keys(urlObj.query).forEach(function (qs) { + if (['acl', 'location', 'logging', 'notification', 'partNumber', 'policy', 'requestPayment', 'torrent', 'uploadId', 'uploads', 'versionId', 'versioning', 'versions', 'website','response-content-type', 'response-content-language','response-expires','reponse-cache-control','response-content-disposition','response-content-encoding'].indexOf(qs) !== -1) { + qbuf.push(qs + (urlObj.query[qs] !== '' ? '=' + /*encodeURIComponent*/(urlObj.query[qs]) : '')); + } + }); + return buf + (qbuf.length !== 0 ? '?' + qbuf.sort().join('&') : ''); +}; diff --git a/blob_s3/lib/knox/client.js b/blob_s3/lib/knox/client.js new file mode 100644 index 0000000..21310c3 --- /dev/null +++ b/blob_s3/lib/knox/client.js @@ -0,0 +1,361 @@ +/*! + * knox - Client + * Copyright(c) 2010 LearnBoost + * MIT Licensed + */ + +/** + * Module dependencies. + */ + +var utils = require('./utils') + , auth = require('./auth') + , http = require('http') + , https = require('https') + , url = require('url') + , join = require('path').join + , mime = require('./mime') + , fs = require('fs'); + +/** + * Initialize a `Client` with the given `options`. + * + * Required: + * + * - `key` amazon api key + * - `secret` amazon secret + * + * @param {Object} options + * @api public + */ + +var Client = module.exports = function Client(options) { + if (!options.key) { throw new Error('aws "key" required'); } + if (!options.secret) { throw new Error('aws "secret" required'); } + this.endpoint = 's3.amazonaws.com'; + utils.merge(this, options); +}; + +/** + * Request with optional `targets.filename` and optional `targets.bucket` with the given `method`, and optional `headers`. + * + * @param {String} method + * @param {Hash} targets + * @param {Object} headers + * @return {ClientRequest} + * @api private + */ + +Client.prototype.request = function(method, targets, headers){ + var content_md5 = ""; + if (method==="PUT" && targets.filename) //only creating file checkes md5 + { + var keys = Object.keys(headers); + for (var idx =0; idx new Date().valueOf()) { dest = dest.name; } + else { dest = this.endpoint; } + } + var options = { host: ( (targets.bucket!==undefined && targets.bucket !== null)?targets.bucket+".":"")+ dest, port: 443 } + , date = new Date(); + + if (headers === null || headers === undefined) { headers = {}; } + + // Default headers + utils.merge(headers, { + Date: date.toUTCString() + , Host: ((targets.bucket!==undefined && targets.bucket !== null)?targets.bucket+".":"")+ dest + }); + + // Authorization header + //resource: "/" for listing buckets; otherwise bucket or file level operations + headers.Authorization = auth.authorization({ + key: this.key + , secret: this.secret + , verb: method + , md5 : content_md5 + , date: date + , resource: auth.canonicalizeResource((targets.bucket===undefined || targets.bucket === null)?'/':(targets.filename ?/* join('/', targets.bucket, targets.filename)*/ '/'+targets.bucket+'/'+targets.filename+utils.to_query_string(targets.query):join('/',targets.bucket)+'/'+utils.to_query_string(targets.query))) + , contentType: headers['Content-Type'] + , amazonHeaders: auth.canonicalizeHeaders(headers) + }); + + // Issue request + options.method = method; + options.path = targets.filename?/*join('/', targets.filename)*/ '/'+targets.filename+utils.to_query_string(targets.query):'/'+ utils.to_query_string(targets.query); + options.headers = headers; + var req = https.request(options); + req.url = this.https(targets.bucket,targets.filename?targets.filename:'', dest); + return req; +}; + +/** + * PUT data to `targets` with optional `headers`. + * If both bucket and filename are not null, create a file, otherwise create a bucket + * @param {Hash} targets + * @param {Object} headers + * @return {ClientRequest} + * @api public + */ + +Client.prototype.put = function(targets, headers){ + headers = utils.merge({ + Expect: '100-continue' + }, headers || {}); + return this.request('PUT', targets, headers); +}; + +/** + * PUT the file at `src` to `targets`, with callback `fn` + * receiving a possible exception, and the response object. + * + * NOTE: this method reads the _entire_ file into memory using + * fs.readFile(), and is not recommended or large files. + * + * Example: + * + * client + * .putFile('package.json', {filename:'test/package.json',bucket:'bucket1'}, function(err, res){ + * if (err) throw err; + * console.log(res.statusCode); + * console.log(res.headers); + * }); + * + * @param {String} src + * @param {Hash} targets + * @param {Object|Function} headers + * @param {Function} fn + * @api public + */ + +Client.prototype.putFile = function(src, targets, headers, fn){ + var self = this; + if ('function' === typeof headers) { + fn = headers; + headers = {}; + } + fs.readFile(src, function(err, buf){ + if (err) { return fn(err); } + headers = utils.merge({ + 'Content-Length': buf.length + , 'Content-Type': mime.lookup(src) + }, headers); + self.put(targets, headers).on('response', function(res){ + fn(null, res); + }).end(buf); + }); +}; + +/** + * PUT the given `stream` as `targets` with optional `headers`. + * + * @param {Stream} stream + * @param {Hash} targets + * @param {Object|Function} headers + * @param {Function} fn + * @api public + */ + +Client.prototype.putStream = function(stream, targets, headers, fn){ + var self = this; + if ('function' === typeof headers) { + fn = headers; + headers = {}; + } + fs.stat(stream.path, function(err, stat){ + if (err) { return fn(err); } + // TODO: sys.pump() wtf? + var req = self.put(targets, utils.merge({ + 'Content-Length': stat.size + , 'Content-Type': mime.lookup(stream.path) + }, headers)); + req.on('response', function(res){ + fn(null, res); + }); + stream + .on('error', function(err){fn(null, err); }) + .on('data', function(chunk){ req.write(chunk); }) + .on('end', function(){ req.end(); }); + }); +}; + + +Client.prototype.putStream2 = function(stream, targets, headers, fn){ + var self = this; + if ('function' === typeof headers) { + fn = headers; + headers = {}; + } + // TODO: sys.pump() wtf? + var req = self.put(targets, headers); + req.on('response', function(res){ + fn(null, res,null); + res.on('data', function(chunk) { + fn(null,null,chunk); + }); + res.on('end',function() { fn(null,null,null);}); + }); + req.on('error', function(err) { + fn(err,null); + }); + stream + .on('error', function(err){fn(err, null); req.end(); }) + .on('data', function(chunk){ req.write(chunk); }) + .on('end', function(){ req.end(); }); +}; + +/** + * GET `targets` with optional `headers`. + * If both bucket and file are specified, get the actual file; if only bucket is specified list + * bucket; otherwise list buckets + * @param {Hash} targets + * @param {Object} headers + * @return {ClientRequest} + * @api public + */ + +Client.prototype.get = function(targets, headers){ + return this.request('GET', targets, headers); +}; + +/** + * GET `targets` with optional `headers` and callback `fn` + * with a possible exception and the response. + * + * @param {Hash} targets + * @param {Object|Function} headers + * @param {Function} fn + * @api public + */ + +Client.prototype.getFile = function(targets, headers, fn){ + if ('function' === typeof headers) { + fn = headers; + headers = {}; + } + return this.get(targets, headers).on('response', function(res){ + fn(null, res); + }).end(); +}; + +/** + * Issue a HEAD request on `targets` with optional `headers. + * + * @param {Hash} targets + * @param {Object} headers + * @return {ClientRequest} + * @api public + */ + +Client.prototype.head = function(targets, headers){ + return this.request('HEAD', targets, headers); +}; + +/** + * Issue a HEAD request on `targets` with optional `headers` + * and callback `fn` with a possible exception and the response. + * + * @param {Hash} targets + * @param {Object|Function} headers + * @param {Function} fn + * @api public + */ + +Client.prototype.headFile = function(targets, headers, fn){ + if ('function' === typeof headers) { + fn = headers; + headers = {}; + } + return this.head(targets, headers).on('response', function(res){ + fn(null, res); + }).end(); +}; + +/** + * DELETE `targets` with optional `headers. + * + * @param {Hash} targets + * @param {Object} headers + * @return {ClientRequest} + * @api public + */ + +Client.prototype.del = function(targets, headers){ + return this.request('DELETE', targets, headers); +}; + +/** + * DELETE `targets` with optional `headers` + * and callback `fn` with a possible exception and the response. + * + * @param {Hash} targets + * @param {Object|Function} headers + * @param {Function} fn + * @api public + */ + +Client.prototype.deleteFile = function(targets, headers, fn){ + if ('function' === typeof headers) { + fn = headers; + headers = {}; + } + return this.del(targets, headers).on('response', function(res){ + fn(null, res); + }).end(); +}; + +/** + * Return a url to the given resource. + * + */ + +Client.prototype.url = +Client.prototype.http = function(bucket,filename,ep){ + var dest = ep; if (dest === null || dest === undefined) { dest = this.endpoint; } + return 'http://' + join((bucket!==null?bucket+".":"")+dest, filename); +}; + +/** + * Return an HTTPS url to the given resource. + */ + +Client.prototype.https = function(bucket,filename,ep){ + var dest = ep; if (dest === null || dest === undefined) { dest = this.endpoint; } + return 'https://' + join(bucket+"."+this.endpoint, filename); +}; + +/** + * Return an S3 presigned url + * + */ + +Client.prototype.signedUrl = function(targets, expiration){ + var epoch = Math.floor(expiration.getTime()/1000); + var signature = auth.signQuery({ + secret: this.secret, + date: epoch, + resource: '/' + targets.bucket + url.parse(targets.filename).pathname + }); + + return this.url(targets.filename) + + '?Expires=' + epoch + + '&AWSAccessKeyId=' + this.key + + '&Signature=' + escape(signature); +}; + +/** + * Shortcut for `new Client()`. + * + * @param {Object} options + * @see Client() + * @api public + */ + +module.exports.createClient = function(options){ + return new Client(options); +}; diff --git a/blob_s3/lib/knox/index.js b/blob_s3/lib/knox/index.js new file mode 100644 index 0000000..6c88054 --- /dev/null +++ b/blob_s3/lib/knox/index.js @@ -0,0 +1,35 @@ +/*! + * knox + * Copyright(c) 2010 LearnBoost + * MIT Licensed + */ + +/** + * Client is the main export. + */ + +module.exports = require('./client'); + +/** + * Library version. + * + * @type String + */ + +module.exports.version = '0.0.5'; + +/** + * Expose utilities. + * + * @type Object + */ + +module.exports.utils = require('./utils'); + +/** + * Expose auth utils. + * + * @type Object + */ + +module.exports.auth = require('./auth'); diff --git a/blob_s3/lib/knox/mime/README.md b/blob_s3/lib/knox/mime/README.md new file mode 100644 index 0000000..8616532 --- /dev/null +++ b/blob_s3/lib/knox/mime/README.md @@ -0,0 +1,19 @@ +A library for doing simple mime-type lookups. + + var mime = require('mime'); + + // all these return 'text/plain' + mime.lookup('/path/to/file.txt'); + mime.lookup('file.txt'); + mime.lookup('.txt'); + +It can also look up common charsets for specific mime-types. (Useful in a web +framework): + + var mime = require('mime'); + + // returns 'UTF-8' + mime.charset.lookup('text/plain'); + +Please help me make sure my lookup tables are complete! Any additions will be +appreciatively merged! diff --git a/blob_s3/lib/knox/mime/index.js b/blob_s3/lib/knox/mime/index.js new file mode 120000 index 0000000..fb92baa --- /dev/null +++ b/blob_s3/lib/knox/mime/index.js @@ -0,0 +1 @@ +mime.js \ No newline at end of file diff --git a/blob_s3/lib/knox/mime/mime.js b/blob_s3/lib/knox/mime/mime.js new file mode 100644 index 0000000..5f58192 --- /dev/null +++ b/blob_s3/lib/knox/mime/mime.js @@ -0,0 +1,306 @@ +var path = require('path'); + +/* Functions for translating file extensions into mime types + * + * based on simonw's djangode: http://github.com/simonw/djangode + * with extensions/mimetypes added from felixge's + * node-paperboy: http://github.com/felixge/node-paperboy + */ +exports.lookup = function(filename, fallback) { + // the path library's extname function won't return an extension if the + // entire string IS the extesion, so we check for that case + var ext; + if (filename.charAt(0) === '.' && filename.substr(1).indexOf('.') < 0) { + ext = filename.substr(1); + } + // either the string doesn't have an extension or it is the extension, we + // assume the latter. Most of the time we're right. + else if (filename.indexOf('.') < 0) { + ext = filename; + } + // simple lookup + else { + ext = path.extname(filename).substr(1); + } + + ext = ext.toLowerCase(); + + return exports.types[ext] || fallback || exports.default_type; +}; + +exports.default_type = 'application/octet-stream'; + +exports.types = + { "3gp" : "video/3gpp" + , "a" : "application/octet-stream" + , "ai" : "application/postscript" + , "aif" : "audio/x-aiff" + , "aiff" : "audio/x-aiff" + , "arj" : "application/x-arj-compressed" + , "asc" : "application/pgp-signature" + , "asf" : "video/x-ms-asf" + , "asm" : "text/x-asm" + , "asx" : "video/x-ms-asf" + , "atom" : "application/atom+xml" + , "au" : "audio/basic" + , "avi" : "video/x-msvideo" + , "bat" : "application/x-msdownload" + , "bcpio" : "application/x-bcpio" + , "bin" : "application/octet-stream" + , "bmp" : "image/bmp" + , "bz2" : "application/x-bzip2" + , "c" : "text/x-c" + , "cab" : "application/vnd.ms-cab-compressed" + , "cc" : "text/x-c" + , "ccad" : "application/clariscad" + , "chm" : "application/vnd.ms-htmlhelp" + , "class" : "application/octet-stream" + , "cod" : "application/vnd.rim.cod" + , "com" : "application/x-msdownload" + , "conf" : "text/plain" + , "cpio" : "application/x-cpio" + , "cpp" : "text/x-c" + , "cpt" : "application/mac-compactpro" + , "crt" : "application/x-x509-ca-cert" + , "csh" : "application/x-csh" + , "css" : "text/css" + , "csv" : "text/csv" + , "cxx" : "text/x-c" + , "deb" : "application/x-debian-package" + , "der" : "application/x-x509-ca-cert" + , "diff" : "text/x-diff" + , "djv" : "image/vnd.djvu" + , "djvu" : "image/vnd.djvu" + , "dl" : "video/dl" + , "dll" : "application/x-msdownload" + , "dmg" : "application/octet-stream" + , "doc" : "application/msword" + , "dot" : "application/msword" + , "drw" : "application/drafting" + , "dtd" : "application/xml-dtd" + , "dvi" : "application/x-dvi" + , "dwg" : "application/acad" + , "dxf" : "application/dxf" + , "dxr" : "application/x-director" + , "ear" : "application/java-archive" + , "eml" : "message/rfc822" + , "eps" : "application/postscript" + , "etx" : "text/x-setext" + , "exe" : "application/x-msdownload" + , "ez" : "application/andrew-inset" + , "f" : "text/x-fortran" + , "f77" : "text/x-fortran" + , "f90" : "text/x-fortran" + , "fli" : "video/x-fli" + , "flv" : "video/x-flv" + , "for" : "text/x-fortran" + , "gem" : "application/octet-stream" + , "gemspec" : "text/x-script.ruby" + , "gif" : "image/gif" + , "gl" : "video/gl" + , "gtar" : "application/x-gtar" + , "gz" : "application/x-gzip" + , "h" : "text/x-c" + , "hdf" : "application/x-hdf" + , "hh" : "text/x-c" + , "hqx" : "application/mac-binhex40" + , "htm" : "text/html" + , "html" : "text/html" + , "ice" : "x-conference/x-cooltalk" + , "ico" : "image/vnd.microsoft.icon" + , "ics" : "text/calendar" + , "ief" : "image/ief" + , "ifb" : "text/calendar" + , "igs" : "model/iges" + , "ips" : "application/x-ipscript" + , "ipx" : "application/x-ipix" + , "iso" : "application/octet-stream" + , "jad" : "text/vnd.sun.j2me.app-descriptor" + , "jar" : "application/java-archive" + , "java" : "text/x-java-source" + , "jnlp" : "application/x-java-jnlp-file" + , "jpeg" : "image/jpeg" + , "jpg" : "image/jpeg" + , "js" : "application/javascript" + , "json" : "application/json" + , "latex" : "application/x-latex" + , "log" : "text/plain" + , "lsp" : "application/x-lisp" + , "lzh" : "application/octet-stream" + , "m" : "text/plain" + , "m3u" : "audio/x-mpegurl" + , "m4v" : "video/mp4" + , "man" : "text/troff" + , "mathml" : "application/mathml+xml" + , "mbox" : "application/mbox" + , "mdoc" : "text/troff" + , "me" : "application/x-troff-me" + , "mid" : "audio/midi" + , "midi" : "audio/midi" + , "mif" : "application/x-mif" + , "mime" : "www/mime" + , "mml" : "application/mathml+xml" + , "mng" : "video/x-mng" + , "mov" : "video/quicktime" + , "movie" : "video/x-sgi-movie" + , "mp3" : "audio/mpeg" + , "mp4" : "video/mp4" + , "mp4v" : "video/mp4" + , "mpeg" : "video/mpeg" + , "mpg" : "video/mpeg" + , "mpga" : "audio/mpeg" + , "ms" : "text/troff" + , "msi" : "application/x-msdownload" + , "nc" : "application/x-netcdf" + , "oda" : "application/oda" + , "odp" : "application/vnd.oasis.opendocument.presentation" + , "ods" : "application/vnd.oasis.opendocument.spreadsheet" + , "odt" : "application/vnd.oasis.opendocument.text" + , "ogg" : "application/ogg" + , "ogm" : "application/ogg" + , "p" : "text/x-pascal" + , "pas" : "text/x-pascal" + , "pbm" : "image/x-portable-bitmap" + , "pdf" : "application/pdf" + , "pem" : "application/x-x509-ca-cert" + , "pgm" : "image/x-portable-graymap" + , "pgn" : "application/x-chess-pgn" + , "pgp" : "application/pgp" + , "pkg" : "application/octet-stream" + , "pl" : "text/x-script.perl" + , "pm" : "application/x-perl" + , "png" : "image/png" + , "pnm" : "image/x-portable-anymap" + , "ppm" : "image/x-portable-pixmap" + , "pps" : "application/vnd.ms-powerpoint" + , "ppt" : "application/vnd.ms-powerpoint" + , "ppz" : "application/vnd.ms-powerpoint" + , "pre" : "application/x-freelance" + , "prt" : "application/pro_eng" + , "ps" : "application/postscript" + , "psd" : "image/vnd.adobe.photoshop" + , "py" : "text/x-script.python" + , "qt" : "video/quicktime" + , "ra" : "audio/x-realaudio" + , "rake" : "text/x-script.ruby" + , "ram" : "audio/x-pn-realaudio" + , "rar" : "application/x-rar-compressed" + , "ras" : "image/x-cmu-raster" + , "rb" : "text/x-script.ruby" + , "rdf" : "application/rdf+xml" + , "rgb" : "image/x-rgb" + , "rm" : "audio/x-pn-realaudio" + , "roff" : "text/troff" + , "rpm" : "application/x-redhat-package-manager" + , "rss" : "application/rss+xml" + , "rtf" : "text/rtf" + , "rtx" : "text/richtext" + , "ru" : "text/x-script.ruby" + , "s" : "text/x-asm" + , "scm" : "application/x-lotusscreencam" + , "set" : "application/set" + , "sgm" : "text/sgml" + , "sgml" : "text/sgml" + , "sh" : "application/x-sh" + , "shar" : "application/x-shar" + , "sig" : "application/pgp-signature" + , "silo" : "model/mesh" + , "sit" : "application/x-stuffit" + , "skt" : "application/x-koan" + , "smil" : "application/smil" + , "snd" : "audio/basic" + , "so" : "application/octet-stream" + , "sol" : "application/solids" + , "spl" : "application/x-futuresplash" + , "src" : "application/x-wais-source" + , "stl" : "application/SLA" + , "stp" : "application/STEP" + , "sv4cpio" : "application/x-sv4cpio" + , "sv4crc" : "application/x-sv4crc" + , "svg" : "image/svg+xml" + , "svgz" : "image/svg+xml" + , "swf" : "application/x-shockwave-flash" + , "t" : "text/troff" + , "tar" : "application/x-tar" + , "tbz" : "application/x-bzip-compressed-tar" + , "tcl" : "application/x-tcl" + , "tex" : "application/x-tex" + , "texi" : "application/x-texinfo" + , "texinfo" : "application/x-texinfo" + , "text" : "text/plain" + , "tgz" : "application/x-tar-gz" + , "tif" : "image/tiff" + , "tiff" : "image/tiff" + , "torrent" : "application/x-bittorrent" + , "tr" : "text/troff" + , "tsi" : "audio/TSP-audio" + , "tsp" : "application/dsptype" + , "tsv" : "text/tab-separated-values" + , "txt" : "text/plain" + , "unv" : "application/i-deas" + , "ustar" : "application/x-ustar" + , "vcd" : "application/x-cdlink" + , "vcf" : "text/x-vcard" + , "vcs" : "text/x-vcalendar" + , "vda" : "application/vda" + , "vivo" : "video/vnd.vivo" + , "vrm" : "x-world/x-vrml" + , "vrml" : "model/vrml" + , "war" : "application/java-archive" + , "wav" : "audio/x-wav" + , "wax" : "audio/x-ms-wax" + , "wma" : "audio/x-ms-wma" + , "wmv" : "video/x-ms-wmv" + , "wmx" : "video/x-ms-wmx" + , "wrl" : "model/vrml" + , "wsdl" : "application/wsdl+xml" + , "wvx" : "video/x-ms-wvx" + , "xbm" : "image/x-xbitmap" + , "xhtml" : "application/xhtml+xml" + , "xls" : "application/vnd.ms-excel" + , "xlw" : "application/vnd.ms-excel" + , "xml" : "application/xml" + , "xpm" : "image/x-xpixmap" + , "xsl" : "application/xml" + , "xslt" : "application/xslt+xml" + , "xwd" : "image/x-xwindowdump" + , "xyz" : "chemical/x-pdb" + , "yaml" : "text/yaml" + , "yml" : "text/yaml" + , "zip" : "application/zip" + }; + +var charsets = exports.charsets = { + lookup: function(mimetype, fallback) { + return charsets.sets[mimetype] || fallback; + }, + + sets: { + "text/calendar": "UTF-8", + "text/css": "UTF-8", + "text/csv": "UTF-8", + "text/html": "UTF-8", + "text/plain": "UTF-8", + "text/rtf": "UTF-8", + "text/richtext": "UTF-8", + "text/sgml": "UTF-8", + "text/tab-separated-values": "UTF-8", + "text/troff": "UTF-8", + "text/x-asm": "UTF-8", + "text/x-c": "UTF-8", + "text/x-diff": "UTF-8", + "text/x-fortran": "UTF-8", + "text/x-java-source": "UTF-8", + "text/x-pascal": "UTF-8", + "text/x-script.perl": "UTF-8", + "text/x-script.perl-module": "UTF-8", + "text/x-script.python": "UTF-8", + "text/x-script.ruby": "UTF-8", + "text/x-setext": "UTF-8", + "text/vnd.sun.j2me.app-descriptor": "UTF-8", + "text/x-vcalendar": "UTF-8", + "text/x-vcard": "UTF-8", + "text/yaml": "UTF-8" + } +}; + diff --git a/blob_s3/lib/knox/mime/package.json b/blob_s3/lib/knox/mime/package.json new file mode 100644 index 0000000..ce2309c --- /dev/null +++ b/blob_s3/lib/knox/mime/package.json @@ -0,0 +1,12 @@ +{ + "name" : "mime", + "description" : "A super simple utility library for dealing with mime-types", + "url" : "http://github.com/bentomas/node-mime", + "keywords" : ["util", "mime"], + "author" : "Benjamin Thomas ", + "contributors" : [], + "dependencies" : [], + "lib" : ".", + "main" : "mime.js", + "version" : "1.0.0" +} diff --git a/blob_s3/lib/knox/utils.js b/blob_s3/lib/knox/utils.js new file mode 100644 index 0000000..3b516a4 --- /dev/null +++ b/blob_s3/lib/knox/utils.js @@ -0,0 +1,70 @@ +/*! + * knox - utils + * Copyright(c) 2010 LearnBoost + * MIT Licensed + */ + +exports.to_query_string = function(options) { + if (options === null || options === undefined) { return ''; } + var filter = ['prefix','max-keys','marker','delimiter','location','logging','response-content-type', 'response-content-language','response-expires','reponse-cache-control','response-content-disposition','response-content-encoding']; + var keys = Object.keys(options); + var query_string = ''; + for (var i = 0, len = keys.length; i < len; ++i) { + var key = keys[i]; + var lkey = key.toLowerCase(); + if (filter.indexOf(lkey) !== -1) { + if (query_string === '') { query_string += '?'; } else { query_string += '&'; } + query_string += lkey + (options[key]?('=' + options[key]):""); + } + } + return query_string; +}; + +/** + * Merge object `b` with object `a`. + * + * @param {Object} a + * @param {Object} b + * @return {Object} a + * @api private + */ + +exports.merge = function(a, b){ + var keys = Object.keys(b); + for (var i = 0, len = keys.length; i < len; ++i) { + var key = keys[i]; + a[key] = b[key]; + } + return a; +}; + +/** + * Base64. + */ + +exports.base64 = { + + /** + * Base64 encode the given `str`. + * + * @param {String} str + * @return {String} + * @api private + */ + + encode: function(str){ + return new Buffer(str).toString('base64'); + }, + + /** + * Base64 decode the given `str`. + * + * @param {String} str + * @return {String} + * @api private + */ + + decode: function(str){ + return new Buffer(str, 'base64').toString(); + } +}; diff --git a/blob_s3/sax-js b/blob_s3/sax-js new file mode 160000 index 0000000..e9680df --- /dev/null +++ b/blob_s3/sax-js @@ -0,0 +1 @@ +Subproject commit e9680df4e1ed0ddd281199871275636b606dfad5 diff --git a/config.json.sample b/config.json.sample new file mode 100644 index 0000000..ebf41e7 --- /dev/null +++ b/config.json.sample @@ -0,0 +1,37 @@ +{ + "drivers":[ + {"fs-sonic" : { + "type" : "fs", + "option" : {"root" : "/home/sonicwcl/Workspace/data2", + "mds" : { + "host" : "127.0.0.1", + "port" : "28100", + "db" : "test2", + "user" : "sonic1", + "pwd" : "password1" + } + } + } + }, + {"s3-sonic" : { + "type" : "s3", + "option" : { + "key" : "dummy", + "secret" : "dummy" + } + } + }, + {"swift-sonic" : { + "type" : "swift", + "option" : { + "username" : "account:user", + "apikey" : "dummy", + "auth_url" : "http://IP:PORT/auth/v1.0" + } + } + } + ], + "port" : "8080", + "default" : "fs", + "logfile" : "/var/vcap/services/blob/instance/log" +} diff --git a/server.js b/server.js new file mode 100644 index 0000000..ae065ca --- /dev/null +++ b/server.js @@ -0,0 +1,321 @@ +/* + Additional modules: express winston +*/ +var winston = require('winston'); +var express = require("express"); +var util = require('util'); +var fs = require('fs'); +var drivers = { }; //storing backend driver objects +var driver_order = { }; //storing the precedence of drivers for resolving bucket name conflict +var default_driver = null; //the driver object for creating a new bucket +var BucketToDriverMap = { }; //bucket name to driver map +var argv = process.argv; +var conf_file = "./config.json"; + +for (var idx = 0; idx < argv.length; idx++) +{ + if (argv[idx] === "-f" && idx+1 < argv.length) + { conf_file = argv[idx+1]; } +} + +var config; +try +{ + config = JSON.parse(fs.readFileSync(conf_file)); +} catch (err) +{ + console.error("error:"+(new Date())+" - Reading configuration error: " + err); + return; +} + +if (config.logfile !== undefined && config.logfile !== null) { + winston.add(winston.transports.File, {filename:config.logfile}).remove(winston.transports.Console); +} + +var ResponseMock = function (dr) { this.driver = dr;}; + +var driver_list_buckets = function(obj) { + var resp = new ResponseMock(obj); + resp.resp_end = function () { + //callback handling list_buckets output + if (resp.resp_code !== 200) { throw resp.resp_body; } + var buckets = resp.resp_body; + buckets = buckets.ListAllMyBucketsResult.Buckets.Bucket; + if (buckets.push === undefined) { buckets = [buckets];} + for (var idx = 0, leng = buckets.length; idx < leng; ++idx) { + var dr = BucketToDriverMap[buckets[idx].Name]; + if (dr === undefined || driver_order[dr.driver.driver_key] > driver_order[resp.driver.driver_key]) { + BucketToDriverMap[buckets[idx].Name] = {"driver" : resp.driver, "CreationDate" :buckets[idx].CreationDate}; + } + } + }; + //handle connection error + /* + Due to a bug in node, 'connection refused' exception cannot be caught by http.request + This will result in terminating the whole process + This function is here for working around the bug: + 1. explicitly test if the destination is reachable/responding + 2. return error if not; otherwise proceed to connect + Every driver must implement this function "pingDest" + */ + obj.pingDest(function(err) { + if (err) { + winston.log('error',(new Date())+" - "+obj.driver_key+".pingDest error: " + err); + //clearInterval(obj["IntervalID"]); + var keys = Object.keys(BucketToDriverMap); + for (var i1 = 0; i1 < keys.length; i1++) + { + var va = BucketToDriverMap[keys[i1]]; + if (va.driver === obj) { + BucketToDriverMap[keys[i1]] = null; delete BucketToDriverMap[keys[i1]]; + } + } + } else + { obj.list_buckets(null,resp); } + }); +}; + +var driver_refresh_BucketToDriverMap = function (key) { + return function (obj) { + obj.driver_key = key; + obj.IntervalID = setInterval(driver_list_buckets, 30000, obj); + driver_list_buckets(obj); + }; +}; + +if (true) { +// var keys = Object.keys(config["drivers"]); + var drs = config.drivers; + for (var i = 0, len = drs.length; i < len; ++i) { + var dr = drs[i]; + var key = Object.keys(dr)[0]; + var value = dr[key]; + driver_order[key] = i; + if (value.type === 's3') { drivers[key] = require('./blob_s3/blob_s3.js').createDriver(value.option, driver_refresh_BucketToDriverMap(key) ); } + else if (value.type === 'fs') { drivers[key] = require('./blob_fs/blob_fs.js').createDriver(value.option, driver_refresh_BucketToDriverMap(key));} + else if (value.type === 'swift') { drivers[key] = require('./blob_sw/blob_sw.js').createDriver(value.option, driver_refresh_BucketToDriverMap(key));} + else { throw "unknown type of driver!"; } + if (default_driver === null) { + if (config["default"] === undefined) {default_driver = drivers[key];} + else if (config["default"].toLowerCase() === value.type) {default_driver = drivers[key];} + else if (!(config["default"].toLowerCase() in {'s3':1,'fs':1,'swift':1})) {default_driver = drivers[key];} + } + } +} + +//TODO: middleware to find proper driver +var app = express.createServer( ); +app.get('/',function(req,res) { + if (req.method === 'HEAD') { + res.header('Connection','close'); + res.end(); + return; + } + //driver.list_buckets(req,res); + res.writeHeader(200, { 'Connection' :'close', 'Content-Type' : 'application/json', 'Date' : new Date().toString() } ); + res.write('{"Buckets" : ['); + var keys = Object.keys(BucketToDriverMap); + for (var i = 0, j=0, len = keys.length; i < len; ++i) { + if (j > 0) { res.write(','); } + if (BucketToDriverMap[keys[i]].book_keeping === true) { continue; } + j++; + res.write('{"Name":"'+keys[i]+'"'); + if (BucketToDriverMap[keys[i]].CreationDate) + { res.write(',"CreationDate":"'+BucketToDriverMap[keys[i]].CreationDate+'"}');} + else { res.write('}'); } + } + res.write(']}'); + res.end(); +}); + +var exists_bucket = function(req,res,next) { + var bucket = BucketToDriverMap[req.params.contain]; + if (bucket === undefined) { + res.header('Connection','close'); + res.statusCode = 404; + res.end('{"Code":"BucketNotFound","Message":"No Such Bucket"}'); + return; + } + req.driver = bucket.driver; + next(); +}; + +var non_exists_bucket = function(req,res,next) { + var bucket = BucketToDriverMap[req.params.contain]; + if (bucket && bucket.driver !== default_driver) { + res.header('Connection','close'); + res.statusCode = 409; + res.end('{"Code":"BucketExists","Message":"Can not create a bucket with existing name"}'); + return; + } + next(); +}; + +var remove_entry = function (container) { + if (BucketToDriverMap[container] === undefined) { + winston.log('warn',(new Date())+" - "+container+" already removed"); + return; + } + if (BucketToDriverMap[container].book_keeping === true) + { delete BucketToDriverMap[container]; } +}; + +var general_resp = function (res) { + return function () { + if (res.client_closed) { return; } + var headers = res.resp_header; + headers.Connection = "close"; + if (headers.connection) { delete headers.Connection; } + res.writeHeader(res.resp_code,headers); + if (res.resp_body) + { res.write(JSON.stringify(res.resp_body)); } + res.end(); + }; +}; + +app.get('/:contain/$', exists_bucket); +app.get('/:contain/$',function(req,res) { + res.client_closed = false; + req.connection.addListener('close', function () { + res.client_closed = true; + }); + if (req.method === 'HEAD') { + res.header('Connection','close'); + res.end(); + return; + } + if (BucketToDriverMap[req.params.contain].book_keeping === true) { + res.header('Connection','close'); + res.statusCode = 503; + res.end('{"Code":"SlowDown","Message":"Bucket temporarily unavailable"}'); + return; + } + var opt = {}; + if (req.query.marker) { opt.marker = req.query.marker; } + if (req.query.prefix) { opt.prefix = req.query.prefix; } + if (req.query.delimiter) { opt.delimiter = req.query.delimiter; } + if (req.query["max-keys"]) { opt["max-keys"] = req.query["max-keys"]; } + if (req.query.location !== undefined) { opt.location = req.query.location; } + if (req.query.logging !== undefined) { opt.logging = req.query.logging; } + res.resp_end = general_resp(res); + req.driver.list_bucket(req.params.contain,opt,res); +}); + +app.get('/:contain/*',exists_bucket); +app.get('/:contain/*',function(req,res) { + res.client_closed = false; + req.connection.addListener('close', function () { + res.client_closed = true; + }); + if (BucketToDriverMap[req.params.contain].book_keeping === true) { + res.header('Connection','close'); + res.statusCode = 503; + res.end('{"Code":"SlowDown","Message":"Bucket temporarily unavailable"}'); + return; + } + res.resp_send = false; + res.resp_handler = function (chunk) { + if (res.resp_send === true) { res.write(chunk); return; } + res.resp_send = true; + var headers = res.resp_header; + headers.Connection = "close"; + if (headers.connection) { delete headers.Connection; } + res.writeHeader(res.resp_code,headers); + res.write(chunk); + }; + res.resp_end = function () { + if (res.client_closed) { return; } + if (res.resp_send === true) { res.end(); return; } + var headers = res.resp_header; + headers.Connection = "close"; + if (headers.connection) { delete headers.Connection; } + res.writeHeader(res.resp_code,headers); + if (res.resp_body) + { res.write(JSON.stringify(res.resp_body)); } + res.end(); + }; + req.driver.read_file(req.params.contain, req.params[0], req.headers.range, req.method.toLowerCase(),res,req); +}); + +app.put('/:contain/*', exists_bucket); +app.put('/:contain/*', function(req,res) { + if (BucketToDriverMap[req.params.contain].book_keeping === true) { + res.header('Connection','close'); + res.statusCode = 503; + res.end('{"Code":"SlowDown","Message":"Bucket temporarily unavailable"}'); + return; + } + //could put following handling to middle ware + //here we only need: copy (src, dest), either intra or inter drivers + if (req.headers['x-amz-copy-source'] ) { + var src = req.headers['x-amz-copy-source']; + var src_buck = src.slice(1,src.indexOf('/',1)); + var src_file = src.substr(src.indexOf('/',1)+1); + //res.header('Connection','closed'); res.write(src_buck+'\n'+src_file+'\n');res.end(); return; + if ( !BucketToDriverMap[src_buck] || + BucketToDriverMap[src_buck].book_keeping === true ) + { + res.header('Connection','close'); + res.statusCode = 404; + res.end('{"Code":"BucketNotFound","Message":"Source bucket not found"}'); + return; + } + //copy object, for now assume intra backend + var driver2 = BucketToDriverMap[src_buck].driver; + if ((driver2 !== req.driver)) { + res.header('Connection','close'); + res.statusCode = 501; + res.end('{"Code":"NotImplemented","Message":"Copying across backends is not implemented"}'); + return; + } + res.resp_end = general_resp(res); + req.driver.copy_file(req.params.contain, req.params[0], src_buck, src_file, req,res); + //res.header('Connection','closed'); res.write('copy from bucket '+ src_buck + ' file ' + src_file);res.end(); return; + } else { + res.resp_end = general_resp(res); + req.driver.create_file(req.params.contain,req.params[0],req,res); + } +}); + +app.put('/:contain', non_exists_bucket); +app.put('/:contain',function(req,res) { + BucketToDriverMap[req.params.contain] = { "driver" : default_driver, "book_keeping" : true}; + res.resp_end = general_resp(res); + default_driver.create_bucket(req.params.contain,res,req); //res then req + //heuristic + setTimeout(driver_list_buckets,4000,default_driver); + setTimeout(remove_entry,6000,req.params.contain); //still have race condition +}); + +app.delete('/:contain/*',exists_bucket); +app.delete('/:contain/*',function(req,res) { + if (BucketToDriverMap[req.params.contain].book_keeping === true) { + res.header('Connection','close'); + res.statusCode = 503; + res.end('{"Code":"SlowDown","Message":"Bucket temporarily unavailable"}'); + return; + } + res.resp_end = general_resp(res); + req.driver.delete_file(req.params.contain,req.params[0],res); +}); + +app.delete('/:contain', exists_bucket); +app.delete('/:contain',function(req,res) { + if (BucketToDriverMap[req.params.contain].book_keeping === true) { + res.header('Connection','close'); + res.statusCode = 404; + res.end('{"Code":"BucketNotFound","Message":"No Such Bucket"}'); + return; + } + BucketToDriverMap[req.params.contain].book_keeping = true; + res.resp_end = general_resp(res); + req.driver.delete_bucket(req.params.contain,res); + //heuristic + setTimeout(driver_list_buckets,4000,req.driver); + setTimeout(remove_entry,6000,req.params.contain); //still have race condition +}); + +winston.log('info',(new Date())+' - listening to port ' + config.port); +if (config.port) +{ app.listen(parseInt(config.port,10));} //should load from config file +exports.vblob_gateway = app;